mcp-scraper 0.3.5 → 0.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +18 -26
  2. package/dist/bin/api-server.cjs +164 -329
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +1 -1
  5. package/dist/bin/browser-agent-stdio-server.cjs +39 -1750
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +2 -4
  8. package/dist/bin/browser-agent-stdio-server.js.map +1 -1
  9. package/dist/bin/mcp-scraper-cli.cjs +47 -328
  10. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-cli.js +7 -91
  12. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +78 -1796
  14. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  15. package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -6
  16. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  17. package/dist/bin/mcp-scraper-install.cjs +4 -4
  18. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  19. package/dist/bin/mcp-scraper-install.js +2 -2
  20. package/dist/bin/mcp-stdio-server.cjs +18 -25
  21. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  22. package/dist/bin/mcp-stdio-server.js +2 -2
  23. package/dist/chunk-2NEVRPK3.js +7 -0
  24. package/dist/chunk-2NEVRPK3.js.map +1 -0
  25. package/dist/{chunk-WZT3XKES.js → chunk-2O7RAK6E.js} +19 -26
  26. package/dist/chunk-2O7RAK6E.js.map +1 -0
  27. package/dist/{chunk-RVYHP7QN.js → chunk-ELI6NDEG.js} +4 -4
  28. package/dist/{chunk-RVYHP7QN.js.map → chunk-ELI6NDEG.js.map} +1 -1
  29. package/dist/{chunk-CMK5MMEE.js → chunk-IWNLHCKA.js} +13 -694
  30. package/dist/chunk-IWNLHCKA.js.map +1 -0
  31. package/dist/{server-BC4Z2L4P.js → server-HRVD52CX.js} +842 -122
  32. package/dist/server-HRVD52CX.js.map +1 -0
  33. package/docs/mcp-tool-craft-lint.generated.md +2 -5
  34. package/docs/mcp-tool-manifest.generated.json +9 -101
  35. package/package.json +1 -1
  36. package/dist/chunk-CMK5MMEE.js.map +0 -1
  37. package/dist/chunk-DUEW4EOO.js +0 -214
  38. package/dist/chunk-DUEW4EOO.js.map +0 -1
  39. package/dist/chunk-GHBU5SGQ.js +0 -7
  40. package/dist/chunk-GHBU5SGQ.js.map +0 -1
  41. package/dist/chunk-NXRWFOEZ.js +0 -816
  42. package/dist/chunk-NXRWFOEZ.js.map +0 -1
  43. package/dist/chunk-WZT3XKES.js.map +0 -1
  44. package/dist/server-BC4Z2L4P.js.map +0 -1
@@ -1,32 +1,10 @@
1
1
  #!/usr/bin/env node
2
2
  "use strict";
3
- var __create = Object.create;
4
- var __defProp = Object.defineProperty;
5
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
6
- var __getOwnPropNames = Object.getOwnPropertyNames;
7
- var __getProtoOf = Object.getPrototypeOf;
8
- var __hasOwnProp = Object.prototype.hasOwnProperty;
9
- var __copyProps = (to, from, except, desc) => {
10
- if (from && typeof from === "object" || typeof from === "function") {
11
- for (let key of __getOwnPropNames(from))
12
- if (!__hasOwnProp.call(to, key) && key !== except)
13
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
14
- }
15
- return to;
16
- };
17
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
18
- // If the importer is in node compatibility mode or this is not an ESM
19
- // file that has been converted to a CommonJS file using a Babel-
20
- // compatible transform (i.e. "__esModule" has not been set), then set
21
- // "default" to the CommonJS "module.exports" for node compatibility.
22
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
23
- mod
24
- ));
25
3
 
26
4
  // bin/mcp-scraper-combined-stdio-server.ts
27
- var import_node_fs7 = require("fs");
28
- var import_node_os7 = require("os");
29
- var import_node_path9 = require("path");
5
+ var import_node_fs4 = require("fs");
6
+ var import_node_os4 = require("os");
7
+ var import_node_path5 = require("path");
30
8
  var import_mcp3 = require("@modelcontextprotocol/sdk/server/mcp.js");
31
9
  var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
32
10
 
@@ -274,41 +252,9 @@ var HttpMcpToolExecutor = class {
274
252
 
275
253
  // src/mcp/browser-agent-mcp-server.ts
276
254
  var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
277
- var import_node_fs4 = require("fs");
278
- var import_node_os5 = require("os");
279
- var import_node_path6 = require("path");
280
-
281
- // src/lib/chrome-profiles.ts
282
- var import_promises = require("fs/promises");
283
- var import_node_os = require("os");
284
- var import_node_path = require("path");
285
- function defaultChromeUserDataDir() {
286
- if (process.platform === "darwin") return (0, import_node_path.join)((0, import_node_os.homedir)(), "Library", "Application Support", "Google", "Chrome");
287
- if (process.platform === "win32") {
288
- return (0, import_node_path.join)(process.env.LOCALAPPDATA ?? (0, import_node_path.join)((0, import_node_os.homedir)(), "AppData", "Local"), "Google", "Chrome", "User Data");
289
- }
290
- return (0, import_node_path.join)(process.env.XDG_CONFIG_HOME ?? (0, import_node_path.join)((0, import_node_os.homedir)(), ".config"), "google-chrome");
291
- }
292
- function recommendedKernelProfileName(value) {
293
- return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
294
- }
295
- async function listLocalChromeProfiles(userDataDir = defaultChromeUserDataDir(), emailFilter) {
296
- const raw = await (0, import_promises.readFile)((0, import_node_path.join)(userDataDir, "Local State"), "utf8");
297
- const localState = JSON.parse(raw);
298
- const wanted = emailFilter?.trim().toLowerCase();
299
- return Object.entries(localState.profile?.info_cache ?? {}).map(([directory, info]) => {
300
- const email = (info.user_name ?? "").trim();
301
- const displayName = (info.name ?? info.shortcut_name ?? "").trim();
302
- const labelSource = email || displayName || directory;
303
- return {
304
- email,
305
- displayName,
306
- chromeProfileDirectory: directory,
307
- chromeProfilePath: (0, import_node_path.join)(userDataDir, directory),
308
- recommendedKernelProfileName: recommendedKernelProfileName(labelSource)
309
- };
310
- }).filter((row) => !wanted || row.email.toLowerCase() === wanted || row.displayName.toLowerCase().includes(wanted));
311
- }
255
+ var import_node_fs = require("fs");
256
+ var import_node_os2 = require("os");
257
+ var import_node_path2 = require("path");
312
258
 
313
259
  // src/lib/browser-service-env.ts
314
260
  function browserServiceProfileName() {
@@ -323,1363 +269,8 @@ function browserServiceProfileSaveChanges() {
323
269
  return void 0;
324
270
  }
325
271
 
326
- // src/lib/local-browser-profiles.ts
327
- var import_node_fs = require("fs");
328
- var import_promises2 = require("fs/promises");
329
- var import_node_os2 = require("os");
330
- var import_node_path2 = require("path");
331
- var EXCLUDED_PROFILE_NAMES = [
332
- "Cache",
333
- "Code Cache",
334
- "Crashpad",
335
- "DawnCache",
336
- "DawnGraphiteCache",
337
- "DawnWebGPUCache",
338
- "GPUCache",
339
- "GrShaderCache",
340
- "ShaderCache",
341
- "Shared Dictionary"
342
- ];
343
- function defaultImportedBrowserProfilesDir() {
344
- return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), ".mcp-scraper", "browser-profiles");
345
- }
346
- function defaultChromeExecutablePath() {
347
- const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
348
- if (envPath) return envPath;
349
- const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
350
- (0, import_node_path2.join)(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
351
- (0, import_node_path2.join)(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
352
- ] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
353
- return candidates.find((candidate) => (0, import_node_fs.existsSync)(candidate));
354
- }
355
- function localBrowserModeEnabled() {
356
- return process.env.MCP_SCRAPER_BROWSER_MODE?.trim().toLowerCase() === "local";
357
- }
358
- function importedProfileName(value) {
359
- return recommendedKernelProfileName(value);
360
- }
361
- function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
362
- return (0, import_node_path2.join)(outputDir, importedProfileName(name));
363
- }
364
- function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
365
- return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "manifest.json");
366
- }
367
- function importedProfileUserDataDir(name, outputDir = defaultImportedBrowserProfilesDir()) {
368
- return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "user-data");
369
- }
370
- async function readExistingManifest(path) {
371
- try {
372
- return JSON.parse(await (0, import_promises2.readFile)(path, "utf8"));
373
- } catch {
374
- return null;
375
- }
376
- }
377
- function isExcludedProfilePath(sourceRoot, sourcePath) {
378
- const rel = (0, import_node_path2.relative)(sourceRoot, sourcePath);
379
- if (!rel) return false;
380
- const parts = rel.split(import_node_path2.sep).filter(Boolean);
381
- const last = parts[parts.length - 1] ?? "";
382
- if (last.startsWith("Singleton")) return true;
383
- if (last === "LOCK" || last.endsWith(".lock")) return true;
384
- if (parts.some((part) => EXCLUDED_PROFILE_NAMES.includes(part))) return true;
385
- if (parts.includes("Service Worker") && parts.includes("CacheStorage")) return true;
386
- return false;
387
- }
388
- async function copyProfileTree(sourceProfilePath, destinationProfilePath) {
389
- await (0, import_promises2.cp)(sourceProfilePath, destinationProfilePath, {
390
- recursive: true,
391
- force: true,
392
- errorOnExist: false,
393
- dereference: false,
394
- filter: (source) => !isExcludedProfilePath(sourceProfilePath, source)
395
- });
396
- }
397
- function selectProfile(profiles, options) {
398
- const requestedDirectory = options.chromeProfileDirectory?.trim();
399
- const requestedEmail = options.email?.trim().toLowerCase();
400
- const exactDirectory = requestedDirectory ? profiles.find((profile) => profile.chromeProfileDirectory === requestedDirectory) : void 0;
401
- if (exactDirectory) return exactDirectory;
402
- const exactEmail = requestedEmail ? profiles.find((profile) => profile.email.toLowerCase() === requestedEmail) : void 0;
403
- if (exactEmail) return exactEmail;
404
- const first = profiles[0];
405
- if (first) return first;
406
- const hint = requestedDirectory ? `Chrome profile directory "${requestedDirectory}"` : requestedEmail ? `Chrome account "${requestedEmail}"` : "Chrome profile";
407
- throw new Error(`${hint} was not found in ${options.sourceUserDataDir ?? defaultChromeUserDataDir()}`);
408
- }
409
- async function ensureProfilePathExists(profile) {
410
- const details = await (0, import_promises2.stat)(profile.chromeProfilePath).catch(() => null);
411
- if (!details?.isDirectory()) {
412
- throw new Error(`Chrome profile path does not exist or is not a directory: ${profile.chromeProfilePath}`);
413
- }
414
- }
415
- async function writeNormalizedLocalState(sourceLocalStatePath, destinationLocalStatePath, sourceProfileDirectory) {
416
- const raw = await (0, import_promises2.readFile)(sourceLocalStatePath, "utf8");
417
- const localState = JSON.parse(raw);
418
- const selectedInfo = localState.profile?.info_cache?.[sourceProfileDirectory] ?? localState.profile?.info_cache?.Default ?? {};
419
- localState.profile = {
420
- ...localState.profile ?? {},
421
- info_cache: { Default: selectedInfo },
422
- last_used: "Default",
423
- last_active_profiles: ["Default"]
424
- };
425
- await (0, import_promises2.writeFile)(destinationLocalStatePath, `${JSON.stringify(localState, null, 2)}
426
- `);
427
- }
428
- async function importChromeProfile(options = {}) {
429
- const sourceUserDataDir = options.sourceUserDataDir?.trim() || defaultChromeUserDataDir();
430
- const profiles = await listLocalChromeProfiles(sourceUserDataDir, options.email);
431
- const sourceProfile = selectProfile(profiles, { ...options, sourceUserDataDir });
432
- await ensureProfilePathExists(sourceProfile);
433
- const name = importedProfileName(options.name?.trim() || sourceProfile.recommendedKernelProfileName);
434
- const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
435
- const profileRoot = importedProfileRoot(name, outputDir);
436
- const manifestPath = importedProfileManifestPath(name, outputDir);
437
- const userDataDir = importedProfileUserDataDir(name, outputDir);
438
- const existingManifest = await readExistingManifest(manifestPath);
439
- const existing = (0, import_node_fs.existsSync)(profileRoot);
440
- if (existing && !options.overwrite) {
441
- throw new Error(`Imported browser profile "${name}" already exists at ${profileRoot}. Re-run with --overwrite to refresh it.`);
442
- }
443
- if (existing) await (0, import_promises2.rm)(profileRoot, { recursive: true, force: true });
444
- await (0, import_promises2.mkdir)((0, import_node_path2.join)(userDataDir, "Default"), { recursive: true });
445
- await writeNormalizedLocalState(
446
- (0, import_node_path2.join)(sourceUserDataDir, "Local State"),
447
- (0, import_node_path2.join)(userDataDir, "Local State"),
448
- sourceProfile.chromeProfileDirectory
449
- );
450
- await copyProfileTree(sourceProfile.chromeProfilePath, (0, import_node_path2.join)(userDataDir, "Default"));
451
- const now = (/* @__PURE__ */ new Date()).toISOString();
452
- const manifest = {
453
- layoutVersion: 1,
454
- name,
455
- profileRoot,
456
- userDataDir,
457
- browserExecutablePath: options.browserExecutablePath?.trim() || defaultChromeExecutablePath() || null,
458
- sourceUserDataDir,
459
- sourceProfileDirectory: sourceProfile.chromeProfileDirectory,
460
- sourceProfilePath: sourceProfile.chromeProfilePath,
461
- email: sourceProfile.email,
462
- displayName: sourceProfile.displayName,
463
- createdAt: existingManifest?.createdAt ?? now,
464
- updatedAt: now,
465
- copyMode: "chrome-profile-clone"
466
- };
467
- await (0, import_promises2.writeFile)(manifestPath, `${JSON.stringify(manifest, null, 2)}
468
- `);
469
- return {
470
- profile: manifest,
471
- sourceProfile,
472
- excludedNames: EXCLUDED_PROFILE_NAMES
473
- };
474
- }
475
- async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
476
- const profileName = importedProfileName(name);
477
- const manifestPath = importedProfileManifestPath(profileName, outputDir);
478
- const manifest = await readExistingManifest(manifestPath);
479
- if (!manifest) {
480
- throw new Error(
481
- `Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
482
- );
483
- }
484
- return manifest;
485
- }
486
- async function syncImportedChromeProfile(name, options = {}) {
487
- const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
488
- const manifest = await loadImportedBrowserProfile(name, outputDir);
489
- return importChromeProfile({
490
- name: manifest.name,
491
- sourceUserDataDir: manifest.sourceUserDataDir,
492
- chromeProfileDirectory: manifest.sourceProfileDirectory,
493
- outputDir,
494
- overwrite: true,
495
- browserExecutablePath: options.browserExecutablePath ?? manifest.browserExecutablePath ?? void 0
496
- });
497
- }
498
-
499
- // src/services/browser-agent/local-browser-agent-service.ts
500
- var import_node_crypto = require("crypto");
501
- var import_node_fs3 = require("fs");
502
- var import_node_path4 = require("path");
503
- var import_playwright_extra = require("playwright-extra");
504
- var import_puppeteer_extra_plugin_stealth = __toESM(require("puppeteer-extra-plugin-stealth"), 1);
505
-
506
- // src/services/fanout/cdp-capture.ts
507
- var ANSWER_RE = /\/backend-api\/(f\/)?conversation$|\/chat_conversations\/[^/]+\/(completion|retry_completion)$|\/completion$/i;
508
- var FanoutCdpCapture = class {
509
- client = null;
510
- reqUrl = /* @__PURE__ */ new Map();
511
- ctById = /* @__PURE__ */ new Map();
512
- answerIds = /* @__PURE__ */ new Set();
513
- bodies = [];
514
- pending = [];
515
- answerStreams = 0;
516
- async attach(page) {
517
- if (this.client) return;
518
- const client = await page.context().newCDPSession(page);
519
- this.client = client;
520
- await client.send("Network.enable", { maxTotalBufferSize: 2e8, maxResourceBufferSize: 2e8 });
521
- client.on("Network.requestWillBeSent", (e) => {
522
- const id = e.requestId;
523
- const req = e.request;
524
- if (id) this.reqUrl.set(id, req?.url || "");
525
- });
526
- client.on("Network.responseReceived", (e) => {
527
- const id = e.requestId;
528
- const res = e.response;
529
- const url = res?.url || this.reqUrl.get(id) || "";
530
- const headers = res?.headers || {};
531
- const ct = String(headers["content-type"] || headers["Content-Type"] || "");
532
- if (id) this.ctById.set(id, ct);
533
- if (id && (this.isAnswerUrl(url) || /event-stream/i.test(ct))) this.answerIds.add(id);
534
- });
535
- client.on("Network.loadingFinished", (e) => this.grab(e.requestId));
536
- }
537
- reset() {
538
- this.bodies = [];
539
- this.answerStreams = 0;
540
- this.answerIds.clear();
541
- this.pending = [];
542
- }
543
- answerStreamCount() {
544
- return this.answerStreams;
545
- }
546
- async drain() {
547
- await Promise.all(this.pending);
548
- }
549
- bestAnswerText() {
550
- const eventStreams = this.bodies.filter((b) => /event-stream/i.test(b.ct));
551
- const answerStreams = eventStreams.filter((b) => this.isAnswerUrl(b.url)).sort((a, b) => b.text.length - a.text.length);
552
- if (answerStreams.length) return answerStreams.map((b) => b.text).join("\n");
553
- const answerJson = this.bodies.filter((b) => this.isAnswerUrl(b.url)).sort((a, b) => b.text.length - a.text.length);
554
- if (answerJson.length) return answerJson[0].text;
555
- if (eventStreams.length) return eventStreams.sort((a, b) => b.text.length - a.text.length)[0].text;
556
- return "";
557
- }
558
- async detach() {
559
- try {
560
- await this.client?.detach();
561
- } catch {
562
- this.client = null;
563
- }
564
- this.client = null;
565
- }
566
- isAnswerUrl(url) {
567
- let path = "";
568
- try {
569
- path = new URL(url).pathname;
570
- } catch {
571
- path = String(url || "").split("?")[0];
572
- }
573
- return ANSWER_RE.test(path);
574
- }
575
- grab(requestId) {
576
- if (!requestId || !this.client || !this.answerIds.has(requestId)) return;
577
- const url = this.reqUrl.get(requestId) || "";
578
- const ct = this.ctById.get(requestId) || "";
579
- const client = this.client;
580
- const p = client.send("Network.getResponseBody", { requestId }).then((r) => {
581
- const text = r.base64Encoded ? Buffer.from(String(r.body), "base64").toString("utf8") : String(r.body || "");
582
- if (text) {
583
- this.bodies.push({ url, ct, text });
584
- if (this.isAnswerUrl(url)) this.answerStreams++;
585
- }
586
- }).catch(() => void 0);
587
- this.pending.push(p);
588
- }
589
- };
590
-
591
- // src/services/fanout/parse-helpers.ts
592
- var INTERNAL = /chatgpt\.com|openai\.com|oaiusercontent|oaistatic|claude\.ai|anthropic\.com/i;
593
- var TRACK = /^(utm_|oai|fbclid|gclid|ref$|ref_|sa$|usg$|ved$|sca_esv$|sources$)/i;
594
- function domainOf(u) {
595
- try {
596
- return new URL(u).hostname.replace(/^www\./, "");
597
- } catch {
598
- return "";
599
- }
600
- }
601
- function normUrl(u) {
602
- try {
603
- const x = new URL(u);
604
- x.hash = "";
605
- for (const k of [...x.searchParams.keys()]) {
606
- if (TRACK.test(k)) x.searchParams.delete(k);
607
- }
608
- return x.toString().replace(/\/$/, "").toLowerCase();
609
- } catch {
610
- return String(u || "").toLowerCase();
611
- }
612
- }
613
- function isExternal(u) {
614
- const h = domainOf(u);
615
- return Boolean(h) && !INTERNAL.test(h);
616
- }
617
- function unesc(s) {
618
- try {
619
- return JSON.parse('"' + s + '"');
620
- } catch {
621
- return String(s).replace(/\\(.)/g, "$1");
622
- }
623
- }
624
- function stripCitationMarkers(s) {
625
- return String(s || "").replace(/[-]/g, "").replace(/cite(?:turn\d+\w+?\d+)+/g, "").replace(/\s+/g, " ").trim();
626
- }
627
- function sliceBalanced(text, openIdx) {
628
- const open = text[openIdx];
629
- const close = open === "[" ? "]" : "}";
630
- let depth = 0;
631
- let inStr = false;
632
- let esc2 = false;
633
- for (let i = openIdx; i < text.length; i++) {
634
- const ch = text[i];
635
- if (inStr) {
636
- if (esc2) esc2 = false;
637
- else if (ch === "\\") esc2 = true;
638
- else if (ch === '"') inStr = false;
639
- continue;
640
- }
641
- if (ch === '"') {
642
- inStr = true;
643
- continue;
644
- }
645
- if (ch === open) depth++;
646
- else if (ch === close) {
647
- depth--;
648
- if (depth === 0) return text.slice(openIdx, i + 1);
649
- }
650
- }
651
- return null;
652
- }
653
- function findJsonValues(text, field) {
654
- const out = [];
655
- const re = new RegExp('"' + field + '"\\s*:\\s*(\\[|\\{)', "g");
656
- let m;
657
- while (m = re.exec(text)) {
658
- const openIdx = m.index + m[0].length - 1;
659
- const slice = sliceBalanced(text, openIdx);
660
- if (slice) {
661
- try {
662
- out.push(JSON.parse(slice));
663
- } catch {
664
- continue;
665
- }
666
- }
667
- }
668
- return out;
669
- }
670
- function stringsInArrayBody(arrBody) {
671
- const out = [];
672
- const re = /"((?:[^"\\]|\\.){2,300}?)"/g;
673
- let m;
674
- while (m = re.exec(arrBody)) out.push(unesc(m[1]));
675
- return out;
676
- }
677
- function cleanQuery(q) {
678
- const t = (q || "").trim();
679
- if (t.length < 2 || t.length > 300) return null;
680
- return t;
681
- }
682
-
683
- // src/services/fanout/adapters/chatgpt.ts
684
- function roundOf(refId) {
685
- const m = /^turn(\d+)/.exec(refId || "");
686
- return m ? Number(m[1]) : null;
687
- }
688
- function normRef(r) {
689
- if (typeof r === "string") {
690
- const m = /turn(\d+)(search|news|ref|video|image|product)(\d+)/.exec(r);
691
- return m ? `turn${m[1]}${m[2]}${m[3]}` : null;
692
- }
693
- if (r && typeof r === "object") {
694
- const o = r;
695
- if (o.turn_index != null) {
696
- return `turn${o.turn_index}${o.ref_type || "search"}${o.ref_index != null ? o.ref_index : ""}`;
697
- }
698
- }
699
- return null;
700
- }
701
- function refsInText(s) {
702
- return (String(s || "").match(/turn\d+(?:search|news|ref|video|image|product)\d+/g) || []).map(normRef).filter((x) => Boolean(x));
703
- }
704
- function collectInto(arr, v) {
705
- if (Array.isArray(v)) arr.push(...v);
706
- else if (v && typeof v === "object") arr.push(v);
707
- }
708
- function reconstructFromSsePatches(raw) {
709
- const contentReferences = [];
710
- const searchResultGroups = [];
711
- const citations = [];
712
- const citedUrls = /* @__PURE__ */ new Set();
713
- const state = { found: false };
714
- const handleOp = (op) => {
715
- if (!op || typeof op !== "object") return;
716
- const o = op;
717
- const v = o.v;
718
- if (o.o === "patch" && Array.isArray(v)) {
719
- state.found = true;
720
- v.forEach(handleOp);
721
- return;
722
- }
723
- const path = typeof o.p === "string" ? o.p : "";
724
- if ((path === "" || path === "/message") && v && typeof v === "object") {
725
- const root = v;
726
- const msg = root.message || root;
727
- const md = msg && msg.metadata;
728
- if (md) {
729
- collectInto(contentReferences, md.content_references);
730
- collectInto(searchResultGroups, md.search_result_groups);
731
- collectInto(citations, md.citations);
732
- state.found = true;
733
- }
734
- }
735
- if (!path) return;
736
- if (/\/safe_urls$/.test(path) && Array.isArray(v)) {
737
- for (const u of v) if (typeof u === "string") citedUrls.add(u);
738
- state.found = true;
739
- } else if (path.includes("content_references")) {
740
- collectInto(contentReferences, v);
741
- state.found = true;
742
- } else if (path.includes("search_result_groups")) {
743
- collectInto(searchResultGroups, v);
744
- state.found = true;
745
- } else if (path.includes("citations")) {
746
- collectInto(citations, v);
747
- state.found = true;
748
- }
749
- };
750
- for (const line of raw.split("\n")) {
751
- const s = (line.startsWith("data:") ? line.slice(5) : line).trim();
752
- if (!s || s === "[DONE]" || s[0] !== "{" && s[0] !== "[") continue;
753
- let parsed;
754
- try {
755
- parsed = JSON.parse(s);
756
- } catch {
757
- continue;
758
- }
759
- const ops = Array.isArray(parsed) ? parsed : [parsed];
760
- ops.forEach(handleOp);
761
- }
762
- if (!state.found) return raw;
763
- const citedRefs = [...citedUrls].map((url) => ({ type: "grouped_webpages", items: [{ url }] }));
764
- const synthetic = JSON.stringify({
765
- content_references: [...contentReferences, ...citedRefs],
766
- search_result_groups: searchResultGroups,
767
- citations
768
- });
769
- return raw + "\n" + synthetic;
770
- }
771
- var chatgptAdapter = {
772
- platform: "ChatGPT",
773
- matchesHost(hostname) {
774
- return /(^|\.)chatgpt\.com$|(^|\.)openai\.com$/i.test(hostname);
775
- },
776
- parse(rawInput, prompt) {
777
- const raw = reconstructFromSsePatches(rawInput);
778
- const sources = /* @__PURE__ */ new Map();
779
- const snippets = /* @__PURE__ */ new Map();
780
- const queries = /* @__PURE__ */ new Set();
781
- const unmapped = /* @__PURE__ */ new Set();
782
- const meta = { model: "", finishType: "", title: "", rounds: 0 };
783
- const ensure = (url) => {
784
- if (!isExternal(url)) return null;
785
- const k = normUrl(url);
786
- let s = sources.get(k);
787
- if (!s) {
788
- s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, firstOrder: Infinity, snippet: "", round: null };
789
- sources.set(k, s);
790
- }
791
- return s;
792
- };
793
- const addSnippet = (url, title, text) => {
794
- const clean = stripCitationMarkers(text);
795
- if (!isExternal(url) || clean.length < 2) return;
796
- snippets.set(normUrl(url), { url, domain: domainOf(url), title: title || domainOf(url), text: clean.slice(0, 400) });
797
- };
798
- for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
799
- for (const m of raw.matchAll(/"search_model_queries"\s*:\s*\[([^\]]*)\]/g)) {
800
- for (const q of stringsInArrayBody(m[1])) {
801
- const c = cleanQuery(q);
802
- if (c) queries.add(c);
803
- }
804
- }
805
- for (const m of raw.matchAll(/"(?:search_queries|queries|search_query)"\s*:\s*\[([^\]]*)\]/g)) {
806
- for (const q of stringsInArrayBody(m[1])) {
807
- const c = cleanQuery(q);
808
- if (c) queries.add(c);
809
- }
810
- }
811
- for (const v of findJsonValues(raw, "search_result_groups")) {
812
- if (!Array.isArray(v)) continue;
813
- for (const g of v) {
814
- const entries = g && typeof g === "object" && Array.isArray(g.entries) ? g.entries : [];
815
- for (const e of entries) {
816
- const url = typeof e.url === "string" ? e.url : "";
817
- const s = ensure(url);
818
- if (!s) continue;
819
- if (!s.title && typeof e.title === "string") s.title = e.title.slice(0, 200);
820
- const ref = normRef(e.ref_id);
821
- if (ref && s.round == null) s.round = roundOf(ref);
822
- if (typeof e.snippet === "string" && e.snippet) addSnippet(url, s.title, e.snippet);
823
- }
824
- }
825
- }
826
- for (const m of raw.matchAll(/"url"\s*:\s*"(https?:\/\/[^"\\]+)"(?:[\s\S]{0,120}?"(?:title|name)"\s*:\s*"((?:[^"\\]|\\.){0,200})")?/g)) {
827
- const s = ensure(m[1]);
828
- if (s && !s.title && m[2]) s.title = unesc(m[2]).slice(0, 200);
829
- }
830
- let spanTotal = 0;
831
- for (const cr of findJsonValues(raw, "content_references")) {
832
- if (!Array.isArray(cr)) continue;
833
- for (const ref of cr) {
834
- const type = typeof ref.type === "string" ? ref.type : "";
835
- const isBundle = /sources_footnote|nav_list|alt_text/.test(type);
836
- const refs = refsInText(typeof ref.matched_text === "string" ? ref.matched_text : "");
837
- if (!isBundle) spanTotal++;
838
- const ord = typeof ref.start_idx === "number" ? ref.start_idx : isBundle ? Infinity : spanTotal;
839
- const items = Array.isArray(ref.items) ? ref.items : [];
840
- items.forEach((it, idx) => {
841
- const refForItem = refs.length === items.length ? refs[idx] : refs[0];
842
- const urls = [it.url].concat((Array.isArray(it.supporting_websites) ? it.supporting_websites : []).map((s) => s?.url)).filter((u) => typeof u === "string");
843
- for (const u of urls) {
844
- const s = ensure(u);
845
- if (!s) continue;
846
- if (!isBundle) {
847
- s.cited = true;
848
- s.timesCited++;
849
- if (ord < s.firstOrder) s.firstOrder = ord;
850
- } else if (!s.cited) {
851
- s.cited = true;
852
- }
853
- if (refForItem && s.round == null) s.round = roundOf(refForItem);
854
- }
855
- if (typeof it.url === "string" && typeof it.snippet === "string") addSnippet(it.url, typeof it.title === "string" ? it.title : "", it.snippet);
856
- });
857
- }
858
- }
859
- for (const v of findJsonValues(raw, "citations")) {
860
- if (!Array.isArray(v)) continue;
861
- v.forEach((c, i) => {
862
- const md = c && typeof c === "object" ? c.metadata : null;
863
- if (!md || typeof md.url !== "string") return;
864
- const s = ensure(md.url);
865
- if (!s) return;
866
- s.cited = true;
867
- s.timesCited++;
868
- const ord = typeof c.start_ix === "number" ? c.start_ix : i;
869
- if (ord < s.firstOrder) s.firstOrder = ord;
870
- if (typeof md.text === "string") addSnippet(md.url, typeof md.title === "string" ? md.title : "", md.text);
871
- });
872
- }
873
- for (const m of raw.matchAll(/"content_type"\s*:\s*"tether_quote"([\s\S]{0,4000}?)"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g)) {
874
- const url = (m[1].match(/"url"\s*:\s*"([^"]+)"/) || [])[1];
875
- const title = (m[1].match(/"title"\s*:\s*"((?:[^"\\]|\\.)*)"/) || [])[1];
876
- if (url) {
877
- ensure(url);
878
- addSnippet(url, title ? unesc(title) : "", unesc(m[2]));
879
- }
880
- }
881
- const modelMatches = [...raw.matchAll(/"resolved_model_slug"\s*:\s*"([^"]+)"/g)];
882
- const slugMatches = modelMatches.length ? modelMatches : [...raw.matchAll(/"model_slug"\s*:\s*"([^"]+)"/g)];
883
- if (slugMatches.length) meta.model = slugMatches[slugMatches.length - 1][1];
884
- const finish = raw.match(/"finish_details"\s*:\s*\{[^}]*"type"\s*:\s*"([^"]+)"/);
885
- if (finish) meta.finishType = finish[1];
886
- const searchRounds = (raw.match(/"recipient"\s*:\s*"(?:web|browser)"[\s\S]{0,400}?"content_type"\s*:\s*"code"[\s\S]{0,200}?search\(/g) || []).length;
887
- const sourceRounds = /* @__PURE__ */ new Set();
888
- for (const s of sources.values()) if (s.round != null) sourceRounds.add(s.round);
889
- meta.rounds = Math.max(searchRounds, sourceRounds.size);
890
- const ordered = [...sources.values()].sort((a, b) => a.cited === b.cited ? b.timesCited - a.timesCited || a.firstOrder - b.firstOrder : a.cited ? -1 : 1).map((s) => ({
891
- url: s.url,
892
- domain: s.domain,
893
- title: s.title || s.domain,
894
- cited: s.cited,
895
- timesCited: s.timesCited,
896
- snippet: snippets.get(normUrl(s.url))?.text || "",
897
- round: s.round
898
- }));
899
- return {
900
- subQueries: [...queries],
901
- sources: ordered,
902
- snippets: [...snippets.values()],
903
- meta,
904
- unmappedKeys: [...unmapped].slice(0, 50)
905
- };
906
- }
907
- };
908
-
909
- // src/services/fanout/adapters/claude.ts
910
- function isFavicon(u) {
911
- return /\/s2\/favicons/i.test(u) || /favicon/i.test(u);
912
- }
913
- var claudeAdapter = {
914
- platform: "Claude",
915
- matchesHost(hostname) {
916
- return /(^|\.)claude\.ai$/i.test(hostname);
917
- },
918
- parse(raw, prompt) {
919
- const sources = /* @__PURE__ */ new Map();
920
- const snippets = /* @__PURE__ */ new Map();
921
- const queries = /* @__PURE__ */ new Set();
922
- const unmapped = /* @__PURE__ */ new Set();
923
- const meta = { model: "", finishType: "", title: "", rounds: 0 };
924
- const ensure = (url) => {
925
- if (!url || !isExternal(url) || isFavicon(url)) return null;
926
- const k = normUrl(url);
927
- let s = sources.get(k);
928
- if (!s) {
929
- s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, snippet: "", round: null };
930
- sources.set(k, s);
931
- }
932
- return s;
933
- };
934
- const toolInputs = /* @__PURE__ */ new Map();
935
- const searchBlocks = /* @__PURE__ */ new Set();
936
- const finalizeTool = (index) => {
937
- const t = toolInputs.get(index);
938
- if (!t || !/web_search|web_fetch|search|browse/i.test(t.name)) return;
939
- try {
940
- const input = JSON.parse(t.buf);
941
- if (typeof input.query === "string") {
942
- const q = cleanQuery(input.query);
943
- if (q) queries.add(q);
944
- }
945
- } catch {
946
- const m = t.buf.match(/"query"\s*:\s*"((?:[^"\\]|\\.)*)"/);
947
- if (m) {
948
- const q = cleanQuery(unesc(m[1]));
949
- if (q) queries.add(q);
950
- }
951
- }
952
- };
953
- for (const line of raw.split("\n")) {
954
- const s = (line.startsWith("data:") ? line.slice(5) : line).trim();
955
- if (!s || s[0] !== "{" && s[0] !== "[") continue;
956
- let ev;
957
- try {
958
- ev = JSON.parse(s);
959
- } catch {
960
- continue;
961
- }
962
- const type = ev.type;
963
- if (type === "content_block_start") {
964
- const cb = ev.content_block || {};
965
- const index = typeof ev.index === "number" ? ev.index : -1;
966
- if (cb.type === "tool_use" && index >= 0) {
967
- const name = typeof cb.name === "string" ? cb.name : "";
968
- toolInputs.set(index, { name, buf: "" });
969
- if (/web_search|web_fetch|search|browse/i.test(name)) searchBlocks.add(index);
970
- }
971
- } else if (type === "content_block_delta") {
972
- const d = ev.delta || {};
973
- const index = typeof ev.index === "number" ? ev.index : -1;
974
- if (d.type === "input_json_delta" && typeof d.partial_json === "string") {
975
- const t = toolInputs.get(index);
976
- if (t) t.buf += d.partial_json;
977
- } else if (d.type === "citation_start_delta") {
978
- const c = d.citation || {};
979
- if (typeof c.url === "string") {
980
- const src = ensure(c.url);
981
- if (src) {
982
- src.cited = true;
983
- src.timesCited++;
984
- if (!src.title && typeof c.title === "string") src.title = c.title.slice(0, 200);
985
- if (typeof c.cited_text === "string" && c.cited_text) {
986
- snippets.set(normUrl(c.url), { url: c.url, domain: domainOf(c.url), title: src.title, text: String(c.cited_text).slice(0, 400) });
987
- }
988
- }
989
- }
990
- }
991
- } else if (type === "content_block_stop") {
992
- const index = typeof ev.index === "number" ? ev.index : -1;
993
- if (index >= 0) finalizeTool(index);
994
- } else if (type === "message_delta" || type === "message_stop" || type === "message_start") {
995
- const d = ev.delta || ev.message || {};
996
- if (typeof d.stop_reason === "string" && !meta.finishType) meta.finishType = d.stop_reason;
997
- if (typeof d.model === "string" && !meta.model) meta.model = d.model;
998
- }
999
- }
1000
- for (const index of searchBlocks) finalizeTool(index);
1001
- for (const m of raw.matchAll(/"title"\s*:\s*"((?:[^"\\]|\\.){1,200})"\s*,\s*"url"\s*:\s*"(https?:\/\/[^"\\]+)"/g)) {
1002
- const src = ensure(m[2]);
1003
- if (src && !src.title) src.title = unesc(m[1]).slice(0, 200);
1004
- }
1005
- for (const m of raw.matchAll(/"url"\s*:\s*"(https?:\/\/[^"\\]+)"/g)) ensure(m[1]);
1006
- for (const m of raw.matchAll(/\\"title\\"\s*:\s*\\"((?:[^"\\]|\\.){1,160}?)\\"[\s\S]{0,160}?\\"url\\"\s*:\s*\\"(https?:[^"\\]+)\\"/g)) {
1007
- const src = ensure(m[2]);
1008
- if (src && !src.title) src.title = unesc(unesc(m[1])).slice(0, 200);
1009
- }
1010
- for (const m of raw.matchAll(/\\"url\\"\s*:\s*\\"(https?:[^"\\]+)\\"/g)) ensure(m[1]);
1011
- if (!meta.model) {
1012
- const mm = raw.match(/"model"\s*:\s*"(claude[^"]+)"/);
1013
- if (mm) meta.model = mm[1];
1014
- }
1015
- meta.rounds = queries.size > 0 ? searchBlocks.size : 0;
1016
- for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
1017
- const ordered = [...sources.values()].sort((a, b) => a.cited === b.cited ? b.timesCited - a.timesCited : a.cited ? -1 : 1).map((s) => ({
1018
- url: s.url,
1019
- domain: s.domain,
1020
- title: s.title || s.domain,
1021
- cited: s.cited,
1022
- timesCited: s.timesCited,
1023
- snippet: snippets.get(normUrl(s.url))?.text || "",
1024
- round: s.round
1025
- }));
1026
- return {
1027
- subQueries: [...queries],
1028
- sources: ordered,
1029
- snippets: [...snippets.values()],
1030
- meta,
1031
- unmappedKeys: [...unmapped].slice(0, 50)
1032
- };
1033
- }
1034
- };
1035
-
1036
- // src/services/fanout/adapters/index.ts
1037
- var FANOUT_ADAPTERS = [chatgptAdapter, claudeAdapter];
1038
- function adapterForHost(hostname) {
1039
- return FANOUT_ADAPTERS.find((a) => a.matchesHost(hostname)) ?? null;
1040
- }
1041
-
1042
- // src/services/fanout/build.ts
1043
- function buildFanoutResult(ctx) {
1044
- const { platform, parsed, ready, rawBytes } = ctx;
1045
- const browsedUrls = parsed.sources;
1046
- const citedUrls = browsedUrls.filter((s) => s.cited);
1047
- const browsedOnly = browsedUrls.filter((s) => !s.cited);
1048
- const prompt = ctx.prompt || parsed.meta.title || "";
1049
- const result = {
1050
- platform,
1051
- capturedAt: (/* @__PURE__ */ new Date()).toISOString(),
1052
- prompt,
1053
- meta: parsed.meta,
1054
- subQueries: parsed.subQueries,
1055
- browsedUrls,
1056
- citedUrls,
1057
- browsedOnly,
1058
- snippets: parsed.snippets,
1059
- counts: {
1060
- subQueries: parsed.subQueries.length,
1061
- browsed: browsedUrls.length,
1062
- cited: citedUrls.length,
1063
- browsedOnly: browsedOnly.length
1064
- }
1065
- };
1066
- if (parsed.subQueries.length === 0 && browsedUrls.length === 0) {
1067
- result.debug = {
1068
- interceptorReady: ready,
1069
- rawBytes,
1070
- unmappedKeys: parsed.unmappedKeys,
1071
- note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call browser_capture_fanout again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page. The session must be opened with MCP_SCRAPER_BROWSER_MODE=local and navigated to chatgpt.com or claude.ai after opening, so the hook installs before the site scripts run."
1072
- };
1073
- }
1074
- return result;
1075
- }
1076
-
1077
- // src/services/fanout/classify.ts
1078
- var SOCIAL = /(^|\.)(youtube\.com|youtu\.be|instagram\.com|tiktok\.com|facebook\.com|fb\.com|x\.com|twitter\.com|linkedin\.com|pinterest\.com|threads\.net|vimeo\.com)$/i;
1079
- var ENCYCLOPEDIA = /(^|\.)(wikipedia\.org|wikimedia\.org|britannica\.com|merriam-webster\.com|dictionary\.com|fandom\.com|investopedia\.com)$/i;
1080
- var REVIEW = /(^|\.)(yelp\.com|trustpilot\.com|g2\.com|capterra\.com|angi\.com|angieslist\.com|bbb\.org|consumeraffairs\.com|bestcompany\.com|sitejabber\.com|clutch\.co|bestpickreports\.com|homeguide\.com|porch\.com|ontoplist\.com|thumbtack\.com)$/i;
1081
- var NEWS = /(^|\.)(reuters\.com|apnews\.com|nytimes\.com|wsj\.com|washingtonpost\.com|bloomberg\.com|cnbc\.com|forbes\.com|bbc\.com|theguardian\.com|cnn\.com|npr\.org|usatoday\.com|expressnews\.com|mysanantonio\.com|beaumontenterprise\.com)$/i;
1082
- function classifyDomain(domain, firstPartyDomain) {
1083
- const d = domain.toLowerCase().replace(/^www\./, "");
1084
- if (firstPartyDomain) {
1085
- const fp = firstPartyDomain.toLowerCase().replace(/^www\./, "");
1086
- if (fp && (d === fp || d.endsWith("." + fp))) return "First-party/vendor";
1087
- }
1088
- if (/(^|\.)reddit\.com$/i.test(d)) return "Reddit";
1089
- if (SOCIAL.test(d)) return "Social/video";
1090
- if (ENCYCLOPEDIA.test(d)) return "Encyclopedia";
1091
- if (REVIEW.test(d)) return "Review site";
1092
- if (NEWS.test(d) || /(^|\.)news\./i.test(d)) return "News/media";
1093
- if (/^docs?\.|^developer\.|^developers\.|readthedocs|\.dev$/i.test(d)) return "Docs";
1094
- return "Blog";
1095
- }
1096
- function emptyCategoryCounts() {
1097
- return { "First-party/vendor": 0, "News/media": 0, Reddit: 0, "Social/video": 0, Encyclopedia: 0, "Review site": 0, Docs: 0, Blog: 0 };
1098
- }
1099
- function enrichFanout(result, opts = {}) {
1100
- const addSiteType = (s) => ({ ...s, siteType: classifyDomain(s.domain, opts.firstPartyDomain) });
1101
- const browsedUrls = result.browsedUrls.map(addSiteType);
1102
- const citedUrls = result.citedUrls.map(addSiteType);
1103
- const browsedOnly = result.browsedOnly.map(addSiteType);
1104
- const byCategory = emptyCategoryCounts();
1105
- const domainAgg = /* @__PURE__ */ new Map();
1106
- for (const s of browsedUrls) {
1107
- byCategory[s.siteType]++;
1108
- const cur = domainAgg.get(s.domain);
1109
- if (!cur) domainAgg.set(s.domain, { domain: s.domain, count: 1, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType });
1110
- else {
1111
- cur.count++;
1112
- cur.cited = cur.cited || s.cited;
1113
- cur.timesCited += s.timesCited;
1114
- }
1115
- }
1116
- const topSites = [...domainAgg.values()].sort((a, b) => b.count - a.count || b.timesCited - a.timesCited);
1117
- const citationOrder = citedUrls.slice().sort((a, b) => b.timesCited - a.timesCited).map((s, i) => ({ rank: i + 1, domain: s.domain, url: s.url, timesCited: s.timesCited }));
1118
- return {
1119
- platform: result.platform,
1120
- capturedAt: result.capturedAt,
1121
- prompt: result.prompt,
1122
- meta: result.meta,
1123
- queries: result.subQueries,
1124
- browsedUrls,
1125
- citedUrls,
1126
- browsedOnly,
1127
- snippets: result.snippets,
1128
- counts: result.counts,
1129
- aggregates: { topSites, citationOrder, byCategory },
1130
- firstPartyDomain: opts.firstPartyDomain ? opts.firstPartyDomain.toLowerCase().replace(/^www\./, "") : null,
1131
- ...result.debug ? { debug: result.debug } : {}
1132
- };
1133
- }
1134
-
1135
- // src/services/fanout/export.ts
1136
- var import_node_fs2 = require("fs");
1137
- var import_node_os3 = require("os");
1138
- var import_node_path3 = require("path");
1139
- var import_papaparse = __toESM(require("papaparse"), 1);
1140
- function outputBaseDir() {
1141
- return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
1142
- }
1143
- function safe(value) {
1144
- return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
1145
- }
1146
- function writeTable(path, rows, delimiter) {
1147
- (0, import_node_fs2.writeFileSync)(path, import_papaparse.default.unparse(rows.length ? rows : [{}], { delimiter }));
1148
- }
1149
- function exportFanout(enriched) {
1150
- const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
1151
- const dir = (0, import_node_path3.join)(outputBaseDir(), "fanout", `${stamp}-${safe(enriched.platform)}`);
1152
- (0, import_node_fs2.mkdirSync)(dir, { recursive: true });
1153
- const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
1154
- const citationRows = enriched.aggregates.citationOrder.map((c) => {
1155
- const src = enriched.citedUrls.find((s) => s.url === c.url);
1156
- return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
1157
- });
1158
- const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
1159
- const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
1160
- const paths = {
1161
- dir,
1162
- json: (0, import_node_path3.join)(dir, "fanout.json"),
1163
- queriesCsv: (0, import_node_path3.join)(dir, "queries.csv"),
1164
- queriesTsv: (0, import_node_path3.join)(dir, "queries.tsv"),
1165
- citationsCsv: (0, import_node_path3.join)(dir, "citations.csv"),
1166
- sourcesCsv: (0, import_node_path3.join)(dir, "sources.csv"),
1167
- domainsCsv: (0, import_node_path3.join)(dir, "domains.csv"),
1168
- report: (0, import_node_path3.join)(dir, "report.html")
1169
- };
1170
- (0, import_node_fs2.writeFileSync)(paths.json, JSON.stringify(enriched, null, 2));
1171
- writeTable(paths.queriesCsv, queryRows, ",");
1172
- writeTable(paths.queriesTsv, queryRows, " ");
1173
- writeTable(paths.citationsCsv, citationRows, ",");
1174
- writeTable(paths.sourcesCsv, sourceRows, ",");
1175
- writeTable(paths.domainsCsv, domainRows, ",");
1176
- (0, import_node_fs2.writeFileSync)(paths.report, renderReportHtml(enriched));
1177
- return paths;
1178
- }
1179
- function esc(s) {
1180
- return String(s).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
1181
- }
1182
- function renderReportHtml(enriched) {
1183
- const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
1184
- return `<!doctype html>
1185
- <html lang="en">
1186
- <head>
1187
- <meta charset="utf-8" />
1188
- <meta name="viewport" content="width=device-width, initial-scale=1" />
1189
- <title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
1190
- <style>
1191
- :root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
1192
- *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
1193
- .wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
1194
- h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
1195
- .stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
1196
- .cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
1197
- .card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
1198
- .card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
1199
- .bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
1200
- .bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
1201
- .bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
1202
- .tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
1203
- .tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
1204
- .chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
1205
- .chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
1206
- .chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
1207
- .row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
1208
- .row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
1209
- .tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
1210
- .dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
1211
- .pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
1212
- .copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
1213
- .hide{display:none}
1214
- </style>
1215
- </head>
1216
- <body><div class="wrap">
1217
- <h1>AI Search Fan-Out</h1>
1218
- <p class="sub" id="sub"></p>
1219
- <div class="stat" id="stat"></div>
1220
- <div class="cards" id="cards"></div>
1221
- <div id="charts"></div>
1222
- <div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
1223
- <div class="chips" id="chips"></div>
1224
- <div id="list"></div>
1225
- </div>
1226
- <script>
1227
- const D=${data};
1228
- const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
1229
- document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
1230
- document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
1231
- const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
1232
- const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
1233
- function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
1234
- const charts=document.getElementById('charts');
1235
- charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
1236
- let tab='q',filter=null;
1237
- const chips=document.getElementById('chips'),list=document.getElementById('list');
1238
- function render(){
1239
- chips.innerHTML='';list.innerHTML='';
1240
- if(tab==='q'){
1241
- D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
1242
- } else {
1243
- const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
1244
- Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
1245
- D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
1246
- }
1247
- }
1248
- function esc(s){return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;')}
1249
- document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
1250
- render();
1251
- </script>
1252
- </body></html>`;
1253
- }
1254
-
1255
- // src/services/fanout/run-capture.ts
1256
- var COMPOSER_SELECTORS = [
1257
- "#prompt-textarea",
1258
- "div#prompt-textarea",
1259
- 'textarea[data-testid="prompt-textarea"]',
1260
- 'div.ProseMirror[contenteditable="true"]',
1261
- 'div[contenteditable="true"]',
1262
- "textarea"
1263
- ];
1264
- async function sendPrompt(page, text) {
1265
- for (const selector of COMPOSER_SELECTORS) {
1266
- const locator = page.locator(selector).first();
1267
- try {
1268
- await locator.waitFor({ state: "visible", timeout: 12e3 });
1269
- } catch {
1270
- continue;
1271
- }
1272
- await locator.click({ timeout: 4e3 }).catch(() => void 0);
1273
- await locator.focus().catch(() => void 0);
1274
- await page.keyboard.type(text, { delay: 12 });
1275
- await page.waitForTimeout(300).catch(() => void 0);
1276
- await page.keyboard.press("Enter");
1277
- return true;
1278
- }
1279
- return false;
1280
- }
1281
- async function waitForAnswer(page, cap, baseline, waitMs) {
1282
- const start = Date.now();
1283
- while (Date.now() - start < waitMs && cap.answerStreamCount() <= baseline) {
1284
- await page.waitForTimeout(400).catch(() => void 0);
1285
- }
1286
- await page.waitForTimeout(1e3).catch(() => void 0);
1287
- }
1288
- async function runFanoutCapture(page, input) {
1289
- const cap = new FanoutCdpCapture();
1290
- await cap.attach(page);
1291
- try {
1292
- const baseline = cap.answerStreamCount();
1293
- if (input.prompt) await sendPrompt(page, input.prompt).catch(() => false);
1294
- const waitMs = input.wait_ms ?? (input.prompt ? 9e4 : 8e3);
1295
- await waitForAnswer(page, cap, baseline, waitMs);
1296
- await cap.drain();
1297
- let hostname = "";
1298
- try {
1299
- hostname = new URL(page.url()).hostname;
1300
- } catch {
1301
- hostname = "";
1302
- }
1303
- const adapter = adapterForHost(hostname);
1304
- if (!adapter) {
1305
- throw new Error(`browser_capture_fanout supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
1306
- }
1307
- const text = cap.bestAnswerText();
1308
- const parsed = adapter.parse(text, input.prompt || "");
1309
- const result = buildFanoutResult({ platform: adapter.platform, parsed, prompt: input.prompt || "", ready: true, rawBytes: text.length });
1310
- const enriched = enrichFanout(result, { firstPartyDomain: input.first_party_domain });
1311
- const exports2 = input.export ? exportFanout(enriched) : null;
1312
- return { result: enriched, exports: exports2 };
1313
- } finally {
1314
- await cap.detach().catch(() => void 0);
1315
- }
1316
- }
1317
-
1318
- // src/services/browser-agent/local-browser-agent-service.ts
1319
- import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
1320
- function directProfileDir() {
1321
- return process.env.MCP_SCRAPER_BROWSER_PROFILE_DIR?.trim() || void 0;
1322
- }
1323
- function configuredProfileName(profile) {
1324
- return profile?.trim() || process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || (directProfileDir() ? (0, import_node_path4.basename)(directProfileDir()) : "default");
1325
- }
1326
- function launchExecutablePath(profile) {
1327
- const configured = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || profile.browserExecutablePath || defaultChromeExecutablePath();
1328
- return configured && (0, import_node_fs3.existsSync)(configured) ? configured : void 0;
1329
- }
1330
- function ignoreDefaultArgsFor(executablePath) {
1331
- if (!executablePath) return void 0;
1332
- if (process.platform !== "darwin") return void 0;
1333
- if (!/Google Chrome\.app\/Contents\/MacOS\/Google Chrome$/.test(executablePath)) return void 0;
1334
- return ["--password-store=basic", "--use-mock-keychain"];
1335
- }
1336
- function normalizeKeyCombo(value) {
1337
- return value.split("+").map((part) => {
1338
- const trimmed = part.trim();
1339
- const lower = trimmed.toLowerCase();
1340
- if (lower === "ctrl" || lower === "control") return "Control";
1341
- if (lower === "cmd" || lower === "command" || lower === "meta") return "Meta";
1342
- if (lower === "option" || lower === "alt") return "Alt";
1343
- if (lower === "return") return "Enter";
1344
- if (lower === "esc") return "Escape";
1345
- if (trimmed.length === 1) return trimmed.toUpperCase();
1346
- return trimmed;
1347
- }).join("+");
1348
- }
1349
- function sessionSummary(session) {
1350
- return {
1351
- session_id: session.sessionId,
1352
- status: session.status,
1353
- label: session.label,
1354
- profile: session.profileName,
1355
- user_data_dir: session.userDataDir,
1356
- created_at: session.createdAt,
1357
- last_action_at: session.lastActionAt,
1358
- closed_at: session.closedAt,
1359
- watch_url: `local://browser-session/${session.sessionId}`,
1360
- url: session.status === "open" ? session.page.url() : null
1361
- };
1362
- }
1363
- var LocalBrowserAgentService = class {
1364
- contexts = /* @__PURE__ */ new Map();
1365
- sessions = /* @__PURE__ */ new Map();
1366
- async open(input) {
1367
- const profile = await this.resolveProfile(input.profile);
1368
- const contextState = await this.ensureContext(profile);
1369
- const pages = contextState.context.pages();
1370
- const page = pages.length === 1 && !this.hasOpenSessionForContext(contextState.key) ? pages[0] : await contextState.context.newPage();
1371
- const now = (/* @__PURE__ */ new Date()).toISOString();
1372
- const sessionId = `lbs_${(0, import_node_crypto.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
1373
- const session = {
1374
- sessionId,
1375
- label: input.label ?? null,
1376
- page,
1377
- contextKey: contextState.key,
1378
- profileName: profile.name,
1379
- userDataDir: profile.userDataDir,
1380
- createdAt: now,
1381
- lastActionAt: now,
1382
- closedAt: null,
1383
- status: "open"
1384
- };
1385
- const timeoutSeconds = input.timeout_seconds ?? 600;
1386
- if (timeoutSeconds > 0) {
1387
- session.timeout = setTimeout(() => {
1388
- void this.close(sessionId);
1389
- }, timeoutSeconds * 1e3);
1390
- session.timeout.unref();
1391
- }
1392
- this.sessions.set(sessionId, session);
1393
- if (input.url) await this.goto(sessionId, input.url);
1394
- return {
1395
- session_id: sessionId,
1396
- watch_url: `local://browser-session/${sessionId}`,
1397
- live_view_url: null,
1398
- profile: profile.name,
1399
- user_data_dir: profile.userDataDir,
1400
- url: input.url ?? null,
1401
- mode: "local"
1402
- };
1403
- }
1404
- async screenshot(sessionId) {
1405
- const session = this.requireOpenSession(sessionId);
1406
- await this.waitForPage(session.page);
1407
- const image = await session.page.screenshot({ type: "png", fullPage: false });
1408
- const page = await this.read(sessionId);
1409
- return {
1410
- ...page,
1411
- image_base64: image.toString("base64"),
1412
- mime_type: "image/png"
1413
- };
1414
- }
1415
- async read(sessionId) {
1416
- const session = this.requireOpenSession(sessionId);
1417
- await this.waitForPage(session.page);
1418
- this.touch(session);
1419
- const title = await session.page.title().catch(() => "");
1420
- const snapshot = await session.page.evaluate(() => {
1421
- const cssEscape = (value) => {
1422
- const escapeFn = globalThis.CSS?.escape;
1423
- return escapeFn ? escapeFn(value) : value.replace(/["\\#.:>+~[\]()]/g, "\\$&");
1424
- };
1425
- const textFor = (element) => {
1426
- const html = element;
1427
- return (element.getAttribute("aria-label") || element.getAttribute("title") || html.placeholder || html.value || html.innerText || element.textContent || "").replace(/\s+/g, " ").trim();
1428
- };
1429
- const selectorFor = (element) => {
1430
- const id = element.id;
1431
- if (id) return `#${cssEscape(id)}`;
1432
- const name = element.getAttribute("name");
1433
- const tag = element.tagName.toLowerCase();
1434
- if (name) return `${tag}[name="${name.replace(/"/g, '\\"')}"]`;
1435
- const parts = [];
1436
- let current = element;
1437
- while (current && current !== document.body && parts.length < 4) {
1438
- const currentTag = current.tagName.toLowerCase();
1439
- const parent = current.parentElement;
1440
- if (!parent) {
1441
- parts.unshift(currentTag);
1442
- break;
1443
- }
1444
- const children = Array.from(parent.children);
1445
- const siblings = children.filter((child) => child.tagName === current.tagName);
1446
- const index = siblings.indexOf(current) + 1;
1447
- parts.unshift(siblings.length > 1 ? `${currentTag}:nth-of-type(${index})` : currentTag);
1448
- current = parent;
1449
- }
1450
- return parts.join(" > ") || tag;
1451
- };
1452
- const candidates = Array.from(document.querySelectorAll(
1453
- 'a, button, input, textarea, select, summary, [role], [contenteditable="true"], [tabindex]:not([tabindex="-1"])'
1454
- ));
1455
- const elements = candidates.map((element, index) => {
1456
- const rect = element.getBoundingClientRect();
1457
- const style = window.getComputedStyle(element);
1458
- const visible = rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden" && Number(style.opacity || "1") > 0;
1459
- if (!visible) return null;
1460
- const label = textFor(element);
1461
- return {
1462
- index,
1463
- tag: element.tagName.toLowerCase(),
1464
- role: element.getAttribute("role") || null,
1465
- text: label.slice(0, 180),
1466
- selector: selectorFor(element),
1467
- x: Math.round(rect.left + rect.width / 2),
1468
- y: Math.round(rect.top + rect.height / 2),
1469
- left: Math.round(rect.left),
1470
- top: Math.round(rect.top),
1471
- width: Math.round(rect.width),
1472
- height: Math.round(rect.height)
1473
- };
1474
- }).filter(Boolean).slice(0, 120);
1475
- return {
1476
- text: (document.body?.innerText || "").replace(/\n{3,}/g, "\n\n").slice(0, 3e4),
1477
- elements,
1478
- viewport: {
1479
- width: window.innerWidth,
1480
- height: window.innerHeight,
1481
- devicePixelRatio: window.devicePixelRatio,
1482
- scrollX: window.scrollX,
1483
- scrollY: window.scrollY
1484
- }
1485
- };
1486
- });
1487
- return {
1488
- url: session.page.url(),
1489
- title,
1490
- text: snapshot.text,
1491
- elements: snapshot.elements,
1492
- viewport: snapshot.viewport
1493
- };
1494
- }
1495
- async locate(sessionId, targets) {
1496
- const session = this.requireOpenSession(sessionId);
1497
- this.touch(session);
1498
- const located = [];
1499
- for (const target of targets) {
1500
- try {
1501
- const locator = target.selector ? session.page.locator(target.selector) : session.page.getByText(target.text ?? "", { exact: target.match === "exact" });
1502
- const selected = locator.nth(target.index ?? 0);
1503
- await selected.waitFor({ state: "visible", timeout: 3e3 }).catch(() => void 0);
1504
- const box = await selected.boundingBox();
1505
- const text = await selected.textContent({ timeout: 1e3 }).catch(() => null);
1506
- located.push({
1507
- input: target,
1508
- found: Boolean(box),
1509
- element: box ? {
1510
- left: Math.round(box.x),
1511
- top: Math.round(box.y),
1512
- width: Math.round(box.width),
1513
- height: Math.round(box.height),
1514
- x: Math.round(box.x + box.width / 2),
1515
- y: Math.round(box.y + box.height / 2),
1516
- text: (text ?? "").replace(/\s+/g, " ").trim().slice(0, 220)
1517
- } : null
1518
- });
1519
- } catch (err) {
1520
- located.push({
1521
- input: target,
1522
- found: false,
1523
- element: null,
1524
- error: err instanceof Error ? err.message : String(err)
1525
- });
1526
- }
1527
- }
1528
- const read = await this.read(sessionId);
1529
- return {
1530
- url: read.url,
1531
- title: read.title,
1532
- viewport: read.viewport,
1533
- replay: null,
1534
- targets: located
1535
- };
1536
- }
1537
- async goto(sessionId, url) {
1538
- const session = this.requireOpenSession(sessionId);
1539
- this.touch(session);
1540
- const response = await session.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 }).catch((err) => {
1541
- throw new Error(err instanceof Error ? err.message : String(err));
1542
- });
1543
- await this.waitForPage(session.page);
1544
- return {
1545
- ok: true,
1546
- url: session.page.url(),
1547
- status: response?.status() ?? null,
1548
- mode: "local"
1549
- };
1550
- }
1551
- async click(sessionId, input) {
1552
- const session = this.requireOpenSession(sessionId);
1553
- this.touch(session);
1554
- await session.page.mouse.click(input.x, input.y, {
1555
- button: input.button ?? "left",
1556
- clickCount: input.num_clicks ?? 1
1557
- });
1558
- await this.waitForPage(session.page);
1559
- return { ok: true, mode: "local", url: session.page.url() };
1560
- }
1561
- async type(sessionId, input) {
1562
- const session = this.requireOpenSession(sessionId);
1563
- this.touch(session);
1564
- await session.page.keyboard.type(input.text, { delay: input.delay });
1565
- return { ok: true, mode: "local", url: session.page.url() };
1566
- }
1567
- async scroll(sessionId, input) {
1568
- const session = this.requireOpenSession(sessionId);
1569
- this.touch(session);
1570
- if (typeof input.x === "number" && typeof input.y === "number") {
1571
- await session.page.mouse.move(input.x, input.y);
1572
- }
1573
- await session.page.mouse.wheel((input.delta_x ?? 0) * 120, (input.delta_y ?? 5) * 120);
1574
- await session.page.waitForTimeout(150).catch(() => void 0);
1575
- return { ok: true, mode: "local", url: session.page.url() };
1576
- }
1577
- async press(sessionId, input) {
1578
- const session = this.requireOpenSession(sessionId);
1579
- this.touch(session);
1580
- for (const key of input.keys) {
1581
- await session.page.keyboard.press(normalizeKeyCombo(key));
1582
- }
1583
- await this.waitForPage(session.page);
1584
- return { ok: true, mode: "local", url: session.page.url() };
1585
- }
1586
- async close(sessionId) {
1587
- const session = this.sessions.get(sessionId);
1588
- if (!session || session.status === "closed") {
1589
- return { ok: true, closed: true, already_closed: true, mode: "local" };
1590
- }
1591
- if (session.timeout) clearTimeout(session.timeout);
1592
- session.status = "closed";
1593
- session.closedAt = (/* @__PURE__ */ new Date()).toISOString();
1594
- await session.page.close().catch(() => void 0);
1595
- await this.closeContextIfUnused(session.contextKey);
1596
- return { ok: true, closed: true, mode: "local" };
1597
- }
1598
- list(includeClosed = false) {
1599
- return Array.from(this.sessions.values()).filter((session) => includeClosed || session.status === "open").map(sessionSummary);
1600
- }
1601
- async captureFanout(sessionId, input) {
1602
- const session = this.requireOpenSession(sessionId);
1603
- this.touch(session);
1604
- return runFanoutCapture(session.page, input);
1605
- }
1606
- async screenshotOnly(sessionId) {
1607
- const session = this.requireOpenSession(sessionId);
1608
- await this.waitForPage(session.page);
1609
- const img = await session.page.screenshot({ type: "png", fullPage: false });
1610
- return img.toString("base64");
1611
- }
1612
- async resolveProfile(profile) {
1613
- const directDir = directProfileDir();
1614
- const requestedName = configuredProfileName(profile);
1615
- if (directDir && (!profile || requestedName === process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim())) {
1616
- return {
1617
- name: requestedName,
1618
- userDataDir: directDir,
1619
- browserExecutablePath: process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || defaultChromeExecutablePath() || null,
1620
- manifest: null
1621
- };
1622
- }
1623
- const manifest = await loadImportedBrowserProfile(requestedName);
1624
- return {
1625
- name: manifest.name,
1626
- userDataDir: manifest.userDataDir,
1627
- browserExecutablePath: manifest.browserExecutablePath,
1628
- manifest
1629
- };
1630
- }
1631
- async ensureContext(profile) {
1632
- const existing = this.contexts.get(profile.userDataDir);
1633
- if (existing) return existing;
1634
- const executablePath = launchExecutablePath(profile);
1635
- const context = await import_playwright_extra.chromium.launchPersistentContext(profile.userDataDir, {
1636
- headless: false,
1637
- viewport: { width: 1440, height: 900 },
1638
- acceptDownloads: true,
1639
- ...executablePath ? { executablePath } : {},
1640
- ...ignoreDefaultArgsFor(executablePath) ? { ignoreDefaultArgs: ignoreDefaultArgsFor(executablePath) } : {}
1641
- });
1642
- const state = { key: profile.userDataDir, context, profile };
1643
- context.on("close", () => {
1644
- this.contexts.delete(profile.userDataDir);
1645
- const closedAt = (/* @__PURE__ */ new Date()).toISOString();
1646
- for (const session of this.sessions.values()) {
1647
- if (session.contextKey === profile.userDataDir && session.status === "open") {
1648
- session.status = "closed";
1649
- session.closedAt = closedAt;
1650
- if (session.timeout) clearTimeout(session.timeout);
1651
- }
1652
- }
1653
- });
1654
- this.contexts.set(profile.userDataDir, state);
1655
- return state;
1656
- }
1657
- requireOpenSession(sessionId) {
1658
- const session = this.sessions.get(sessionId);
1659
- if (!session || session.status !== "open") throw new Error(`Local browser session not found or closed: ${sessionId}`);
1660
- return session;
1661
- }
1662
- touch(session) {
1663
- session.lastActionAt = (/* @__PURE__ */ new Date()).toISOString();
1664
- }
1665
- hasOpenSessionForContext(contextKey) {
1666
- return Array.from(this.sessions.values()).some((session) => session.contextKey === contextKey && session.status === "open");
1667
- }
1668
- async closeContextIfUnused(contextKey) {
1669
- if (this.hasOpenSessionForContext(contextKey)) return;
1670
- const state = this.contexts.get(contextKey);
1671
- if (!state) return;
1672
- this.contexts.delete(contextKey);
1673
- await state.context.close().catch(() => void 0);
1674
- }
1675
- async waitForPage(page) {
1676
- await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => void 0);
1677
- await page.waitForTimeout(200).catch(() => void 0);
1678
- }
1679
- };
1680
-
1681
272
  // src/version.ts
1682
- var PACKAGE_VERSION = "0.3.5";
273
+ var PACKAGE_VERSION = "0.3.7";
1683
274
 
1684
275
  // src/mcp/browser-agent-tool-schemas.ts
1685
276
  var import_zod = require("zod");
@@ -1702,32 +293,13 @@ var BrowserOpenInputSchema = {
1702
293
  save_profile_changes: import_zod.z.boolean().optional().describe("Persist cookies and local storage back to the named profile when the session is closed. Use this for profile setup or intentional auth refreshes; avoid parallel sessions writing to the same profile."),
1703
294
  timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy browser session may live before auto-termination. Defaults to 600. The browser idles into a zero-cost standby between actions, so a longer timeout is cheap.")
1704
295
  };
1705
- var BrowserProfileListInputSchema = {
1706
- email: import_zod.z.string().optional().describe("Optional Chrome account email or displayed profile name to filter for, for example seo@example.com."),
1707
- user_data_dir: import_zod.z.string().optional().describe("Optional Chrome user data directory. Defaults to the current OS Chrome profile root.")
1708
- };
1709
296
  var BrowserProfileOnboardInputSchema = {
1710
297
  email: import_zod.z.string().optional().describe("Optional account email used to derive a stable MCP Scraper profile name. This does not import cookies or browser state."),
1711
298
  profile: import_zod.z.string().optional().describe("Optional saved MCP Scraper hosted browser profile name to create or load. If omitted, email is used to derive one."),
1712
- user_data_dir: import_zod.z.string().optional().describe("Optional Chrome metadata directory for deriving a profile name only. This is not a local browser setup path."),
1713
299
  url: import_zod.z.string().url().optional().describe("Setup URL to open after creating the direct no-proxy browser. Defaults to https://google.com/ so the user can click Sign in."),
1714
300
  label: import_zod.z.string().optional().describe("Optional human label for this setup session."),
1715
301
  timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy setup session may live before auto-termination. Defaults to 600.")
1716
302
  };
1717
- var BrowserProfileImportInputSchema = {
1718
- email: import_zod.z.string().optional().describe("Optional Chrome account email or displayed profile name to select from Chrome Local State."),
1719
- name: import_zod.z.string().optional().describe("Name for the managed MCP Scraper browser profile. Defaults to a slug derived from the selected Chrome account."),
1720
- chrome_profile_directory: import_zod.z.string().optional().describe("Optional Chrome profile directory to import, for example Default or Profile 1."),
1721
- user_data_dir: import_zod.z.string().optional().describe("Optional source Chrome user data directory. Defaults to the current OS Chrome profile root."),
1722
- output_dir: import_zod.z.string().optional().describe("Optional managed profile base directory. Defaults to ~/.mcp-scraper/browser-profiles."),
1723
- browser_executable_path: import_zod.z.string().optional().describe("Optional Chrome executable path for local browser launches. Defaults to the installed Google Chrome path when found."),
1724
- overwrite: import_zod.z.boolean().default(false).describe("Overwrite an existing managed profile clone with the same name.")
1725
- };
1726
- var BrowserProfileSyncInputSchema = {
1727
- name: import_zod.z.string().describe("Managed MCP Scraper browser profile name to refresh from its recorded source Chrome profile."),
1728
- output_dir: import_zod.z.string().optional().describe("Optional managed profile base directory. Defaults to ~/.mcp-scraper/browser-profiles."),
1729
- browser_executable_path: import_zod.z.string().optional().describe("Optional Chrome executable path to store in the refreshed profile manifest.")
1730
- };
1731
303
  var BrowserSessionInputSchema = {
1732
304
  session_id: import_zod.z.string().describe("The session id returned by browser_open or browser_list_sessions. Use only a returned session_id; do not construct one yourself.")
1733
305
  };
@@ -1896,14 +468,6 @@ var BrowserOpenOutputSchema = {
1896
468
  hint: import_zod.z.string(),
1897
469
  raw: BrowserRawObject.optional()
1898
470
  };
1899
- var BrowserProfileListOutputSchema = {
1900
- ok: import_zod.z.boolean(),
1901
- tool: import_zod.z.literal("browser_profile_list"),
1902
- session_id: import_zod.z.null(),
1903
- profiles: import_zod.z.array(BrowserRawObject),
1904
- count: import_zod.z.number().int().min(0),
1905
- hint: import_zod.z.string()
1906
- };
1907
471
  var BrowserProfileOnboardOutputSchema = {
1908
472
  ok: import_zod.z.boolean(),
1909
473
  tool: import_zod.z.literal("browser_profile_onboard"),
@@ -1912,27 +476,10 @@ var BrowserProfileOnboardOutputSchema = {
1912
476
  live_view_url: NullableString,
1913
477
  profile: import_zod.z.string(),
1914
478
  setup_url: import_zod.z.string(),
1915
- chrome_profile: BrowserRawObject.nullable(),
479
+ account_email: NullableString,
1916
480
  next_steps: import_zod.z.array(import_zod.z.string()),
1917
481
  raw: BrowserRawObject.optional()
1918
482
  };
1919
- var BrowserProfileImportOutputSchema = {
1920
- ok: import_zod.z.boolean(),
1921
- tool: import_zod.z.literal("browser_profile_import"),
1922
- session_id: import_zod.z.null(),
1923
- profile: BrowserRawObject,
1924
- source_profile: BrowserRawObject,
1925
- next_steps: import_zod.z.array(import_zod.z.string()),
1926
- hint: import_zod.z.string()
1927
- };
1928
- var BrowserProfileSyncOutputSchema = {
1929
- ok: import_zod.z.boolean(),
1930
- tool: import_zod.z.literal("browser_profile_sync"),
1931
- session_id: import_zod.z.null(),
1932
- profile: BrowserRawObject,
1933
- source_profile: BrowserRawObject,
1934
- hint: import_zod.z.string()
1935
- };
1936
483
  var BrowserScreenshotOutputSchema = {
1937
484
  ...BrowserBaseOutput,
1938
485
  tool: import_zod.z.literal("browser_screenshot"),
@@ -2033,9 +580,9 @@ var BrowserListSessionsOutputSchema = {
2033
580
 
2034
581
  // src/mcp/replay-annotator.ts
2035
582
  var import_node_child_process = require("child_process");
2036
- var import_promises3 = require("fs/promises");
2037
- var import_node_os4 = require("os");
2038
- var import_node_path5 = require("path");
583
+ var import_promises = require("fs/promises");
584
+ var import_node_os = require("os");
585
+ var import_node_path = require("path");
2039
586
  var import_node_util = require("util");
2040
587
  var execFileAsync = (0, import_node_util.promisify)(import_node_child_process.execFile);
2041
588
  function finiteNumber(value) {
@@ -2045,11 +592,11 @@ function clamp(value, min, max) {
2045
592
  return Math.min(max, Math.max(min, value));
2046
593
  }
2047
594
  function formatAssTime(seconds) {
2048
- const safe2 = Math.max(0, seconds);
2049
- const hours = Math.floor(safe2 / 3600);
2050
- const minutes = Math.floor(safe2 % 3600 / 60);
2051
- const wholeSeconds = Math.floor(safe2 % 60);
2052
- const centiseconds = Math.floor((safe2 - Math.floor(safe2)) * 100);
595
+ const safe = Math.max(0, seconds);
596
+ const hours = Math.floor(safe / 3600);
597
+ const minutes = Math.floor(safe % 3600 / 60);
598
+ const wholeSeconds = Math.floor(safe % 60);
599
+ const centiseconds = Math.floor((safe - Math.floor(safe)) * 100);
2053
600
  return `${hours}:${String(minutes).padStart(2, "0")}:${String(wholeSeconds).padStart(2, "0")}.${String(centiseconds).padStart(2, "0")}`;
2054
601
  }
2055
602
  function cssHexToAssColor(color) {
@@ -2258,10 +805,10 @@ function ffmpegFilterPath(path) {
2258
805
  async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
2259
806
  if (!options.annotations.length) throw new Error("annotations must include at least one item");
2260
807
  const size = await videoSize(inputFilePath);
2261
- const tmp = await (0, import_promises3.mkdtemp)((0, import_node_path5.join)((0, import_node_os4.tmpdir)(), "mcp-scraper-ass-"));
2262
- const assPath = (0, import_node_path5.join)(tmp, "annotations.ass");
808
+ const tmp = await (0, import_promises.mkdtemp)((0, import_node_path.join)((0, import_node_os.tmpdir)(), "mcp-scraper-ass-"));
809
+ const assPath = (0, import_node_path.join)(tmp, "annotations.ass");
2263
810
  try {
2264
- await (0, import_promises3.writeFile)(assPath, buildAssSubtitle(options, size), "utf8");
811
+ await (0, import_promises.writeFile)(assPath, buildAssSubtitle(options, size), "utf8");
2265
812
  await execFileAsync("ffmpeg", [
2266
813
  "-y",
2267
814
  "-i",
@@ -2278,7 +825,7 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
2278
825
  "copy",
2279
826
  outputFilePath
2280
827
  ], { maxBuffer: 1024 * 1024 * 20 });
2281
- const out = await (0, import_promises3.stat)(outputFilePath);
828
+ const out = await (0, import_promises.stat)(outputFilePath);
2282
829
  return {
2283
830
  filePath: outputFilePath,
2284
831
  bytes: out.size,
@@ -2287,7 +834,7 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
2287
834
  annotationCount: options.annotations.length
2288
835
  };
2289
836
  } finally {
2290
- await (0, import_promises3.rm)(tmp, { recursive: true, force: true });
837
+ await (0, import_promises.rm)(tmp, { recursive: true, force: true });
2291
838
  }
2292
839
  }
2293
840
 
@@ -2327,8 +874,8 @@ function actionResult(tool, sessionId, ok, data, nextRecommendedTool = "browser_
2327
874
  nextRecommendedTool
2328
875
  });
2329
876
  }
2330
- function outputBaseDir2() {
2331
- return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path6.join)((0, import_node_os5.homedir)(), "Downloads", "mcp-scraper");
877
+ function outputBaseDir() {
878
+ return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), "Downloads", "mcp-scraper");
2332
879
  }
2333
880
  function safeFilePart(value) {
2334
881
  return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120) || "replay";
@@ -2337,7 +884,12 @@ function replayFilePath(sessionId, replayId, filename) {
2337
884
  const requested = filename?.trim();
2338
885
  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
2339
886
  const name = requested ? safeFilePart(requested).replace(/\.mp4$/i, "") : `${stamp}-${safeFilePart(sessionId)}-${safeFilePart(replayId)}`;
2340
- return (0, import_node_path6.join)(outputBaseDir2(), "browser-replays", `${name}.mp4`);
887
+ return (0, import_node_path2.join)(outputBaseDir(), "browser-replays", `${name}.mp4`);
888
+ }
889
+ function savedProfileNameFromEmail(email) {
890
+ const value = email?.trim().toLowerCase();
891
+ if (!value) return null;
892
+ return value.replace(/[^a-z0-9._-]+/g, "-").replace(/[._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || null;
2341
893
  }
2342
894
  function annotatedReplayFilePath(sessionId, replayId, filename) {
2343
895
  const requested = filename?.trim();
@@ -2368,7 +920,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2368
920
  const baseUrl2 = opts.baseUrl.replace(/\/$/, "");
2369
921
  const consoleBase = (opts.consoleBaseUrl ?? opts.baseUrl).replace(/\/$/, "");
2370
922
  const timeoutMs = opts.timeoutMs ?? 9e4;
2371
- const localBrowser = new LocalBrowserAgentService();
2372
923
  async function req(method, path, body) {
2373
924
  try {
2374
925
  const res = await fetch(`${baseUrl2}${path}`, {
@@ -2397,8 +948,8 @@ function registerBrowserAgentMcpTools(server2, opts) {
2397
948
  }
2398
949
  const bytes = Buffer.from(await res.arrayBuffer());
2399
950
  const filePath = replayFilePath(sessionId, replayId, filename);
2400
- (0, import_node_fs4.mkdirSync)((0, import_node_path6.join)(outputBaseDir2(), "browser-replays"), { recursive: true });
2401
- (0, import_node_fs4.writeFileSync)(filePath, bytes);
951
+ (0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
952
+ (0, import_node_fs.writeFileSync)(filePath, bytes);
2402
953
  return {
2403
954
  ok: true,
2404
955
  data: {
@@ -2420,131 +971,20 @@ function registerBrowserAgentMcpTools(server2, opts) {
2420
971
  idempotentHint: false,
2421
972
  openWorldHint: true
2422
973
  });
2423
- function selectChromeProfile(matches, email) {
2424
- const wanted = email?.trim().toLowerCase();
2425
- if (!wanted) return matches[0] ?? null;
2426
- return matches.find((profile) => profile.email.toLowerCase() === wanted) ?? matches[0] ?? null;
2427
- }
2428
- function localReplayUnsupported(tool, sessionId = null, replayId = null) {
2429
- return errorResult(tool, {
2430
- error: "Local browser mode does not support replay recording yet. Use hosted mode for browser_replay_* tools.",
2431
- mode: "local"
2432
- }, sessionId, replayId);
2433
- }
2434
- server2.registerTool(
2435
- "browser_profile_list",
2436
- {
2437
- title: "Developer: List Local Chrome Profiles",
2438
- description: "Developer/local-machine helper only. List local Chrome profile metadata visible to this MCP server: account email, Chrome profile directory, local path, and a suggested managed browser profile name. This reads Chrome Local State only; it does not read cookies, passwords, browsing history, or copy local browser state. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
2439
- inputSchema: BrowserProfileListInputSchema,
2440
- outputSchema: BrowserProfileListOutputSchema,
2441
- annotations: annotations("Developer: List Local Chrome Profiles", true)
2442
- },
2443
- async (input) => {
2444
- try {
2445
- const profiles = await listLocalChromeProfiles(input.user_data_dir, input.email);
2446
- return structuredResult({
2447
- ok: true,
2448
- tool: "browser_profile_list",
2449
- session_id: null,
2450
- profiles,
2451
- count: profiles.length,
2452
- hint: "For customer auth setup, use browser_profile_onboard so the user signs in through the hosted watch_url. Local import is only for developer/local-machine mode."
2453
- });
2454
- } catch (err) {
2455
- return errorResult("browser_profile_list", { error: err instanceof Error ? err.message : String(err) });
2456
- }
2457
- }
2458
- );
2459
- server2.registerTool(
2460
- "browser_profile_import",
2461
- {
2462
- title: "Developer: Import Local Chrome Profile",
2463
- description: "Developer/local-machine helper only. Clone one local Chrome profile into MCP Scraper managed storage for local browser mode. This copies browser state files such as cookies, history, local storage, session storage, and password database metadata into ~/.mcp-scraper/browser-profiles by default; it skips cache and lock files and never prints cookie or password values. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
2464
- inputSchema: BrowserProfileImportInputSchema,
2465
- outputSchema: BrowserProfileImportOutputSchema,
2466
- annotations: { title: "Developer: Import Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
2467
- },
2468
- async (input) => {
2469
- try {
2470
- const result = await importChromeProfile({
2471
- email: input.email,
2472
- name: input.name,
2473
- chromeProfileDirectory: input.chrome_profile_directory,
2474
- sourceUserDataDir: input.user_data_dir,
2475
- outputDir: input.output_dir,
2476
- overwrite: input.overwrite,
2477
- browserExecutablePath: input.browser_executable_path
2478
- });
2479
- return structuredResult({
2480
- ok: true,
2481
- tool: "browser_profile_import",
2482
- session_id: null,
2483
- profile: result.profile,
2484
- source_profile: result.sourceProfile,
2485
- next_steps: [
2486
- `Set MCP_SCRAPER_BROWSER_MODE=local in the MCP server config.`,
2487
- `Set MCP_SCRAPER_BROWSER_PROFILE=${result.profile.name}.`,
2488
- "Restart the MCP client so it starts the local browser mode process.",
2489
- "Call browser_open with the same MCP server; it will open a local Chrome window using the cloned profile."
2490
- ],
2491
- hint: "Developer-only local clone. For customer auth setup, use browser_profile_onboard at Google, provide the hosted watch_url, then browser_close to save the hosted profile."
2492
- });
2493
- } catch (err) {
2494
- return errorResult("browser_profile_import", { error: err instanceof Error ? err.message : String(err) });
2495
- }
2496
- }
2497
- );
2498
- server2.registerTool(
2499
- "browser_profile_sync",
2500
- {
2501
- title: "Developer: Sync Local Chrome Profile",
2502
- description: "Developer/local-machine helper only. Refresh an existing MCP Scraper managed local browser profile from the source Chrome profile recorded in its manifest. This overwrites the managed clone, copies browser state files again, skips cache and lock files, and never prints cookie or password values. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
2503
- inputSchema: BrowserProfileSyncInputSchema,
2504
- outputSchema: BrowserProfileSyncOutputSchema,
2505
- annotations: { title: "Developer: Sync Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
2506
- },
2507
- async (input) => {
2508
- try {
2509
- const result = await syncImportedChromeProfile(input.name, {
2510
- outputDir: input.output_dir,
2511
- browserExecutablePath: input.browser_executable_path
2512
- });
2513
- return structuredResult({
2514
- ok: true,
2515
- tool: "browser_profile_sync",
2516
- session_id: null,
2517
- profile: result.profile,
2518
- source_profile: result.sourceProfile,
2519
- hint: `Developer-only local clone refreshed for ${result.profile.name}. Customer reconnects should use browser_profile_onboard with the same hosted profile name.`
2520
- });
2521
- } catch (err) {
2522
- return errorResult("browser_profile_sync", { error: err instanceof Error ? err.message : String(err) });
2523
- }
2524
- }
2525
- );
2526
974
  server2.registerTool(
2527
975
  "browser_profile_onboard",
2528
976
  {
2529
977
  title: "Connect Hosted Browser Profile",
2530
- description: "Create or load a saved MCP Scraper hosted browser profile and open a direct no-proxy setup browser with profile saving enabled. This is the customer auth setup and reconnect flow: open Google, give the user the watch_url, have them click Sign in and complete login, then call browser_close to save cookies and local storage. If a saved profile later shows login controls, run this same setup flow again. Do not use local Chrome import for this customer setup.",
978
+ description: "Create or load a saved MCP Scraper hosted browser profile and open a direct no-proxy setup browser with profile saving enabled. This is the customer auth setup and reconnect flow: open Google, give the user the watch_url, have them click Sign in and complete login, then call browser_close to save cookies and storage. If a saved profile later shows login controls, run this same setup flow again.",
2531
979
  inputSchema: BrowserProfileOnboardInputSchema,
2532
980
  outputSchema: BrowserProfileOnboardOutputSchema,
2533
981
  annotations: annotations("Connect Hosted Browser Profile")
2534
982
  },
2535
983
  async (input) => {
2536
- let chromeProfile = null;
2537
- if (input.email) {
2538
- try {
2539
- chromeProfile = selectChromeProfile(await listLocalChromeProfiles(input.user_data_dir, input.email), input.email);
2540
- } catch (err) {
2541
- return errorResult("browser_profile_onboard", { error: err instanceof Error ? err.message : String(err) });
2542
- }
2543
- }
2544
- const profile = input.profile?.trim() || chromeProfile?.recommendedKernelProfileName || browserServiceProfileName();
984
+ const profile = input.profile?.trim() || savedProfileNameFromEmail(input.email) || browserServiceProfileName();
2545
985
  if (!profile) {
2546
986
  return errorResult("browser_profile_onboard", {
2547
- error: "profile is required when no email match or BROWSER_AGENT_PROFILE_NAME is available"
987
+ error: "profile or email is required when BROWSER_AGENT_PROFILE_NAME is not available"
2548
988
  });
2549
989
  }
2550
990
  const setupUrl = input.url ?? "https://google.com/";
@@ -2567,10 +1007,10 @@ function registerBrowserAgentMcpTools(server2, opts) {
2567
1007
  live_view_url: session.live_view_url ?? null,
2568
1008
  profile,
2569
1009
  setup_url: setupUrl,
2570
- chrome_profile: chromeProfile,
1010
+ account_email: input.email?.trim() || null,
2571
1011
  next_steps: [
2572
1012
  "Open the watch_url and let the user click Sign in on Google.",
2573
- "After the user confirms login is complete, call browser_close with this session_id to persist cookies and local storage to the saved profile.",
1013
+ "After the user confirms login is complete, call browser_close with this session_id to persist cookies and storage to the saved profile.",
2574
1014
  "If the saved profile later shows login controls or disconnects, repeat browser_profile_onboard and close the setup session again.",
2575
1015
  "Use BROWSER_AGENT_PROFILE_NAME or browser_open.profile with this profile name for future authenticated sessions."
2576
1016
  ],
@@ -2588,23 +1028,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2588
1028
  annotations: annotations("Open Browser Session")
2589
1029
  },
2590
1030
  async (input) => {
2591
- if (localBrowserModeEnabled()) {
2592
- try {
2593
- const session2 = await localBrowser.open(input);
2594
- return structuredResult({
2595
- ok: true,
2596
- tool: "browser_open",
2597
- session_id: session2.session_id,
2598
- watch_url: session2.watch_url,
2599
- live_view_url: null,
2600
- url: input.url ?? null,
2601
- hint: "Local Chrome is open on this machine. Call browser_screenshot to see the page and click by x,y coordinates.",
2602
- raw: session2
2603
- });
2604
- } catch (err) {
2605
- return errorResult("browser_open", { error: err instanceof Error ? err.message : String(err), mode: "local" });
2606
- }
2607
- }
2608
1031
  const profile = input.profile ?? browserServiceProfileName();
2609
1032
  const saveProfileChanges = input.save_profile_changes ?? browserServiceProfileSaveChanges();
2610
1033
  const open = await req("POST", "/agent/sessions", {
@@ -2641,27 +1064,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2641
1064
  annotations: annotations("See Page", true)
2642
1065
  },
2643
1066
  async (input) => {
2644
- if (localBrowserModeEnabled()) {
2645
- try {
2646
- const res2 = await localBrowser.screenshot(input.session_id);
2647
- const content2 = [];
2648
- if (res2.image_base64) content2.push({ type: "image", data: res2.image_base64, mimeType: res2.mime_type });
2649
- const structured2 = {
2650
- ok: true,
2651
- tool: "browser_screenshot",
2652
- session_id: input.session_id,
2653
- url: res2.url ?? null,
2654
- title: res2.title ?? null,
2655
- text: res2.text,
2656
- elements: res2.elements,
2657
- screenshot: { mime_type: res2.mime_type, inline: true }
2658
- };
2659
- content2.push({ type: "text", text: JSON.stringify(structured2) });
2660
- return { content: content2, structuredContent: structured2 };
2661
- } catch (err) {
2662
- return errorResult("browser_screenshot", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2663
- }
2664
- }
2665
1067
  const res = await req("POST", `/agent/sessions/${input.session_id}/screenshot`);
2666
1068
  if (!res.ok) return errorResult("browser_screenshot", res.data, input.session_id);
2667
1069
  const { image_base64, mime_type, url, title, elements, text } = res.data;
@@ -2694,23 +1096,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2694
1096
  annotations: annotations("Read Page", true)
2695
1097
  },
2696
1098
  async (input) => {
2697
- if (localBrowserModeEnabled()) {
2698
- try {
2699
- const res2 = await localBrowser.read(input.session_id);
2700
- return structuredResult({
2701
- ok: true,
2702
- tool: "browser_read",
2703
- session_id: input.session_id,
2704
- url: res2.url ?? null,
2705
- title: res2.title ?? null,
2706
- text: res2.text,
2707
- elements: res2.elements,
2708
- raw: res2
2709
- });
2710
- } catch (err) {
2711
- return errorResult("browser_read", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2712
- }
2713
- }
2714
1099
  const res = await req("POST", `/agent/sessions/${input.session_id}/read`);
2715
1100
  if (!res.ok) return errorResult("browser_read", res.data, input.session_id);
2716
1101
  return structuredResult({
@@ -2735,24 +1120,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2735
1120
  annotations: annotations("Locate DOM Targets", true)
2736
1121
  },
2737
1122
  async (input) => {
2738
- if (localBrowserModeEnabled()) {
2739
- try {
2740
- const res2 = await localBrowser.locate(input.session_id, input.targets);
2741
- return structuredResult({
2742
- ok: true,
2743
- tool: "browser_locate",
2744
- session_id: input.session_id,
2745
- url: res2.url ?? null,
2746
- title: res2.title ?? null,
2747
- viewport: res2.viewport ?? null,
2748
- replay: null,
2749
- targets: Array.isArray(res2.targets) ? res2.targets : [],
2750
- raw: res2
2751
- });
2752
- } catch (err) {
2753
- return errorResult("browser_locate", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2754
- }
2755
- }
2756
1123
  const res = await req("POST", `/agent/sessions/${input.session_id}/locate`, { targets: input.targets });
2757
1124
  if (!res.ok) return errorResult("browser_locate", res.data, input.session_id);
2758
1125
  return structuredResult({
@@ -2778,14 +1145,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2778
1145
  annotations: annotations("Navigate To URL")
2779
1146
  },
2780
1147
  async (input) => {
2781
- if (localBrowserModeEnabled()) {
2782
- try {
2783
- const res2 = await localBrowser.goto(input.session_id, input.url);
2784
- return actionResult("browser_goto", input.session_id, true, res2, "browser_screenshot");
2785
- } catch (err) {
2786
- return errorResult("browser_goto", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2787
- }
2788
- }
2789
1148
  const res = await req("POST", `/agent/sessions/${input.session_id}/goto`, { url: input.url });
2790
1149
  return actionResult("browser_goto", input.session_id, res.ok, res.data, "browser_screenshot");
2791
1150
  }
@@ -2800,14 +1159,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2800
1159
  annotations: annotations("Click")
2801
1160
  },
2802
1161
  async (input) => {
2803
- if (localBrowserModeEnabled()) {
2804
- try {
2805
- const res2 = await localBrowser.click(input.session_id, input);
2806
- return actionResult("browser_click", input.session_id, true, res2, "browser_screenshot");
2807
- } catch (err) {
2808
- return errorResult("browser_click", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2809
- }
2810
- }
2811
1162
  const res = await req("POST", `/agent/sessions/${input.session_id}/click`, {
2812
1163
  x: input.x,
2813
1164
  y: input.y,
@@ -2827,14 +1178,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2827
1178
  annotations: annotations("Type Text")
2828
1179
  },
2829
1180
  async (input) => {
2830
- if (localBrowserModeEnabled()) {
2831
- try {
2832
- const res2 = await localBrowser.type(input.session_id, input);
2833
- return actionResult("browser_type", input.session_id, true, res2, "browser_screenshot");
2834
- } catch (err) {
2835
- return errorResult("browser_type", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2836
- }
2837
- }
2838
1181
  const res = await req("POST", `/agent/sessions/${input.session_id}/type`, { text: input.text, delay: input.delay });
2839
1182
  return actionResult("browser_type", input.session_id, res.ok, res.data, "browser_screenshot");
2840
1183
  }
@@ -2849,14 +1192,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2849
1192
  annotations: annotations("Scroll")
2850
1193
  },
2851
1194
  async (input) => {
2852
- if (localBrowserModeEnabled()) {
2853
- try {
2854
- const res2 = await localBrowser.scroll(input.session_id, input);
2855
- return actionResult("browser_scroll", input.session_id, true, res2, "browser_screenshot");
2856
- } catch (err) {
2857
- return errorResult("browser_scroll", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2858
- }
2859
- }
2860
1195
  const res = await req("POST", `/agent/sessions/${input.session_id}/scroll`, {
2861
1196
  delta_y: input.delta_y,
2862
1197
  delta_x: input.delta_x,
@@ -2876,14 +1211,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2876
1211
  annotations: annotations("Press Keys")
2877
1212
  },
2878
1213
  async (input) => {
2879
- if (localBrowserModeEnabled()) {
2880
- try {
2881
- const res2 = await localBrowser.press(input.session_id, input);
2882
- return actionResult("browser_press", input.session_id, true, res2, "browser_screenshot");
2883
- } catch (err) {
2884
- return errorResult("browser_press", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2885
- }
2886
- }
2887
1214
  const res = await req("POST", `/agent/sessions/${input.session_id}/press`, { keys: input.keys });
2888
1215
  return actionResult("browser_press", input.session_id, res.ok, res.data, "browser_screenshot");
2889
1216
  }
@@ -2898,7 +1225,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2898
1225
  annotations: annotations("Start Recording")
2899
1226
  },
2900
1227
  async (input) => {
2901
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_start", input.session_id);
2902
1228
  const res = await req("POST", `/agent/sessions/${input.session_id}/replay/start`);
2903
1229
  if (!res.ok) return errorResult("browser_replay_start", res.data, input.session_id);
2904
1230
  return structuredResult({
@@ -2922,7 +1248,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2922
1248
  annotations: annotations("Stop Recording")
2923
1249
  },
2924
1250
  async (input) => {
2925
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_stop", input.session_id, input.replay_id);
2926
1251
  const res = await req("POST", `/agent/sessions/${input.session_id}/replay/stop`, { replay_id: input.replay_id });
2927
1252
  if (!res.ok) return errorResult("browser_replay_stop", res.data, input.session_id, input.replay_id);
2928
1253
  return structuredResult({
@@ -2946,7 +1271,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2946
1271
  annotations: annotations("List Replay Videos", true)
2947
1272
  },
2948
1273
  async (input) => {
2949
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_list_replays", input.session_id);
2950
1274
  const res = await req("GET", `/agent/sessions/${input.session_id}/replays`);
2951
1275
  if (!res.ok) return errorResult("browser_list_replays", res.data, input.session_id);
2952
1276
  const replays = Array.isArray(res.data?.replays) ? res.data.replays : [];
@@ -2969,7 +1293,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2969
1293
  annotations: annotations("Download Replay MP4")
2970
1294
  },
2971
1295
  async (input) => {
2972
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_download", input.session_id, input.replay_id);
2973
1296
  const res = await downloadReplay(input.session_id, input.replay_id, input.filename);
2974
1297
  if (!res.ok) return errorResult("browser_replay_download", res.data, input.session_id, input.replay_id);
2975
1298
  return structuredResult({
@@ -2994,7 +1317,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2994
1317
  annotations: annotations("Mark Replay Annotation", true)
2995
1318
  },
2996
1319
  async (input) => {
2997
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_mark", input.session_id);
2998
1320
  const res = await req("POST", `/agent/sessions/${input.session_id}/locate`, { targets: [input.target] });
2999
1321
  if (!res.ok) return errorResult("browser_replay_mark", res.data, input.session_id);
3000
1322
  const target = res.data?.targets?.[0];
@@ -3045,14 +1367,13 @@ function registerBrowserAgentMcpTools(server2, opts) {
3045
1367
  annotations: annotations("Annotate Replay MP4")
3046
1368
  },
3047
1369
  async (input) => {
3048
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_annotate", input.session_id, input.replay_id);
3049
1370
  const sourceName = input.filename ? `${input.filename}-source` : void 0;
3050
1371
  const downloaded = await downloadReplay(input.session_id, input.replay_id, sourceName);
3051
1372
  if (!downloaded.ok) return errorResult("browser_replay_annotate", downloaded.data, input.session_id, input.replay_id);
3052
1373
  try {
3053
1374
  const sourcePath = String(downloaded.data.file_path);
3054
1375
  const outputPath = annotatedReplayFilePath(input.session_id, input.replay_id, input.filename);
3055
- (0, import_node_fs4.mkdirSync)((0, import_node_path6.join)(outputBaseDir2(), "browser-replays"), { recursive: true });
1376
+ (0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
3056
1377
  const result = await annotateReplayVideo(sourcePath, outputPath, {
3057
1378
  annotations: input.annotations,
3058
1379
  sourceWidth: input.source_width,
@@ -3088,20 +1409,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
3088
1409
  annotations: { title: "Close Browser Session", readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }
3089
1410
  },
3090
1411
  async (input) => {
3091
- if (localBrowserModeEnabled()) {
3092
- try {
3093
- const res2 = await localBrowser.close(input.session_id);
3094
- return structuredResult({
3095
- ok: true,
3096
- tool: "browser_close",
3097
- session_id: input.session_id,
3098
- closed: true,
3099
- raw: res2
3100
- });
3101
- } catch (err) {
3102
- return errorResult("browser_close", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
3103
- }
3104
- }
3105
1412
  const res = await req("DELETE", `/agent/sessions/${input.session_id}`);
3106
1413
  if (!res.ok) return errorResult("browser_close", res.data, input.session_id);
3107
1414
  return structuredResult({
@@ -3123,16 +1430,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
3123
1430
  annotations: annotations("List Browser Sessions", true)
3124
1431
  },
3125
1432
  async (input) => {
3126
- if (localBrowserModeEnabled()) {
3127
- const sessions2 = localBrowser.list(input.include_closed);
3128
- return structuredResult({
3129
- ok: true,
3130
- tool: "browser_list_sessions",
3131
- session_id: null,
3132
- sessions: sessions2,
3133
- count: sessions2.length
3134
- });
3135
- }
3136
1433
  const res = await req("GET", `/agent/sessions${input.include_closed ? "?all=1" : ""}`);
3137
1434
  if (!res.ok) return errorResult("browser_list_sessions", res.data);
3138
1435
  const sessions = (res.data.sessions ?? []).map((s) => ({ ...s, watch_url: `${consoleBase}/console/${s.session_id}` }));
@@ -3174,14 +1471,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
3174
1471
  exports: exports2 ?? null,
3175
1472
  ...result.debug ? { debug: result.debug } : {}
3176
1473
  });
3177
- if (localBrowserModeEnabled()) {
3178
- try {
3179
- const { result, exports: exports2 } = await localBrowser.captureFanout(input.session_id, input);
3180
- return emit(result, exports2);
3181
- } catch (err) {
3182
- return errorResult("browser_capture_fanout", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
3183
- }
3184
- }
3185
1474
  const res = await req("POST", `/agent/sessions/${input.session_id}/capture-fanout`, {
3186
1475
  prompt: input.prompt,
3187
1476
  wait_ms: input.wait_ms,
@@ -3197,13 +1486,13 @@ function registerBrowserAgentMcpTools(server2, opts) {
3197
1486
 
3198
1487
  // src/mcp/paa-mcp-server.ts
3199
1488
  var import_mcp2 = require("@modelcontextprotocol/sdk/server/mcp.js");
3200
- var import_node_fs6 = require("fs");
3201
- var import_node_path8 = require("path");
1489
+ var import_node_fs3 = require("fs");
1490
+ var import_node_path4 = require("path");
3202
1491
 
3203
1492
  // src/mcp/mcp-response-formatter.ts
3204
- var import_node_fs5 = require("fs");
3205
- var import_node_os6 = require("os");
3206
- var import_node_path7 = require("path");
1493
+ var import_node_fs2 = require("fs");
1494
+ var import_node_os3 = require("os");
1495
+ var import_node_path3 = require("path");
3207
1496
 
3208
1497
  // src/errors.ts
3209
1498
  function sanitizeVendorName(message) {
@@ -3339,17 +1628,17 @@ function reportTitle(full) {
3339
1628
  const title = full.split("\n").find((line) => line.startsWith("# "));
3340
1629
  return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
3341
1630
  }
3342
- function outputBaseDir3() {
3343
- return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path7.join)((0, import_node_os6.homedir)(), "Downloads", "mcp-scraper");
1631
+ function outputBaseDir2() {
1632
+ return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
3344
1633
  }
3345
1634
  function saveFullReport(full) {
3346
1635
  if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
3347
- const outDir = outputBaseDir3();
1636
+ const outDir = outputBaseDir2();
3348
1637
  try {
3349
- (0, import_node_fs5.mkdirSync)(outDir, { recursive: true });
1638
+ (0, import_node_fs2.mkdirSync)(outDir, { recursive: true });
3350
1639
  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
3351
- const file = (0, import_node_path7.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
3352
- (0, import_node_fs5.writeFileSync)(file, full, "utf8");
1640
+ const file = (0, import_node_path3.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
1641
+ (0, import_node_fs2.writeFileSync)(file, full, "utf8");
3353
1642
  return file;
3354
1643
  } catch {
3355
1644
  return null;
@@ -3358,12 +1647,12 @@ function saveFullReport(full) {
3358
1647
  function persistScreenshotLocally(base64, url) {
3359
1648
  if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
3360
1649
  try {
3361
- const dir = (0, import_node_path7.join)(outputBaseDir3(), "screenshots");
3362
- (0, import_node_fs5.mkdirSync)(dir, { recursive: true });
1650
+ const dir = (0, import_node_path3.join)(outputBaseDir2(), "screenshots");
1651
+ (0, import_node_fs2.mkdirSync)(dir, { recursive: true });
3363
1652
  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
3364
1653
  const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
3365
- const filePath = (0, import_node_path7.join)(dir, `${stamp}-${slug}.png`);
3366
- (0, import_node_fs5.writeFileSync)(filePath, Buffer.from(base64, "base64"));
1654
+ const filePath = (0, import_node_path3.join)(dir, `${stamp}-${slug}.png`);
1655
+ (0, import_node_fs2.writeFileSync)(filePath, Buffer.from(base64, "base64"));
3367
1656
  return filePath;
3368
1657
  } catch {
3369
1658
  return null;
@@ -4656,16 +2945,13 @@ ${chunkRows}` : "",
4656
2945
  }
4657
2946
  function structuredInstagramBrowser(raw) {
4658
2947
  const browser = raw && typeof raw === "object" ? raw : {};
4659
- const mode = browser.mode === "local" ? "local" : "hosted";
4660
- const requestedMode = browser.requestedMode === "local" || browser.requestedMode === "hosted" || browser.requestedMode === "auto" ? browser.requestedMode : "auto";
4661
- const profileSource = browser.profileSource === "managed_profile" || browser.profileSource === "direct_profile_dir" ? browser.profileSource : "hosted";
4662
2948
  return {
4663
- mode,
4664
- requestedMode,
2949
+ mode: "hosted",
2950
+ requestedMode: "hosted",
4665
2951
  profileName: typeof browser.profileName === "string" ? browser.profileName : null,
4666
- profileSource,
4667
- profileDirConfigured: browser.profileDirConfigured === true,
4668
- executablePathConfigured: browser.executablePathConfigured === true
2952
+ profileSource: "hosted",
2953
+ profileDirConfigured: false,
2954
+ executablePathConfigured: false
4669
2955
  };
4670
2956
  }
4671
2957
  function structuredInstagramPagination(raw, input) {
@@ -4707,7 +2993,7 @@ function formatInstagramProfileContent(raw, input) {
4707
2993
  const itemRows = items.slice(0, 100).map(
4708
2994
  (item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
4709
2995
  ).join("\n");
4710
- const browserLabel = browser.mode === "local" ? `local profile ${browser.profileName ? `\`${browser.profileName}\`` : "(unnamed)"}` : "hosted browser";
2996
+ const browserLabel = "hosted browser";
4711
2997
  const full = [
4712
2998
  `# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
4713
2999
  `**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
@@ -4782,7 +3068,7 @@ function formatInstagramMediaDownload(raw, input) {
4782
3068
  const transcriptText = transcript?.text ?? "";
4783
3069
  const chunks = transcript?.chunks ?? [];
4784
3070
  const browser = structuredInstagramBrowser(d.browser);
4785
- const browserLabel = browser.mode === "local" ? `local profile ${browser.profileName ? `\`${browser.profileName}\`` : "(unnamed)"}` : "hosted browser";
3071
+ const browserLabel = "hosted browser";
4786
3072
  const downloadRows = downloads.map((download, i) => {
4787
3073
  const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
4788
3074
  return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
@@ -5025,14 +3311,12 @@ var FacebookVideoTranscribeInputSchema = {
5025
3311
  quality: import_zod3.z.enum(["best", "hd", "sd"]).default("best").describe("Preferred progressive MP4 quality. Use best by default; hd prefers the highest HD progressive URL; sd forces the SD URL.")
5026
3312
  };
5027
3313
  var InstagramProfileContentInputSchema = {
5028
- handle: import_zod3.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url. Use only public profiles unless a configured browser profile is available server-side."),
3314
+ handle: import_zod3.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url."),
5029
3315
  url: import_zod3.z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
5030
- maxItems: import_zod3.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values with an authenticated browser profile when the user asks for a fuller archive."),
5031
- maxScrolls: import_zod3.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase when using authenticated Chromium and the profile has a long history."),
3316
+ maxItems: import_zod3.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values only when the user asks for a fuller archive."),
3317
+ maxScrolls: import_zod3.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase for long profiles when Instagram continues loading more grid links."),
5032
3318
  scrollDelayMs: import_zod3.z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
5033
- stableScrollLimit: import_zod3.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4."),
5034
- browserMode: import_zod3.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
5035
- browserProfile: import_zod3.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
3319
+ stableScrollLimit: import_zod3.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
5036
3320
  };
5037
3321
  var InstagramMediaDownloadInputSchema = {
5038
3322
  url: import_zod3.z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
@@ -5040,9 +3324,7 @@ var InstagramMediaDownloadInputSchema = {
5040
3324
  downloadMedia: import_zod3.z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
5041
3325
  downloadAllTracks: import_zod3.z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
5042
3326
  includeTranscript: import_zod3.z.boolean().default(false).describe("Transcribe the selected audio track when available. This adds transcription cost and may take longer."),
5043
- mux: import_zod3.z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable."),
5044
- browserMode: import_zod3.z.enum(["auto", "hosted", "local"]).default("auto").describe("Browser backend. auto uses the hosted browser unless local mode is configured or browserProfile is passed. local uses an imported/authenticated Chrome profile from browser_profile_import/sync. hosted forces the remote browser path."),
5045
- browserProfile: import_zod3.z.string().min(1).optional().describe("Managed local browser profile name created by browser_profile_import/sync. Passing this opts the Instagram tool into local authenticated Chromium for this call.")
3327
+ mux: import_zod3.z.boolean().default(true).describe("When video and audio tracks are downloaded separately, try to mux them into a single MP4 if ffmpeg is available. Returns separate tracks when muxing is unavailable.")
5046
3328
  };
5047
3329
  var MapsPlaceIntelInputSchema = {
5048
3330
  businessName: import_zod3.z.string().min(1).describe('Business name only. If user says "Elite Roofing Denver CO", use businessName="Elite Roofing" and location="Denver, CO".'),
@@ -5441,10 +3723,10 @@ var TranscriptChunkOutput = import_zod3.z.object({
5441
3723
  text: import_zod3.z.string()
5442
3724
  });
5443
3725
  var InstagramBrowserOutput = import_zod3.z.object({
5444
- mode: import_zod3.z.enum(["hosted", "local"]),
5445
- requestedMode: import_zod3.z.enum(["auto", "hosted", "local"]),
3726
+ mode: import_zod3.z.literal("hosted"),
3727
+ requestedMode: import_zod3.z.literal("hosted"),
5446
3728
  profileName: NullableString2,
5447
- profileSource: import_zod3.z.enum(["hosted", "managed_profile", "direct_profile_dir"]),
3729
+ profileSource: import_zod3.z.literal("hosted"),
5448
3730
  profileDirConfigured: import_zod3.z.boolean(),
5449
3731
  executablePathConfigured: import_zod3.z.boolean()
5450
3732
  });
@@ -6065,8 +4347,8 @@ function localPlanningToolAnnotations(title) {
6065
4347
  }
6066
4348
  function listSavedReports() {
6067
4349
  try {
6068
- const dir = outputBaseDir3();
6069
- return (0, import_node_fs6.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs6.statSync)((0, import_node_path8.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
4350
+ const dir = outputBaseDir2();
4351
+ return (0, import_node_fs3.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs3.statSync)((0, import_node_path4.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
6070
4352
  } catch {
6071
4353
  return [];
6072
4354
  }
@@ -6090,9 +4372,9 @@ function registerSavedReportResources(server2) {
6090
4372
  },
6091
4373
  async (uri, variables) => {
6092
4374
  const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
6093
- const filename = (0, import_node_path8.basename)(decodeURIComponent(String(requested ?? "")));
4375
+ const filename = (0, import_node_path4.basename)(decodeURIComponent(String(requested ?? "")));
6094
4376
  if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
6095
- const text = (0, import_node_fs6.readFileSync)((0, import_node_path8.join)(outputBaseDir3(), filename), "utf8");
4377
+ const text = (0, import_node_fs3.readFileSync)((0, import_node_path4.join)(outputBaseDir2(), filename), "utf8");
6096
4378
  return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
6097
4379
  }
6098
4380
  );
@@ -6181,14 +4463,14 @@ function registerPaaExtractorMcpTools(server2, executor, options = {}) {
6181
4463
  }, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
6182
4464
  server2.registerTool("instagram_profile_content", {
6183
4465
  title: "Instagram Profile Content Discovery",
6184
- description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser mode, pagination attempts, stop reason, and limitations. For authenticated collection, first use browser_profile_import/sync, then pass browserProfile and raise maxItems/maxScrolls for longer profile pagination."),
4466
+ description: withReportNote("Discover Instagram profile grid content links for a handle or profile URL. Use this when the user wants a person or brand account content inventory before selecting posts/reels to download. Returns profile stats, collected post/reel/tv URLs, shortcodes, type counts, browser details, pagination attempts, stop reason, and limitations."),
6185
4467
  inputSchema: InstagramProfileContentInputSchema,
6186
4468
  outputSchema: InstagramProfileContentOutputSchema,
6187
4469
  annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
6188
4470
  }, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
6189
4471
  server2.registerTool("instagram_media_download", {
6190
4472
  title: "Instagram Post/Reel Media Download",
6191
- description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available. Pass browserProfile from browser_profile_import/sync when the media is login-gated."),
4473
+ description: withReportNote("Extract and download media from one Instagram post, reel, or tv URL. Use after instagram_profile_content or when the user gives a specific Instagram URL and wants the image, caption/text, reel audio/video tracks, optional muxed MP4, or optional transcript. Reels commonly expose separate video-only and audio-only MP4 tracks; this tool selects the best video and audio tracks and attempts muxing when ffmpeg is available."),
6192
4474
  inputSchema: InstagramMediaDownloadInputSchema,
6193
4475
  outputSchema: InstagramMediaDownloadOutputSchema,
6194
4476
  annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
@@ -6330,16 +4612,16 @@ function renderInstallTerminal(options) {
6330
4612
  "1/1 install surfaces ready",
6331
4613
  colorize("Newest: saved hosted browser profiles for AI visibility. Open Google, share the watch_url, user signs in, close to save. Browser sessions are direct/no-proxy by default.", "lime", color),
6332
4614
  "",
6333
- `${colorize("Tools", "cyan", color)} ${colorize("(46 MCP tools)", "muted", color)}`,
4615
+ `${colorize("Tools", "cyan", color)} ${colorize("(43 MCP tools)", "muted", color)}`,
6334
4616
  toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
6335
4617
  toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "directory_workflow"], color),
6336
4618
  toolRow("build", ["rank_tracker_blueprint", "cron plan", "database prompt"], color),
6337
4619
  toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download"], color),
6338
4620
  toolRow("browser", ["browser_open", "browser_profile_onboard", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
6339
- toolRow("account", ["credits_info", "local reports", "MCP resources"], color),
4621
+ toolRow("account", ["credits_info", "reports", "MCP resources"], color),
6340
4622
  `${colorize("Workflows", "cyan", color)} ${colorize("(MCP + CLI + API)", "muted", color)}`,
6341
4623
  toolRow("route", ["workflow_list", "workflow_suggest", "workflow_run", "workflow_step", "workflow_status", "workflow_artifact_read"], color),
6342
- toolRow("seo", ["directory", "agent-packet", "local audit", "map/serp comparison", "PAA/AIO briefs", "scheduled runs"], color),
4624
+ toolRow("seo", ["directory", "agent-packet", "competitive audit", "map/serp comparison", "PAA/AIO briefs", "scheduled runs"], color),
6343
4625
  "",
6344
4626
  colorize("Usage tips:", "amber", color),
6345
4627
  "Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.",
@@ -6392,10 +4674,10 @@ if (!forceStdio && (interactiveTerminal || wantsHelp)) {
6392
4674
  }
6393
4675
  function readApiKeyFile() {
6394
4676
  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
6395
- const paths = [explicitPath, (0, import_node_path9.join)((0, import_node_os7.homedir)(), ".mcp-scraper-key")].filter(Boolean);
4677
+ const paths = [explicitPath, (0, import_node_path5.join)((0, import_node_os4.homedir)(), ".mcp-scraper-key")].filter(Boolean);
6396
4678
  for (const path of paths) {
6397
4679
  try {
6398
- const value = (0, import_node_fs7.readFileSync)(path, "utf8").trim();
4680
+ const value = (0, import_node_fs4.readFileSync)(path, "utf8").trim();
6399
4681
  if (value) return value;
6400
4682
  } catch {
6401
4683
  }