mcp-scraper 0.3.10 → 0.3.12

package/dist/bin/mcp-scraper-cli.js

@@ -10,7 +10,7 @@ import {
 import "../chunk-LFATOGDF.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-NEGW2ZEJ.js";
+} from "../chunk-FB5ZJGFN.js";
 
 // src/cli/human-cli.ts
 import { Command } from "commander";

package/dist/bin/mcp-scraper-combined-stdio-server.cjs

@@ -1,10 +1,32 @@
 #!/usr/bin/env node
 "use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 
 // bin/mcp-scraper-combined-stdio-server.ts
-var import_node_fs4 = require("fs");
-var import_node_os4 = require("os");
-var import_node_path5 = require("path");
+var import_node_fs5 = require("fs");
+var import_node_os5 = require("os");
+var import_node_path6 = require("path");
 var import_mcp3 = require("@modelcontextprotocol/sdk/server/mcp.js");
 var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
 
@@ -44,6 +66,29 @@ function youtubeVideoIdFromUrl(url) {
   }
   return null;
 }
+async function readResponseData(res) {
+  const text = await res.text();
+  if (!text.trim()) return null;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return text;
+  }
+}
+function httpErrorPayload(path, res, data) {
+  const objectData = data && typeof data === "object" && !Array.isArray(data) ? data : null;
+  const bodyMessage = objectData ? objectData.message ?? objectData.error ?? objectData.error_code : data;
+  return {
+    ...objectData ?? { body: data },
+    error: objectData?.error ?? objectData?.error_code ?? "mcp_http_error",
+    error_type: "http",
+    retryable: res.status === 429 || res.status >= 500,
+    status: res.status,
+    statusText: res.statusText,
+    path,
+    message: typeof bodyMessage === "string" && bodyMessage.trim() ? bodyMessage : `MCP Scraper HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ""} for ${path}`
+  };
+}
 var HttpMcpToolExecutor = class {
   baseUrl;
   apiKey;
@@ -71,9 +116,9 @@ var HttpMcpToolExecutor = class {
         body: JSON.stringify(body),
         signal: AbortSignal.timeout(timeoutMs)
       });
-      const data = await res.json();
+      const data = await readResponseData(res);
       if (!res.ok) {
-        return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
+        return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path, res, data)) }], isError: true };
       }
       return { content: [{ type: "text", text: JSON.stringify(data) }] };
     } catch (err) {
@@ -106,9 +151,9 @@ var HttpMcpToolExecutor = class {
         },
         signal: AbortSignal.timeout(timeoutMs)
       });
-      const data = await res.json();
+      const data = await readResponseData(res);
       if (!res.ok) {
-        return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
+        return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path, res, data)) }], isError: true };
       }
       return { content: [{ type: "text", text: JSON.stringify(data) }] };
     } catch (err) {
@@ -252,9 +297,9 @@ var HttpMcpToolExecutor = class {
 
 // src/mcp/browser-agent-mcp-server.ts
 var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
-var import_node_fs = require("fs");
-var import_node_os2 = require("os");
-var import_node_path2 = require("path");
+var import_node_fs2 = require("fs");
+var import_node_os3 = require("os");
+var import_node_path3 = require("path");
 
 // src/lib/browser-service-env.ts
 function browserServiceProfileName() {
@@ -269,8 +314,137 @@ function browserServiceProfileSaveChanges() {
   return void 0;
 }
 
+// src/services/fanout/export.ts
+var import_node_fs = require("fs");
+var import_node_os = require("os");
+var import_node_path = require("path");
+var import_papaparse = __toESM(require("papaparse"), 1);
+function outputBaseDir() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path.join)((0, import_node_os.homedir)(), "Downloads", "mcp-scraper");
+}
+function safe(value) {
+  return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
+}
+function writeTable(path, rows, delimiter) {
+  (0, import_node_fs.writeFileSync)(path, import_papaparse.default.unparse(rows.length ? rows : [{}], { delimiter }));
+}
+function exportFanout(enriched) {
+  const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
+  const outputDir = outputBaseDir();
+  const relativeDir = (0, import_node_path.join)("fanout", `${stamp}-${safe(enriched.platform)}`);
+  const dir = (0, import_node_path.join)(outputDir, relativeDir);
+  (0, import_node_fs.mkdirSync)(dir, { recursive: true });
+  const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
+  const citationRows = enriched.aggregates.citationOrder.map((c) => {
+    const src = enriched.citedUrls.find((s) => s.url === c.url);
+    return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
+  });
+  const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
+  const browsedOnlyRows = enriched.browsedOnly.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
+  const snippetRows = enriched.snippets.map((s) => ({ domain: s.domain, url: s.url, title: s.title, text: s.text }));
+  const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
+  const relativePaths = {
+    relativeTo: "MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper",
+    dir: relativeDir,
+    json: (0, import_node_path.join)(relativeDir, "fanout.json"),
+    queriesCsv: (0, import_node_path.join)(relativeDir, "queries.csv"),
+    queriesTsv: (0, import_node_path.join)(relativeDir, "queries.tsv"),
+    citationsCsv: (0, import_node_path.join)(relativeDir, "citations.csv"),
+    sourcesCsv: (0, import_node_path.join)(relativeDir, "sources.csv"),
+    browsedOnlyCsv: (0, import_node_path.join)(relativeDir, "browsed-only.csv"),
+    snippetsCsv: (0, import_node_path.join)(relativeDir, "snippets.csv"),
+    domainsCsv: (0, import_node_path.join)(relativeDir, "domains.csv"),
+    report: (0, import_node_path.join)(relativeDir, "report.html")
+  };
+  (0, import_node_fs.writeFileSync)((0, import_node_path.join)(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2));
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.queriesCsv), queryRows, ",");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.queriesTsv), queryRows, "	");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.citationsCsv), citationRows, ",");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.sourcesCsv), sourceRows, ",");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ",");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.snippetsCsv), snippetRows, ",");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.domainsCsv), domainRows, ",");
+  (0, import_node_fs.writeFileSync)((0, import_node_path.join)(outputDir, relativePaths.report), renderReportHtml(enriched));
+  return relativePaths;
+}
+function esc(s) {
+  return String(s).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+function renderReportHtml(enriched) {
+  const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
+  return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
+<style>
+:root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
+*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
+.wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
+h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
+.stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
+.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
+.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
+.card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
+.bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
+.bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
+.bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
+.tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
+.tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
+.chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
+.chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
+.chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
+.row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
+.row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
+.tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
+.dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
+.pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
+.copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
+.hide{display:none}
+</style>
+</head>
+<body><div class="wrap">
+<h1>AI Search Fan-Out</h1>
+<p class="sub" id="sub"></p>
+<div class="stat" id="stat"></div>
+<div class="cards" id="cards"></div>
+<div id="charts"></div>
+<div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
+<div class="chips" id="chips"></div>
+<div id="list"></div>
+</div>
+<script>
+const D=${data};
+const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
+document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
+document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
+const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
+const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
+function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
+const charts=document.getElementById('charts');
+charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
+let tab='q',filter=null;
+const chips=document.getElementById('chips'),list=document.getElementById('list');
+function render(){
+  chips.innerHTML='';list.innerHTML='';
+  if(tab==='q'){
+    D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
+  } else {
+    const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
+    Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
+    D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
+  }
+}
+function esc(s){return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;')}
+document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
+render();
+</script>
+</body></html>`;
+}
+
 // src/version.ts
-var PACKAGE_VERSION = "0.3.10";
+var PACKAGE_VERSION = "0.3.12";
 
 // src/mcp/browser-agent-tool-schemas.ts
 var import_zod = require("zod");
@@ -407,7 +581,7 @@ var BrowserCaptureFanoutInputSchema = {
   wait_ms: import_zod.z.number().int().min(0).max(18e4).optional().describe("How long to wait for the answer stream to finish before parsing. Defaults to 90000 when a prompt is sent, 8000 for passive capture. Capture ends as soon as the answer stream finishes."),
   first_party_domain: import_zod.z.string().optional().describe("The brand/site being researched, for example example.com. Sources on this domain are tagged First-party/vendor. Set this when the prompt is about a specific business."),
   reset: import_zod.z.boolean().default(false).describe("Clear any previously buffered stream for this page before capturing. Use when re-running to avoid mixing turns."),
-  export: import_zod.z.boolean().default(false).describe("When true, write JSON, CSV, TSV exports (queries, citations, sources, domains) and a self-contained HTML report to MCP_SCRAPER_OUTPUT_DIR/fanout and return their paths.")
+  export: import_zod.z.boolean().default(false).describe("When true, write JSON, CSV, TSV exports (queries, citations, all sources, browsed-only sources, snippets, domains) and a self-contained HTML report to MCP_SCRAPER_OUTPUT_DIR/fanout, returning paths relative to MCP_SCRAPER_OUTPUT_DIR.")
 };
 var FanoutSourceOutput = import_zod.z.object({
   url: import_zod.z.string(),
@@ -449,15 +623,18 @@ var BrowserCaptureFanoutOutputSchema = {
   }).describe("Objective aggregates: top sourced sites by frequency, citation order, and URL-category counts."),
   first_party_domain: import_zod.z.string().nullable(),
   exports: import_zod.z.object({
+    relativeTo: import_zod.z.string(),
     dir: import_zod.z.string(),
     json: import_zod.z.string(),
     queriesCsv: import_zod.z.string(),
     queriesTsv: import_zod.z.string(),
     citationsCsv: import_zod.z.string(),
     sourcesCsv: import_zod.z.string(),
+    browsedOnlyCsv: import_zod.z.string(),
+    snippetsCsv: import_zod.z.string(),
     domainsCsv: import_zod.z.string(),
     report: import_zod.z.string()
-  }).nullable().describe("Local file paths when export=true, otherwise null."),
+  }).nullable().describe("Relative export paths when export=true, otherwise null. Paths are relative to MCP_SCRAPER_OUTPUT_DIR, or ~/Downloads/mcp-scraper when that env var is not set."),
   debug: import_zod.z.object({
     interceptorReady: import_zod.z.boolean(),
     rawBytes: import_zod.z.number().int().min(0),
@@ -609,8 +786,8 @@ var BrowserListSessionsOutputSchema = {
 // src/mcp/replay-annotator.ts
 var import_node_child_process = require("child_process");
 var import_promises = require("fs/promises");
-var import_node_os = require("os");
-var import_node_path = require("path");
+var import_node_os2 = require("os");
+var import_node_path2 = require("path");
 var import_node_util = require("util");
 var execFileAsync = (0, import_node_util.promisify)(import_node_child_process.execFile);
 function finiteNumber(value) {
@@ -620,11 +797,11 @@ function clamp(value, min, max) {
   return Math.min(max, Math.max(min, value));
 }
 function formatAssTime(seconds) {
-  const safe = Math.max(0, seconds);
-  const hours = Math.floor(safe / 3600);
-  const minutes = Math.floor(safe % 3600 / 60);
-  const wholeSeconds = Math.floor(safe % 60);
-  const centiseconds = Math.floor((safe - Math.floor(safe)) * 100);
+  const safe2 = Math.max(0, seconds);
+  const hours = Math.floor(safe2 / 3600);
+  const minutes = Math.floor(safe2 % 3600 / 60);
+  const wholeSeconds = Math.floor(safe2 % 60);
+  const centiseconds = Math.floor((safe2 - Math.floor(safe2)) * 100);
   return `${hours}:${String(minutes).padStart(2, "0")}:${String(wholeSeconds).padStart(2, "0")}.${String(centiseconds).padStart(2, "0")}`;
 }
 function cssHexToAssColor(color) {
@@ -833,8 +1010,8 @@ function ffmpegFilterPath(path) {
 async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
   if (!options.annotations.length) throw new Error("annotations must include at least one item");
   const size = await videoSize(inputFilePath);
-  const tmp = await (0, import_promises.mkdtemp)((0, import_node_path.join)((0, import_node_os.tmpdir)(), "mcp-scraper-ass-"));
-  const assPath = (0, import_node_path.join)(tmp, "annotations.ass");
+  const tmp = await (0, import_promises.mkdtemp)((0, import_node_path2.join)((0, import_node_os2.tmpdir)(), "mcp-scraper-ass-"));
+  const assPath = (0, import_node_path2.join)(tmp, "annotations.ass");
   try {
     await (0, import_promises.writeFile)(assPath, buildAssSubtitle(options, size), "utf8");
     await execFileAsync("ffmpeg", [
@@ -902,8 +1079,8 @@ function actionResult(tool, sessionId, ok, data, nextRecommendedTool = "browser_
     nextRecommendedTool
   });
 }
-function outputBaseDir() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), "Downloads", "mcp-scraper");
+function outputBaseDir2() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
 }
 function safeFilePart(value) {
   return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120) || "replay";
@@ -912,7 +1089,7 @@ function replayFilePath(sessionId, replayId, filename) {
   const requested = filename?.trim();
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
   const name = requested ? safeFilePart(requested).replace(/\.mp4$/i, "") : `${stamp}-${safeFilePart(sessionId)}-${safeFilePart(replayId)}`;
-  return (0, import_node_path2.join)(outputBaseDir(), "browser-replays", `${name}.mp4`);
+  return (0, import_node_path3.join)(outputBaseDir2(), "browser-replays", `${name}.mp4`);
 }
 function slugPart(value) {
   const trimmed = value?.trim().toLowerCase();
@@ -925,6 +1102,11 @@ function savedProfileNameFromEmail(email, domain) {
   const domainPart = slugPart(domain);
   return `${value}${domainPart ? `-${domainPart}` : ""}`.slice(0, 80) || null;
 }
+function isEnrichedFanout(value) {
+  if (!value || typeof value !== "object") return false;
+  const candidate = value;
+  return typeof candidate.platform === "string" && typeof candidate.capturedAt === "string" && typeof candidate.prompt === "string" && Array.isArray(candidate.queries) && Array.isArray(candidate.browsedUrls) && Array.isArray(candidate.citedUrls) && Array.isArray(candidate.browsedOnly) && Array.isArray(candidate.snippets) && !!candidate.counts && !!candidate.aggregates;
+}
 function annotatedReplayFilePath(sessionId, replayId, filename) {
   const requested = filename?.trim();
   if (requested) return replayFilePath(sessionId, replayId, requested);
@@ -982,8 +1164,8 @@ function registerBrowserAgentMcpTools(server2, opts) {
       }
       const bytes = Buffer.from(await res.arrayBuffer());
       const filePath = replayFilePath(sessionId, replayId, filename);
-      (0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
-      (0, import_node_fs.writeFileSync)(filePath, bytes);
+      (0, import_node_fs2.mkdirSync)((0, import_node_path3.join)(outputBaseDir2(), "browser-replays"), { recursive: true });
+      (0, import_node_fs2.writeFileSync)(filePath, bytes);
       return {
         ok: true,
         data: {
@@ -1442,7 +1624,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
       try {
         const sourcePath = String(downloaded.data.file_path);
         const outputPath = annotatedReplayFilePath(input.session_id, input.replay_id, input.filename);
-        (0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
+        (0, import_node_fs2.mkdirSync)((0, import_node_path3.join)(outputBaseDir2(), "browser-replays"), { recursive: true });
         const result = await annotateReplayVideo(sourcePath, outputPath, {
           annotations: input.annotations,
           sourceWidth: input.source_width,
@@ -1515,13 +1697,13 @@ function registerBrowserAgentMcpTools(server2, opts) {
     "browser_capture_fanout",
     {
       title: "Capture AI Search Fan-Out",
-      description: `Capture the query fan-out behind a ChatGPT or Claude web-search answer for Answer Engine Optimization (AEO): the exact sub-queries the model issued, every researched URL split into cited vs browsed-only (with citation frequency and snippet), each source tagged by category (First-party/vendor, News/media, Reddit, Social/video, Encyclopedia, Review site, Docs, Blog), plus top sourced sites and citation order. This returns the raw structured data; YOU analyze it \u2014 classify each sub-query by funnel stage (Problem-aware, Solution-aware, Decision-aware, Retention) and type (BoFu, Branded, Comparison, How-to, Search operator), name the brands the model researched, and give AEO insights. Sample asks: "capture the fan-out for this ChatGPT answer", "what did Claude search and cite for best CRM", "show the sub-queries and sources behind this AI answer". WRITE NOTE: passing prompt submits a real message in the user's logged-in account (a new conversation turn) \u2014 only send when the user wants that; omit prompt to capture a prompt the user just ran. Setup, if not already connected or if the profile disconnected: call browser_profile_onboard, give the user the watch_url, let them complete login, then call browser_profile_status until status is AUTHENTICATED. After that, browser_open the saved direct no-proxy profile and go to chatgpt.com or claude.ai. Fan-out is captured only as it streams, so the session must be open when the prompt runs. NOT for Google AI Overview citations \u2014 use harvest_paa for those; this tool is ChatGPT and Claude only.`,
+      description: `Capture the query fan-out behind a ChatGPT or Claude web-search answer for Answer Engine Optimization (AEO): the exact sub-queries the model issued, every researched URL split into cited vs browsed-only (with citation frequency and snippet), each source tagged by category (First-party/vendor, News/media, Reddit, Social/video, Encyclopedia, Review site, Docs, Blog), plus top sourced sites and citation order. Set export=true for durable JSON, CSV, TSV, and HTML artifacts; export paths are returned relative to MCP_SCRAPER_OUTPUT_DIR. This returns the raw structured data; YOU analyze it \u2014 classify each sub-query by funnel stage (Problem-aware, Solution-aware, Decision-aware, Retention) and type (BoFu, Branded, Comparison, How-to, Search operator), name the brands the model researched, and give AEO insights. Sample asks: "capture the fan-out for this ChatGPT answer", "what did Claude search and cite for best CRM", "show the sub-queries and sources behind this AI answer". WRITE NOTE: passing prompt submits a real message in the user's logged-in account (a new conversation turn) \u2014 only send when the user wants that; omit prompt to capture a prompt the user just ran. Setup, if not already connected or if the profile disconnected: call browser_profile_onboard, give the user the watch_url, let them complete login, then call browser_profile_status until status is AUTHENTICATED. After that, browser_open the saved direct no-proxy profile and go to chatgpt.com or claude.ai. Fan-out is captured only as it streams, so the session must be open when the prompt runs. NOT for Google AI Overview citations \u2014 use harvest_paa for those; this tool is ChatGPT and Claude only.`,
       inputSchema: BrowserCaptureFanoutInputSchema,
       outputSchema: BrowserCaptureFanoutOutputSchema,
       annotations: annotations("Capture AI Search Fan-Out")
     },
     async (input) => {
-      const emit = (result, exports2) => structuredResult({
+      const emit = (result, exports3) => structuredResult({
         ok: true,
         tool: "browser_capture_fanout",
         session_id: input.session_id,
@@ -1537,31 +1719,47 @@ function registerBrowserAgentMcpTools(server2, opts) {
         counts: result.counts,
         aggregates: result.aggregates,
         first_party_domain: result.firstPartyDomain,
-        exports: exports2 ?? null,
+        exports: exports3 ?? null,
         ...result.debug ? { debug: result.debug } : {}
       });
       const res = await req("POST", `/agent/sessions/${input.session_id}/capture-fanout`, {
         prompt: input.prompt,
         wait_ms: input.wait_ms,
         first_party_domain: input.first_party_domain,
-        export: false
+        reset: input.reset,
+        export: input.export
       });
       if (!res.ok) return errorResult("browser_capture_fanout", res.data, input.session_id);
       const hosted = res.data?.result ?? res.data;
-      return emit(hosted, res.data?.exports ?? null);
+      let exports2 = res.data?.exports ?? null;
+      if (input.export && !exports2 && isEnrichedFanout(hosted)) {
+        try {
+          exports2 = exportFanout(hosted);
+        } catch (err) {
+          return errorResult(
+            "browser_capture_fanout",
+            {
+              error: `Fan-out captured but local export failed: ${err instanceof Error ? err.message : String(err)}`,
+              result: hosted
+            },
+            input.session_id
+          );
+        }
+      }
+      return emit(hosted, exports2);
     }
   );
 }
 
 // src/mcp/paa-mcp-server.ts
 var import_mcp2 = require("@modelcontextprotocol/sdk/server/mcp.js");
-var import_node_fs3 = require("fs");
-var import_node_path4 = require("path");
+var import_node_fs4 = require("fs");
+var import_node_path5 = require("path");
 
 // src/mcp/mcp-response-formatter.ts
-var import_node_fs2 = require("fs");
-var import_node_os3 = require("os");
-var import_node_path3 = require("path");
+var import_node_fs3 = require("fs");
+var import_node_os4 = require("os");
+var import_node_path4 = require("path");
 
 // src/errors.ts
 function sanitizeVendorName(message) {
@@ -1697,17 +1895,17 @@ function reportTitle(full) {
   const title = full.split("\n").find((line) => line.startsWith("# "));
   return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
 }
-function outputBaseDir2() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
+function outputBaseDir3() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path4.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper");
 }
 function saveFullReport(full) {
   if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
-  const outDir = outputBaseDir2();
+  const outDir = outputBaseDir3();
   try {
-    (0, import_node_fs2.mkdirSync)(outDir, { recursive: true });
+    (0, import_node_fs3.mkdirSync)(outDir, { recursive: true });
     const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-    const file = (0, import_node_path3.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
-    (0, import_node_fs2.writeFileSync)(file, full, "utf8");
+    const file = (0, import_node_path4.join)(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
+    (0, import_node_fs3.writeFileSync)(file, full, "utf8");
     return file;
   } catch {
     return null;
@@ -1716,12 +1914,12 @@ function saveFullReport(full) {
 function persistScreenshotLocally(base64, url) {
   if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
   try {
-    const dir = (0, import_node_path3.join)(outputBaseDir2(), "screenshots");
-    (0, import_node_fs2.mkdirSync)(dir, { recursive: true });
+    const dir = (0, import_node_path4.join)(outputBaseDir3(), "screenshots");
+    (0, import_node_fs3.mkdirSync)(dir, { recursive: true });
     const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
     const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
-    const filePath = (0, import_node_path3.join)(dir, `${stamp}-${slug}.png`);
-    (0, import_node_fs2.writeFileSync)(filePath, Buffer.from(base64, "base64"));
+    const filePath = (0, import_node_path4.join)(dir, `${stamp}-${slug}.png`);
+    (0, import_node_fs3.writeFileSync)(filePath, Buffer.from(base64, "base64"));
     return filePath;
   } catch {
     return null;
@@ -3062,7 +3260,7 @@ function formatInstagramProfileContent(raw, input) {
   const itemRows = items.slice(0, 100).map(
     (item, i) => `| ${i + 1} | ${item.type} | \`${item.shortcode}\` | ${item.url} | ${cell(item.firstSeenStage ?? "")} |`
   ).join("\n");
-  const browserLabel = "hosted browser";
+  const browserLabel = browser.profileName ? `hosted browser profile ${browser.profileName}` : "hosted browser";
   const full = [
     `# Instagram Profile Content: ${d.handle ?? input.handle ?? input.url ?? "profile"}`,
     `**Collected:** ${items.length} items \xB7 posts ${typeCounts?.post ?? 0} \xB7 reels ${typeCounts?.reel ?? 0} \xB7 tv ${typeCounts?.tv ?? 0}`,
@@ -3137,7 +3335,7 @@ function formatInstagramMediaDownload(raw, input) {
   const transcriptText = transcript?.text ?? "";
   const chunks = transcript?.chunks ?? [];
   const browser = structuredInstagramBrowser(d.browser);
-  const browserLabel = "hosted browser";
+  const browserLabel = browser.profileName ? `hosted browser profile ${browser.profileName}` : "hosted browser";
   const downloadRows = downloads.map((download, i) => {
     const status = download.error ? `error: ${cell(download.error)}` : `${download.sizeBytes ?? 0} bytes`;
     return `| ${i + 1} | ${download.kind} | ${download.savedPath ? `\`${download.savedPath}\`` : "\u2014"} | ${status} |`;
@@ -3382,13 +3580,17 @@ var FacebookVideoTranscribeInputSchema = {
 var InstagramProfileContentInputSchema = {
   handle: import_zod3.z.string().min(1).optional().describe("Instagram handle, with or without @. Provide handle or url."),
   url: import_zod3.z.string().url().optional().describe("Instagram profile URL, e.g. https://www.instagram.com/nasaartemis/. Provide handle or url."),
-  maxItems: import_zod3.z.number().int().min(1).max(500).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 500. Use higher values only when the user asks for a fuller archive."),
-  maxScrolls: import_zod3.z.number().int().min(0).max(100).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 100. Increase for long profiles when Instagram continues loading more grid links."),
+  profile: import_zod3.z.string().min(1).optional().describe("Optional saved hosted browser profile name to load authenticated Instagram access. If omitted, the server uses its configured default profile when present."),
+  saveProfileChanges: import_zod3.z.boolean().optional().describe("Whether to save browser changes back to the hosted profile. Leave unset unless intentionally updating the saved login session."),
+  maxItems: import_zod3.z.number().int().min(1).max(2e3).default(50).describe("Maximum profile grid post/reel/tv URLs to collect. Default 50, maximum 2000. Use higher values only when the user asks for a fuller archive."),
+  maxScrolls: import_zod3.z.number().int().min(0).max(250).default(10).describe("Maximum pagination scroll attempts. Default 10, maximum 250. Increase for long profiles when Instagram continues loading more grid links."),
   scrollDelayMs: import_zod3.z.number().int().min(250).max(5e3).default(1200).describe("Delay after each pagination scroll before collecting newly loaded links. Default 1200ms. Increase to 2000-3000ms when Instagram loads slowly."),
   stableScrollLimit: import_zod3.z.number().int().min(1).max(10).default(4).describe("Stop after this many consecutive scrolls with no new links or scroll progress. Default 4.")
 };
 var InstagramMediaDownloadInputSchema = {
   url: import_zod3.z.string().url().describe("Instagram post, reel, or tv URL, e.g. https://www.instagram.com/reel/SHORTCODE/. The tool renders the page, extracts text, image metadata, and Instagram CDN media tracks."),
+  profile: import_zod3.z.string().min(1).optional().describe("Optional saved hosted browser profile name to load authenticated Instagram access. If omitted, the server uses its configured default profile when present."),
+  saveProfileChanges: import_zod3.z.boolean().optional().describe("Whether to save browser changes back to the hosted profile. Leave unset unless intentionally updating the saved login session."),
   mediaTypes: import_zod3.z.array(import_zod3.z.enum(["image", "video", "audio"])).default(["image", "video", "audio"]).describe("Which media types to download when downloadMedia is true. Reels commonly expose separate video-only and audio-only MP4 tracks."),
   downloadMedia: import_zod3.z.boolean().default(true).describe("Download extracted text/media files to the MCP Scraper output directory when the API server can write files. Always returns extracted media URLs even when false."),
   downloadAllTracks: import_zod3.z.boolean().default(false).describe("Download every captured Instagram MP4 track instead of only the selected best video and audio tracks. Use false by default to avoid duplicate bitrates."),
@@ -3800,8 +4002,8 @@ var InstagramBrowserOutput = import_zod3.z.object({
   executablePathConfigured: import_zod3.z.boolean()
 });
 var InstagramPaginationOutput = import_zod3.z.object({
-  maxItems: import_zod3.z.number().int().min(1).max(500),
-  maxScrolls: import_zod3.z.number().int().min(0).max(100),
+  maxItems: import_zod3.z.number().int().min(1).max(2e3),
+  maxScrolls: import_zod3.z.number().int().min(0).max(250),
   attemptedScrolls: import_zod3.z.number().int().min(0),
   stableScrolls: import_zod3.z.number().int().min(0),
   stableScrollLimit: import_zod3.z.number().int().min(1).max(10),
@@ -4416,8 +4618,8 @@ function localPlanningToolAnnotations(title) {
 }
 function listSavedReports() {
   try {
-    const dir = outputBaseDir2();
-    return (0, import_node_fs3.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs3.statSync)((0, import_node_path4.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
+    const dir = outputBaseDir3();
+    return (0, import_node_fs4.readdirSync)(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: (0, import_node_fs4.statSync)((0, import_node_path5.join)(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
   } catch {
     return [];
   }
@@ -4441,9 +4643,9 @@ function registerSavedReportResources(server2) {
     },
     async (uri, variables) => {
       const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
-      const filename = (0, import_node_path4.basename)(decodeURIComponent(String(requested ?? "")));
+      const filename = (0, import_node_path5.basename)(decodeURIComponent(String(requested ?? "")));
       if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
-      const text = (0, import_node_fs3.readFileSync)((0, import_node_path4.join)(outputBaseDir2(), filename), "utf8");
+      const text = (0, import_node_fs4.readFileSync)((0, import_node_path5.join)(outputBaseDir3(), filename), "utf8");
       return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
     }
   );
@@ -4743,10 +4945,10 @@ if (!forceStdio && (interactiveTerminal || wantsHelp)) {
 }
 function readApiKeyFile() {
   const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
-  const paths = [explicitPath, (0, import_node_path5.join)((0, import_node_os4.homedir)(), ".mcp-scraper-key")].filter(Boolean);
+  const paths = [explicitPath, (0, import_node_path6.join)((0, import_node_os5.homedir)(), ".mcp-scraper-key")].filter(Boolean);
   for (const path of paths) {
     try {
-      const value = (0, import_node_fs4.readFileSync)(path, "utf8").trim();
+      const value = (0, import_node_fs5.readFileSync)(path, "utf8").trim();
       if (value) return value;
     } catch {
     }