mcp-scraper 0.3.11 → 0.3.12

package/dist/bin/api-server.js

@@ -17,8 +17,8 @@ loadDotEnv();
 async function main() {
   const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
     import("@hono/node-server"),
-    import("../server-3SA5Q4OF.js"),
-    import("../worker-56IXWOQU.js"),
+    import("../server-CUPDJLWM.js"),
+    import("../worker-FG7ZWEGA.js"),
     import("../db-BE4JVB3V.js")
   ]);
   const PORT = parseInt(process.env.PORT ?? "3001");

package/dist/bin/browser-agent-stdio-server.cjs

@@ -1,17 +1,39 @@
 #!/usr/bin/env node
 "use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 
 // bin/browser-agent-stdio-server.ts
-var import_node_fs2 = require("fs");
-var import_node_os3 = require("os");
-var import_node_path3 = require("path");
+var import_node_fs3 = require("fs");
+var import_node_os4 = require("os");
+var import_node_path4 = require("path");
 var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
 
 // src/mcp/browser-agent-mcp-server.ts
 var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
-var import_node_fs = require("fs");
-var import_node_os2 = require("os");
-var import_node_path2 = require("path");
+var import_node_fs2 = require("fs");
+var import_node_os3 = require("os");
+var import_node_path3 = require("path");
 
 // src/lib/browser-service-env.ts
 function browserServiceProfileName() {
@@ -26,8 +48,137 @@ function browserServiceProfileSaveChanges() {
   return void 0;
 }
 
+// src/services/fanout/export.ts
+var import_node_fs = require("fs");
+var import_node_os = require("os");
+var import_node_path = require("path");
+var import_papaparse = __toESM(require("papaparse"), 1);
+function outputBaseDir() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path.join)((0, import_node_os.homedir)(), "Downloads", "mcp-scraper");
+}
+function safe(value) {
+  return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
+}
+function writeTable(path, rows, delimiter) {
+  (0, import_node_fs.writeFileSync)(path, import_papaparse.default.unparse(rows.length ? rows : [{}], { delimiter }));
+}
+function exportFanout(enriched) {
+  const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
+  const outputDir = outputBaseDir();
+  const relativeDir = (0, import_node_path.join)("fanout", `${stamp}-${safe(enriched.platform)}`);
+  const dir = (0, import_node_path.join)(outputDir, relativeDir);
+  (0, import_node_fs.mkdirSync)(dir, { recursive: true });
+  const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
+  const citationRows = enriched.aggregates.citationOrder.map((c) => {
+    const src = enriched.citedUrls.find((s) => s.url === c.url);
+    return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
+  });
+  const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
+  const browsedOnlyRows = enriched.browsedOnly.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
+  const snippetRows = enriched.snippets.map((s) => ({ domain: s.domain, url: s.url, title: s.title, text: s.text }));
+  const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
+  const relativePaths = {
+    relativeTo: "MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper",
+    dir: relativeDir,
+    json: (0, import_node_path.join)(relativeDir, "fanout.json"),
+    queriesCsv: (0, import_node_path.join)(relativeDir, "queries.csv"),
+    queriesTsv: (0, import_node_path.join)(relativeDir, "queries.tsv"),
+    citationsCsv: (0, import_node_path.join)(relativeDir, "citations.csv"),
+    sourcesCsv: (0, import_node_path.join)(relativeDir, "sources.csv"),
+    browsedOnlyCsv: (0, import_node_path.join)(relativeDir, "browsed-only.csv"),
+    snippetsCsv: (0, import_node_path.join)(relativeDir, "snippets.csv"),
+    domainsCsv: (0, import_node_path.join)(relativeDir, "domains.csv"),
+    report: (0, import_node_path.join)(relativeDir, "report.html")
+  };
+  (0, import_node_fs.writeFileSync)((0, import_node_path.join)(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2));
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.queriesCsv), queryRows, ",");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.queriesTsv), queryRows, "	");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.citationsCsv), citationRows, ",");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.sourcesCsv), sourceRows, ",");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ",");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.snippetsCsv), snippetRows, ",");
+  writeTable((0, import_node_path.join)(outputDir, relativePaths.domainsCsv), domainRows, ",");
+  (0, import_node_fs.writeFileSync)((0, import_node_path.join)(outputDir, relativePaths.report), renderReportHtml(enriched));
+  return relativePaths;
+}
+function esc(s) {
+  return String(s).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+function renderReportHtml(enriched) {
+  const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
+  return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
+<style>
+:root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
+*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
+.wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
+h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
+.stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
+.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
+.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
+.card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
+.bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
+.bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
+.bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
+.tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
+.tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
+.chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
+.chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
+.chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
+.row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
+.row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
+.tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
+.dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
+.pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
+.copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
+.hide{display:none}
+</style>
+</head>
+<body><div class="wrap">
+<h1>AI Search Fan-Out</h1>
+<p class="sub" id="sub"></p>
+<div class="stat" id="stat"></div>
+<div class="cards" id="cards"></div>
+<div id="charts"></div>
+<div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
+<div class="chips" id="chips"></div>
+<div id="list"></div>
+</div>
+<script>
+const D=${data};
+const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
+document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
+document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
+const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
+const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
+function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
+const charts=document.getElementById('charts');
+charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
+let tab='q',filter=null;
+const chips=document.getElementById('chips'),list=document.getElementById('list');
+function render(){
+  chips.innerHTML='';list.innerHTML='';
+  if(tab==='q'){
+    D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
+  } else {
+    const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
+    Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
+    D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
+  }
+}
+function esc(s){return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;')}
+document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
+render();
+</script>
+</body></html>`;
+}
+
 // src/version.ts
-var PACKAGE_VERSION = "0.3.11";
+var PACKAGE_VERSION = "0.3.12";
 
 // src/mcp/browser-agent-tool-schemas.ts
 var import_zod = require("zod");
@@ -369,8 +520,8 @@ var BrowserListSessionsOutputSchema = {
 // src/mcp/replay-annotator.ts
 var import_node_child_process = require("child_process");
 var import_promises = require("fs/promises");
-var import_node_os = require("os");
-var import_node_path = require("path");
+var import_node_os2 = require("os");
+var import_node_path2 = require("path");
 var import_node_util = require("util");
 var execFileAsync = (0, import_node_util.promisify)(import_node_child_process.execFile);
 function finiteNumber(value) {
@@ -380,11 +531,11 @@ function clamp(value, min, max) {
   return Math.min(max, Math.max(min, value));
 }
 function formatAssTime(seconds) {
-  const safe = Math.max(0, seconds);
-  const hours = Math.floor(safe / 3600);
-  const minutes = Math.floor(safe % 3600 / 60);
-  const wholeSeconds = Math.floor(safe % 60);
-  const centiseconds = Math.floor((safe - Math.floor(safe)) * 100);
+  const safe2 = Math.max(0, seconds);
+  const hours = Math.floor(safe2 / 3600);
+  const minutes = Math.floor(safe2 % 3600 / 60);
+  const wholeSeconds = Math.floor(safe2 % 60);
+  const centiseconds = Math.floor((safe2 - Math.floor(safe2)) * 100);
   return `${hours}:${String(minutes).padStart(2, "0")}:${String(wholeSeconds).padStart(2, "0")}.${String(centiseconds).padStart(2, "0")}`;
 }
 function cssHexToAssColor(color) {
@@ -593,8 +744,8 @@ function ffmpegFilterPath(path) {
 async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
   if (!options.annotations.length) throw new Error("annotations must include at least one item");
   const size = await videoSize(inputFilePath);
-  const tmp = await (0, import_promises.mkdtemp)((0, import_node_path.join)((0, import_node_os.tmpdir)(), "mcp-scraper-ass-"));
-  const assPath = (0, import_node_path.join)(tmp, "annotations.ass");
+  const tmp = await (0, import_promises.mkdtemp)((0, import_node_path2.join)((0, import_node_os2.tmpdir)(), "mcp-scraper-ass-"));
+  const assPath = (0, import_node_path2.join)(tmp, "annotations.ass");
   try {
     await (0, import_promises.writeFile)(assPath, buildAssSubtitle(options, size), "utf8");
     await execFileAsync("ffmpeg", [
@@ -662,8 +813,8 @@ function actionResult(tool, sessionId, ok, data, nextRecommendedTool = "browser_
     nextRecommendedTool
   });
 }
-function outputBaseDir() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), "Downloads", "mcp-scraper");
+function outputBaseDir2() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
 }
 function safeFilePart(value) {
   return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120) || "replay";
@@ -672,7 +823,7 @@ function replayFilePath(sessionId, replayId, filename) {
   const requested = filename?.trim();
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
   const name = requested ? safeFilePart(requested).replace(/\.mp4$/i, "") : `${stamp}-${safeFilePart(sessionId)}-${safeFilePart(replayId)}`;
-  return (0, import_node_path2.join)(outputBaseDir(), "browser-replays", `${name}.mp4`);
+  return (0, import_node_path3.join)(outputBaseDir2(), "browser-replays", `${name}.mp4`);
 }
 function slugPart(value) {
   const trimmed = value?.trim().toLowerCase();
@@ -685,6 +836,11 @@ function savedProfileNameFromEmail(email, domain) {
   const domainPart = slugPart(domain);
   return `${value}${domainPart ? `-${domainPart}` : ""}`.slice(0, 80) || null;
 }
+function isEnrichedFanout(value) {
+  if (!value || typeof value !== "object") return false;
+  const candidate = value;
+  return typeof candidate.platform === "string" && typeof candidate.capturedAt === "string" && typeof candidate.prompt === "string" && Array.isArray(candidate.queries) && Array.isArray(candidate.browsedUrls) && Array.isArray(candidate.citedUrls) && Array.isArray(candidate.browsedOnly) && Array.isArray(candidate.snippets) && !!candidate.counts && !!candidate.aggregates;
+}
 function annotatedReplayFilePath(sessionId, replayId, filename) {
   const requested = filename?.trim();
   if (requested) return replayFilePath(sessionId, replayId, requested);
@@ -747,8 +903,8 @@ function registerBrowserAgentMcpTools(server2, opts) {
       }
       const bytes = Buffer.from(await res.arrayBuffer());
       const filePath = replayFilePath(sessionId, replayId, filename);
-      (0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
-      (0, import_node_fs.writeFileSync)(filePath, bytes);
+      (0, import_node_fs2.mkdirSync)((0, import_node_path3.join)(outputBaseDir2(), "browser-replays"), { recursive: true });
+      (0, import_node_fs2.writeFileSync)(filePath, bytes);
       return {
         ok: true,
         data: {
@@ -1207,7 +1363,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
       try {
         const sourcePath = String(downloaded.data.file_path);
         const outputPath = annotatedReplayFilePath(input.session_id, input.replay_id, input.filename);
-        (0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
+        (0, import_node_fs2.mkdirSync)((0, import_node_path3.join)(outputBaseDir2(), "browser-replays"), { recursive: true });
         const result = await annotateReplayVideo(sourcePath, outputPath, {
           annotations: input.annotations,
           sourceWidth: input.source_width,
@@ -1286,7 +1442,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
       annotations: annotations("Capture AI Search Fan-Out")
     },
     async (input) => {
-      const emit = (result, exports2) => structuredResult({
+      const emit = (result, exports3) => structuredResult({
         ok: true,
         tool: "browser_capture_fanout",
         session_id: input.session_id,
@@ -1302,7 +1458,7 @@ function registerBrowserAgentMcpTools(server2, opts) {
         counts: result.counts,
         aggregates: result.aggregates,
         first_party_domain: result.firstPartyDomain,
-        exports: exports2 ?? null,
+        exports: exports3 ?? null,
         ...result.debug ? { debug: result.debug } : {}
       });
       const res = await req("POST", `/agent/sessions/${input.session_id}/capture-fanout`, {
@@ -1314,7 +1470,22 @@ function registerBrowserAgentMcpTools(server2, opts) {
       });
       if (!res.ok) return errorResult("browser_capture_fanout", res.data, input.session_id);
       const hosted = res.data?.result ?? res.data;
-      return emit(hosted, res.data?.exports ?? null);
+      let exports2 = res.data?.exports ?? null;
+      if (input.export && !exports2 && isEnrichedFanout(hosted)) {
+        try {
+          exports2 = exportFanout(hosted);
+        } catch (err) {
+          return errorResult(
+            "browser_capture_fanout",
+            {
+              error: `Fan-out captured but local export failed: ${err instanceof Error ? err.message : String(err)}`,
+              result: hosted
+            },
+            input.session_id
+          );
+        }
+      }
+      return emit(hosted, exports2);
     }
   );
 }
@@ -1322,10 +1493,10 @@ function registerBrowserAgentMcpTools(server2, opts) {
 // bin/browser-agent-stdio-server.ts
 function readApiKeyFile() {
   const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
-  const paths = [explicitPath, (0, import_node_path3.join)((0, import_node_os3.homedir)(), ".mcp-scraper-key")].filter(Boolean);
+  const paths = [explicitPath, (0, import_node_path4.join)((0, import_node_os4.homedir)(), ".mcp-scraper-key")].filter(Boolean);
   for (const path of paths) {
     try {
-      const value = (0, import_node_fs2.readFileSync)(path, "utf8").trim();
+      const value = (0, import_node_fs3.readFileSync)(path, "utf8").trim();
       if (value) return value;
     } catch {
     }