mcp-scraper 0.3.10 → 0.3.11

package/README.md

@@ -218,6 +218,7 @@ env = { MCP_SCRAPER_API_KEY = "sk_live_your_key" }
 - `browser_replay_download` — download and save the replay MP4 under `MCP_SCRAPER_OUTPUT_DIR/browser-replays`.
 - `browser_replay_mark` — while recording, locate a DOM target and return a replay-timed annotation object.
 - `browser_replay_annotate` — download a replay MP4, render timed boxes, circles, underlines, arrows, and labels using annotation objects from `browser_replay_mark` or exact bounds from `browser_locate`, and save a new annotated MP4 under `MCP_SCRAPER_OUTPUT_DIR/browser-replays`.
+- `browser_capture_fanout` — capture ChatGPT/Claude AI-search fan-out from an open logged-in hosted session. Use `export=true` for durable artifacts. It writes full `fanout.json`, query/source/citation/domain/snippet CSVs, TSV, and `report.html` under `MCP_SCRAPER_OUTPUT_DIR/fanout`, returning paths relative to `MCP_SCRAPER_OUTPUT_DIR` so users can share portable paths.
 - `browser_close`
 - `browser_list_sessions`
 
@@ -247,7 +248,7 @@ The `mcp-scraper` and `mcp-scraper-combined` NPX stdio servers also expose saved
 - `BROWSER_AGENT_PROFILE_NAME` is optional and sets the default saved hosted browser profile for `browser-agent` and `mcp-scraper-combined` stdio sessions. Aliases: `BROWSER_SERVICE_PROFILE_NAME`, `KERNEL_BROWSER_PROFILE_NAME`, `KERNEL_PROFILE_NAME`.
 - `BROWSER_AGENT_PROFILE_SAVE_CHANGES=true` is optional hosted setup behavior. It persists cookies and storage back to the named profile when `browser_close` deletes the hosted browser session. Aliases: `BROWSER_SERVICE_PROFILE_SAVE_CHANGES`, `KERNEL_BROWSER_PROFILE_SAVE_CHANGES`, `KERNEL_PROFILE_SAVE_CHANGES`.
 
-Every web intelligence tool call made through `mcp-scraper` or `mcp-scraper-combined` saves a full Markdown report to disk by default and returns the file path in the MCP response. The hosted `/mcp` endpoint returns reports inline only and never writes files. Browser replay downloads are saved by `browser_replay_download` under `MCP_SCRAPER_OUTPUT_DIR/browser-replays`.
+Every web intelligence tool call made through `mcp-scraper` or `mcp-scraper-combined` saves a full Markdown report to disk by default and returns the file path in the MCP response. The hosted `/mcp` endpoint returns reports inline only and never writes files. Browser replay downloads are saved by `browser_replay_download` under `MCP_SCRAPER_OUTPUT_DIR/browser-replays`. AI fan-out exports are saved by `browser_capture_fanout` with `export=true` under `MCP_SCRAPER_OUTPUT_DIR/fanout`, and return relative paths.
 
 ## Updating Existing Installs
 

package/dist/bin/api-server.cjs

@@ -21921,7 +21921,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.3.10";
+    PACKAGE_VERSION = "0.3.11";
   }
 });
 
@@ -24432,7 +24432,9 @@ function writeTable(path6, rows, delimiter) {
 }
 function exportFanout(enriched) {
   const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
-  const dir = (0, import_node_path11.join)(outputBaseDir3(), "fanout", `${stamp}-${safe(enriched.platform)}`);
+  const outputDir = outputBaseDir3();
+  const relativeDir = (0, import_node_path11.join)("fanout", `${stamp}-${safe(enriched.platform)}`);
+  const dir = (0, import_node_path11.join)(outputDir, relativeDir);
   (0, import_node_fs8.mkdirSync)(dir, { recursive: true });
   const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
   const citationRows = enriched.aggregates.citationOrder.map((c) => {
@@ -24440,25 +24442,32 @@ function exportFanout(enriched) {
     return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
   });
   const sourceRows2 = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
+  const browsedOnlyRows = enriched.browsedOnly.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
+  const snippetRows = enriched.snippets.map((s) => ({ domain: s.domain, url: s.url, title: s.title, text: s.text }));
   const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
-  const paths = {
-    dir,
-    json: (0, import_node_path11.join)(dir, "fanout.json"),
-    queriesCsv: (0, import_node_path11.join)(dir, "queries.csv"),
-    queriesTsv: (0, import_node_path11.join)(dir, "queries.tsv"),
-    citationsCsv: (0, import_node_path11.join)(dir, "citations.csv"),
-    sourcesCsv: (0, import_node_path11.join)(dir, "sources.csv"),
-    domainsCsv: (0, import_node_path11.join)(dir, "domains.csv"),
-    report: (0, import_node_path11.join)(dir, "report.html")
+  const relativePaths = {
+    relativeTo: "MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper",
+    dir: relativeDir,
+    json: (0, import_node_path11.join)(relativeDir, "fanout.json"),
+    queriesCsv: (0, import_node_path11.join)(relativeDir, "queries.csv"),
+    queriesTsv: (0, import_node_path11.join)(relativeDir, "queries.tsv"),
+    citationsCsv: (0, import_node_path11.join)(relativeDir, "citations.csv"),
+    sourcesCsv: (0, import_node_path11.join)(relativeDir, "sources.csv"),
+    browsedOnlyCsv: (0, import_node_path11.join)(relativeDir, "browsed-only.csv"),
+    snippetsCsv: (0, import_node_path11.join)(relativeDir, "snippets.csv"),
+    domainsCsv: (0, import_node_path11.join)(relativeDir, "domains.csv"),
+    report: (0, import_node_path11.join)(relativeDir, "report.html")
   };
-  (0, import_node_fs8.writeFileSync)(paths.json, JSON.stringify(enriched, null, 2));
-  writeTable(paths.queriesCsv, queryRows, ",");
-  writeTable(paths.queriesTsv, queryRows, "	");
-  writeTable(paths.citationsCsv, citationRows, ",");
-  writeTable(paths.sourcesCsv, sourceRows2, ",");
-  writeTable(paths.domainsCsv, domainRows, ",");
-  (0, import_node_fs8.writeFileSync)(paths.report, renderReportHtml(enriched));
-  return paths;
+  (0, import_node_fs8.writeFileSync)((0, import_node_path11.join)(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2));
+  writeTable((0, import_node_path11.join)(outputDir, relativePaths.queriesCsv), queryRows, ",");
+  writeTable((0, import_node_path11.join)(outputDir, relativePaths.queriesTsv), queryRows, "	");
+  writeTable((0, import_node_path11.join)(outputDir, relativePaths.citationsCsv), citationRows, ",");
+  writeTable((0, import_node_path11.join)(outputDir, relativePaths.sourcesCsv), sourceRows2, ",");
+  writeTable((0, import_node_path11.join)(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ",");
+  writeTable((0, import_node_path11.join)(outputDir, relativePaths.snippetsCsv), snippetRows, ",");
+  writeTable((0, import_node_path11.join)(outputDir, relativePaths.domainsCsv), domainRows, ",");
+  (0, import_node_fs8.writeFileSync)((0, import_node_path11.join)(outputDir, relativePaths.report), renderReportHtml(enriched));
+  return relativePaths;
 }
 function esc(s) {
   return String(s).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
@@ -24575,6 +24584,7 @@ async function runFanoutCapture(page, input) {
   const cap = new FanoutCdpCapture();
   await cap.attach(page);
   try {
+    if (input.reset) cap.reset();
     const baseline = cap.answerStreamCount();
     if (input.prompt) await sendPrompt(page, input.prompt).catch(() => false);
     const waitMs = input.wait_ms ?? (input.prompt ? 9e4 : 8e3);
@@ -25256,6 +25266,7 @@ function buildBrowserAgentRoutes() {
       prompt: typeof body.prompt === "string" ? body.prompt : void 0,
       wait_ms: typeof body.wait_ms === "number" ? body.wait_ms : void 0,
       first_party_domain: typeof body.first_party_domain === "string" ? body.first_party_domain : void 0,
+      reset: body.reset === true,
       export: body.export === true
     };
     const t0 = Date.now();