mcp-scraper 0.2.24 → 0.3.1

package/dist/chunk-NXRWFOEZ.js

@@ -0,0 +1,816 @@
+// src/services/fanout/cdp-capture.ts
+var ANSWER_RE = /\/backend-api\/(f\/)?conversation$|\/chat_conversations\/[^/]+\/(completion|retry_completion)$|\/completion$/i;
+var FanoutCdpCapture = class {
+  client = null;
+  reqUrl = /* @__PURE__ */ new Map();
+  ctById = /* @__PURE__ */ new Map();
+  answerIds = /* @__PURE__ */ new Set();
+  bodies = [];
+  pending = [];
+  answerStreams = 0;
+  async attach(page) {
+    if (this.client) return;
+    const client = await page.context().newCDPSession(page);
+    this.client = client;
+    await client.send("Network.enable", { maxTotalBufferSize: 2e8, maxResourceBufferSize: 2e8 });
+    client.on("Network.requestWillBeSent", (e) => {
+      const id = e.requestId;
+      const req = e.request;
+      if (id) this.reqUrl.set(id, req?.url || "");
+    });
+    client.on("Network.responseReceived", (e) => {
+      const id = e.requestId;
+      const res = e.response;
+      const url = res?.url || this.reqUrl.get(id) || "";
+      const headers = res?.headers || {};
+      const ct = String(headers["content-type"] || headers["Content-Type"] || "");
+      if (id) this.ctById.set(id, ct);
+      if (id && (this.isAnswerUrl(url) || /event-stream/i.test(ct))) this.answerIds.add(id);
+    });
+    client.on("Network.loadingFinished", (e) => this.grab(e.requestId));
+  }
+  reset() {
+    this.bodies = [];
+    this.answerStreams = 0;
+    this.answerIds.clear();
+    this.pending = [];
+  }
+  answerStreamCount() {
+    return this.answerStreams;
+  }
+  async drain() {
+    await Promise.all(this.pending);
+  }
+  bestAnswerText() {
+    const eventStreams = this.bodies.filter((b) => /event-stream/i.test(b.ct));
+    const answerStreams = eventStreams.filter((b) => this.isAnswerUrl(b.url)).sort((a, b) => b.text.length - a.text.length);
+    if (answerStreams.length) return answerStreams.map((b) => b.text).join("\n");
+    const answerJson = this.bodies.filter((b) => this.isAnswerUrl(b.url)).sort((a, b) => b.text.length - a.text.length);
+    if (answerJson.length) return answerJson[0].text;
+    if (eventStreams.length) return eventStreams.sort((a, b) => b.text.length - a.text.length)[0].text;
+    return "";
+  }
+  async detach() {
+    try {
+      await this.client?.detach();
+    } catch {
+      this.client = null;
+    }
+    this.client = null;
+  }
+  isAnswerUrl(url) {
+    let path = "";
+    try {
+      path = new URL(url).pathname;
+    } catch {
+      path = String(url || "").split("?")[0];
+    }
+    return ANSWER_RE.test(path);
+  }
+  grab(requestId) {
+    if (!requestId || !this.client || !this.answerIds.has(requestId)) return;
+    const url = this.reqUrl.get(requestId) || "";
+    const ct = this.ctById.get(requestId) || "";
+    const client = this.client;
+    const p = client.send("Network.getResponseBody", { requestId }).then((r) => {
+      const text = r.base64Encoded ? Buffer.from(String(r.body), "base64").toString("utf8") : String(r.body || "");
+      if (text) {
+        this.bodies.push({ url, ct, text });
+        if (this.isAnswerUrl(url)) this.answerStreams++;
+      }
+    }).catch(() => void 0);
+    this.pending.push(p);
+  }
+};
+
+// src/services/fanout/parse-helpers.ts
+var INTERNAL = /chatgpt\.com|openai\.com|oaiusercontent|oaistatic|claude\.ai|anthropic\.com/i;
+var TRACK = /^(utm_|oai|fbclid|gclid|ref$|ref_|sa$|usg$|ved$|sca_esv$|sources$)/i;
+function domainOf(u) {
+  try {
+    return new URL(u).hostname.replace(/^www\./, "");
+  } catch {
+    return "";
+  }
+}
+function normUrl(u) {
+  try {
+    const x = new URL(u);
+    x.hash = "";
+    for (const k of [...x.searchParams.keys()]) {
+      if (TRACK.test(k)) x.searchParams.delete(k);
+    }
+    return x.toString().replace(/\/$/, "").toLowerCase();
+  } catch {
+    return String(u || "").toLowerCase();
+  }
+}
+function isExternal(u) {
+  const h = domainOf(u);
+  return Boolean(h) && !INTERNAL.test(h);
+}
+function unesc(s) {
+  try {
+    return JSON.parse('"' + s + '"');
+  } catch {
+    return String(s).replace(/\\(.)/g, "$1");
+  }
+}
+function stripCitationMarkers(s) {
+  return String(s || "").replace(/[-]/g, "").replace(/cite(?:turn\d+\w+?\d+)+/g, "").replace(/\s+/g, " ").trim();
+}
+function sliceBalanced(text, openIdx) {
+  const open = text[openIdx];
+  const close = open === "[" ? "]" : "}";
+  let depth = 0;
+  let inStr = false;
+  let esc2 = false;
+  for (let i = openIdx; i < text.length; i++) {
+    const ch = text[i];
+    if (inStr) {
+      if (esc2) esc2 = false;
+      else if (ch === "\\") esc2 = true;
+      else if (ch === '"') inStr = false;
+      continue;
+    }
+    if (ch === '"') {
+      inStr = true;
+      continue;
+    }
+    if (ch === open) depth++;
+    else if (ch === close) {
+      depth--;
+      if (depth === 0) return text.slice(openIdx, i + 1);
+    }
+  }
+  return null;
+}
+function findJsonValues(text, field) {
+  const out = [];
+  const re = new RegExp('"' + field + '"\\s*:\\s*(\\[|\\{)', "g");
+  let m;
+  while (m = re.exec(text)) {
+    const openIdx = m.index + m[0].length - 1;
+    const slice = sliceBalanced(text, openIdx);
+    if (slice) {
+      try {
+        out.push(JSON.parse(slice));
+      } catch {
+        continue;
+      }
+    }
+  }
+  return out;
+}
+function stringsInArrayBody(arrBody) {
+  const out = [];
+  const re = /"((?:[^"\\]|\\.){2,300}?)"/g;
+  let m;
+  while (m = re.exec(arrBody)) out.push(unesc(m[1]));
+  return out;
+}
+function cleanQuery(q) {
+  const t = (q || "").trim();
+  if (t.length < 2 || t.length > 300) return null;
+  return t;
+}
+
+// src/services/fanout/adapters/chatgpt.ts
+function roundOf(refId) {
+  const m = /^turn(\d+)/.exec(refId || "");
+  return m ? Number(m[1]) : null;
+}
+function normRef(r) {
+  if (typeof r === "string") {
+    const m = /turn(\d+)(search|news|ref|video|image|product)(\d+)/.exec(r);
+    return m ? `turn${m[1]}${m[2]}${m[3]}` : null;
+  }
+  if (r && typeof r === "object") {
+    const o = r;
+    if (o.turn_index != null) {
+      return `turn${o.turn_index}${o.ref_type || "search"}${o.ref_index != null ? o.ref_index : ""}`;
+    }
+  }
+  return null;
+}
+function refsInText(s) {
+  return (String(s || "").match(/turn\d+(?:search|news|ref|video|image|product)\d+/g) || []).map(normRef).filter((x) => Boolean(x));
+}
+function collectInto(arr, v) {
+  if (Array.isArray(v)) arr.push(...v);
+  else if (v && typeof v === "object") arr.push(v);
+}
+function reconstructFromSsePatches(raw) {
+  const contentReferences = [];
+  const searchResultGroups = [];
+  const citations = [];
+  const citedUrls = /* @__PURE__ */ new Set();
+  const state = { found: false };
+  const handleOp = (op) => {
+    if (!op || typeof op !== "object") return;
+    const o = op;
+    const v = o.v;
+    if (o.o === "patch" && Array.isArray(v)) {
+      state.found = true;
+      v.forEach(handleOp);
+      return;
+    }
+    const path = typeof o.p === "string" ? o.p : "";
+    if ((path === "" || path === "/message") && v && typeof v === "object") {
+      const root = v;
+      const msg = root.message || root;
+      const md = msg && msg.metadata;
+      if (md) {
+        collectInto(contentReferences, md.content_references);
+        collectInto(searchResultGroups, md.search_result_groups);
+        collectInto(citations, md.citations);
+        state.found = true;
+      }
+    }
+    if (!path) return;
+    if (/\/safe_urls$/.test(path) && Array.isArray(v)) {
+      for (const u of v) if (typeof u === "string") citedUrls.add(u);
+      state.found = true;
+    } else if (path.includes("content_references")) {
+      collectInto(contentReferences, v);
+      state.found = true;
+    } else if (path.includes("search_result_groups")) {
+      collectInto(searchResultGroups, v);
+      state.found = true;
+    } else if (path.includes("citations")) {
+      collectInto(citations, v);
+      state.found = true;
+    }
+  };
+  for (const line of raw.split("\n")) {
+    const s = (line.startsWith("data:") ? line.slice(5) : line).trim();
+    if (!s || s === "[DONE]" || s[0] !== "{" && s[0] !== "[") continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(s);
+    } catch {
+      continue;
+    }
+    const ops = Array.isArray(parsed) ? parsed : [parsed];
+    ops.forEach(handleOp);
+  }
+  if (!state.found) return raw;
+  const citedRefs = [...citedUrls].map((url) => ({ type: "grouped_webpages", items: [{ url }] }));
+  const synthetic = JSON.stringify({
+    content_references: [...contentReferences, ...citedRefs],
+    search_result_groups: searchResultGroups,
+    citations
+  });
+  return raw + "\n" + synthetic;
+}
+var chatgptAdapter = {
+  platform: "ChatGPT",
+  matchesHost(hostname) {
+    return /(^|\.)chatgpt\.com$|(^|\.)openai\.com$/i.test(hostname);
+  },
+  parse(rawInput, prompt) {
+    const raw = reconstructFromSsePatches(rawInput);
+    const sources = /* @__PURE__ */ new Map();
+    const snippets = /* @__PURE__ */ new Map();
+    const queries = /* @__PURE__ */ new Set();
+    const unmapped = /* @__PURE__ */ new Set();
+    const meta = { model: "", finishType: "", title: "", rounds: 0 };
+    const ensure = (url) => {
+      if (!isExternal(url)) return null;
+      const k = normUrl(url);
+      let s = sources.get(k);
+      if (!s) {
+        s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, firstOrder: Infinity, snippet: "", round: null };
+        sources.set(k, s);
+      }
+      return s;
+    };
+    const addSnippet = (url, title, text) => {
+      const clean = stripCitationMarkers(text);
+      if (!isExternal(url) || clean.length < 2) return;
+      snippets.set(normUrl(url), { url, domain: domainOf(url), title: title || domainOf(url), text: clean.slice(0, 400) });
+    };
+    for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
+    for (const m of raw.matchAll(/"search_model_queries"\s*:\s*\[([^\]]*)\]/g)) {
+      for (const q of stringsInArrayBody(m[1])) {
+        const c = cleanQuery(q);
+        if (c) queries.add(c);
+      }
+    }
+    for (const m of raw.matchAll(/"(?:search_queries|queries|search_query)"\s*:\s*\[([^\]]*)\]/g)) {
+      for (const q of stringsInArrayBody(m[1])) {
+        const c = cleanQuery(q);
+        if (c) queries.add(c);
+      }
+    }
+    for (const v of findJsonValues(raw, "search_result_groups")) {
+      if (!Array.isArray(v)) continue;
+      for (const g of v) {
+        const entries = g && typeof g === "object" && Array.isArray(g.entries) ? g.entries : [];
+        for (const e of entries) {
+          const url = typeof e.url === "string" ? e.url : "";
+          const s = ensure(url);
+          if (!s) continue;
+          if (!s.title && typeof e.title === "string") s.title = e.title.slice(0, 200);
+          const ref = normRef(e.ref_id);
+          if (ref && s.round == null) s.round = roundOf(ref);
+          if (typeof e.snippet === "string" && e.snippet) addSnippet(url, s.title, e.snippet);
+        }
+      }
+    }
+    for (const m of raw.matchAll(/"url"\s*:\s*"(https?:\/\/[^"\\]+)"(?:[\s\S]{0,120}?"(?:title|name)"\s*:\s*"((?:[^"\\]|\\.){0,200})")?/g)) {
+      const s = ensure(m[1]);
+      if (s && !s.title && m[2]) s.title = unesc(m[2]).slice(0, 200);
+    }
+    let spanTotal = 0;
+    for (const cr of findJsonValues(raw, "content_references")) {
+      if (!Array.isArray(cr)) continue;
+      for (const ref of cr) {
+        const type = typeof ref.type === "string" ? ref.type : "";
+        const isBundle = /sources_footnote|nav_list|alt_text/.test(type);
+        const refs = refsInText(typeof ref.matched_text === "string" ? ref.matched_text : "");
+        if (!isBundle) spanTotal++;
+        const ord = typeof ref.start_idx === "number" ? ref.start_idx : isBundle ? Infinity : spanTotal;
+        const items = Array.isArray(ref.items) ? ref.items : [];
+        items.forEach((it, idx) => {
+          const refForItem = refs.length === items.length ? refs[idx] : refs[0];
+          const urls = [it.url].concat((Array.isArray(it.supporting_websites) ? it.supporting_websites : []).map((s) => s?.url)).filter((u) => typeof u === "string");
+          for (const u of urls) {
+            const s = ensure(u);
+            if (!s) continue;
+            if (!isBundle) {
+              s.cited = true;
+              s.timesCited++;
+              if (ord < s.firstOrder) s.firstOrder = ord;
+            } else if (!s.cited) {
+              s.cited = true;
+            }
+            if (refForItem && s.round == null) s.round = roundOf(refForItem);
+          }
+          if (typeof it.url === "string" && typeof it.snippet === "string") addSnippet(it.url, typeof it.title === "string" ? it.title : "", it.snippet);
+        });
+      }
+    }
+    for (const v of findJsonValues(raw, "citations")) {
+      if (!Array.isArray(v)) continue;
+      v.forEach((c, i) => {
+        const md = c && typeof c === "object" ? c.metadata : null;
+        if (!md || typeof md.url !== "string") return;
+        const s = ensure(md.url);
+        if (!s) return;
+        s.cited = true;
+        s.timesCited++;
+        const ord = typeof c.start_ix === "number" ? c.start_ix : i;
+        if (ord < s.firstOrder) s.firstOrder = ord;
+        if (typeof md.text === "string") addSnippet(md.url, typeof md.title === "string" ? md.title : "", md.text);
+      });
+    }
+    for (const m of raw.matchAll(/"content_type"\s*:\s*"tether_quote"([\s\S]{0,4000}?)"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g)) {
+      const url = (m[1].match(/"url"\s*:\s*"([^"]+)"/) || [])[1];
+      const title = (m[1].match(/"title"\s*:\s*"((?:[^"\\]|\\.)*)"/) || [])[1];
+      if (url) {
+        ensure(url);
+        addSnippet(url, title ? unesc(title) : "", unesc(m[2]));
+      }
+    }
+    const modelMatches = [...raw.matchAll(/"resolved_model_slug"\s*:\s*"([^"]+)"/g)];
+    const slugMatches = modelMatches.length ? modelMatches : [...raw.matchAll(/"model_slug"\s*:\s*"([^"]+)"/g)];
+    if (slugMatches.length) meta.model = slugMatches[slugMatches.length - 1][1];
+    const finish = raw.match(/"finish_details"\s*:\s*\{[^}]*"type"\s*:\s*"([^"]+)"/);
+    if (finish) meta.finishType = finish[1];
+    const searchRounds = (raw.match(/"recipient"\s*:\s*"(?:web|browser)"[\s\S]{0,400}?"content_type"\s*:\s*"code"[\s\S]{0,200}?search\(/g) || []).length;
+    const sourceRounds = /* @__PURE__ */ new Set();
+    for (const s of sources.values()) if (s.round != null) sourceRounds.add(s.round);
+    meta.rounds = Math.max(searchRounds, sourceRounds.size);
+    const ordered = [...sources.values()].sort((a, b) => a.cited === b.cited ? b.timesCited - a.timesCited || a.firstOrder - b.firstOrder : a.cited ? -1 : 1).map((s) => ({
+      url: s.url,
+      domain: s.domain,
+      title: s.title || s.domain,
+      cited: s.cited,
+      timesCited: s.timesCited,
+      snippet: snippets.get(normUrl(s.url))?.text || "",
+      round: s.round
+    }));
+    return {
+      subQueries: [...queries],
+      sources: ordered,
+      snippets: [...snippets.values()],
+      meta,
+      unmappedKeys: [...unmapped].slice(0, 50)
+    };
+  }
+};
+
+// src/services/fanout/adapters/claude.ts
+function isFavicon(u) {
+  return /\/s2\/favicons/i.test(u) || /favicon/i.test(u);
+}
+var claudeAdapter = {
+  platform: "Claude",
+  matchesHost(hostname) {
+    return /(^|\.)claude\.ai$/i.test(hostname);
+  },
+  parse(raw, prompt) {
+    const sources = /* @__PURE__ */ new Map();
+    const snippets = /* @__PURE__ */ new Map();
+    const queries = /* @__PURE__ */ new Set();
+    const unmapped = /* @__PURE__ */ new Set();
+    const meta = { model: "", finishType: "", title: "", rounds: 0 };
+    const ensure = (url) => {
+      if (!url || !isExternal(url) || isFavicon(url)) return null;
+      const k = normUrl(url);
+      let s = sources.get(k);
+      if (!s) {
+        s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, snippet: "", round: null };
+        sources.set(k, s);
+      }
+      return s;
+    };
+    const toolInputs = /* @__PURE__ */ new Map();
+    const searchBlocks = /* @__PURE__ */ new Set();
+    const finalizeTool = (index) => {
+      const t = toolInputs.get(index);
+      if (!t || !/web_search|web_fetch|search|browse/i.test(t.name)) return;
+      try {
+        const input = JSON.parse(t.buf);
+        if (typeof input.query === "string") {
+          const q = cleanQuery(input.query);
+          if (q) queries.add(q);
+        }
+      } catch {
+        const m = t.buf.match(/"query"\s*:\s*"((?:[^"\\]|\\.)*)"/);
+        if (m) {
+          const q = cleanQuery(unesc(m[1]));
+          if (q) queries.add(q);
+        }
+      }
+    };
+    for (const line of raw.split("\n")) {
+      const s = (line.startsWith("data:") ? line.slice(5) : line).trim();
+      if (!s || s[0] !== "{" && s[0] !== "[") continue;
+      let ev;
+      try {
+        ev = JSON.parse(s);
+      } catch {
+        continue;
+      }
+      const type = ev.type;
+      if (type === "content_block_start") {
+        const cb = ev.content_block || {};
+        const index = typeof ev.index === "number" ? ev.index : -1;
+        if (cb.type === "tool_use" && index >= 0) {
+          const name = typeof cb.name === "string" ? cb.name : "";
+          toolInputs.set(index, { name, buf: "" });
+          if (/web_search|web_fetch|search|browse/i.test(name)) searchBlocks.add(index);
+        }
+      } else if (type === "content_block_delta") {
+        const d = ev.delta || {};
+        const index = typeof ev.index === "number" ? ev.index : -1;
+        if (d.type === "input_json_delta" && typeof d.partial_json === "string") {
+          const t = toolInputs.get(index);
+          if (t) t.buf += d.partial_json;
+        } else if (d.type === "citation_start_delta") {
+          const c = d.citation || {};
+          if (typeof c.url === "string") {
+            const src = ensure(c.url);
+            if (src) {
+              src.cited = true;
+              src.timesCited++;
+              if (!src.title && typeof c.title === "string") src.title = c.title.slice(0, 200);
+              if (typeof c.cited_text === "string" && c.cited_text) {
+                snippets.set(normUrl(c.url), { url: c.url, domain: domainOf(c.url), title: src.title, text: String(c.cited_text).slice(0, 400) });
+              }
+            }
+          }
+        }
+      } else if (type === "content_block_stop") {
+        const index = typeof ev.index === "number" ? ev.index : -1;
+        if (index >= 0) finalizeTool(index);
+      } else if (type === "message_delta" || type === "message_stop" || type === "message_start") {
+        const d = ev.delta || ev.message || {};
+        if (typeof d.stop_reason === "string" && !meta.finishType) meta.finishType = d.stop_reason;
+        if (typeof d.model === "string" && !meta.model) meta.model = d.model;
+      }
+    }
+    for (const index of searchBlocks) finalizeTool(index);
+    for (const m of raw.matchAll(/"title"\s*:\s*"((?:[^"\\]|\\.){1,200})"\s*,\s*"url"\s*:\s*"(https?:\/\/[^"\\]+)"/g)) {
+      const src = ensure(m[2]);
+      if (src && !src.title) src.title = unesc(m[1]).slice(0, 200);
+    }
+    for (const m of raw.matchAll(/"url"\s*:\s*"(https?:\/\/[^"\\]+)"/g)) ensure(m[1]);
+    for (const m of raw.matchAll(/\\"title\\"\s*:\s*\\"((?:[^"\\]|\\.){1,160}?)\\"[\s\S]{0,160}?\\"url\\"\s*:\s*\\"(https?:[^"\\]+)\\"/g)) {
+      const src = ensure(m[2]);
+      if (src && !src.title) src.title = unesc(unesc(m[1])).slice(0, 200);
+    }
+    for (const m of raw.matchAll(/\\"url\\"\s*:\s*\\"(https?:[^"\\]+)\\"/g)) ensure(m[1]);
+    if (!meta.model) {
+      const mm = raw.match(/"model"\s*:\s*"(claude[^"]+)"/);
+      if (mm) meta.model = mm[1];
+    }
+    meta.rounds = queries.size > 0 ? searchBlocks.size : 0;
+    for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
+    const ordered = [...sources.values()].sort((a, b) => a.cited === b.cited ? b.timesCited - a.timesCited : a.cited ? -1 : 1).map((s) => ({
+      url: s.url,
+      domain: s.domain,
+      title: s.title || s.domain,
+      cited: s.cited,
+      timesCited: s.timesCited,
+      snippet: snippets.get(normUrl(s.url))?.text || "",
+      round: s.round
+    }));
+    return {
+      subQueries: [...queries],
+      sources: ordered,
+      snippets: [...snippets.values()],
+      meta,
+      unmappedKeys: [...unmapped].slice(0, 50)
+    };
+  }
+};
+
+// src/services/fanout/adapters/index.ts
+var FANOUT_ADAPTERS = [chatgptAdapter, claudeAdapter];
+function adapterForHost(hostname) {
+  return FANOUT_ADAPTERS.find((a) => a.matchesHost(hostname)) ?? null;
+}
+
+// src/services/fanout/build.ts
+function buildFanoutResult(ctx) {
+  const { platform, parsed, ready, rawBytes } = ctx;
+  const browsedUrls = parsed.sources;
+  const citedUrls = browsedUrls.filter((s) => s.cited);
+  const browsedOnly = browsedUrls.filter((s) => !s.cited);
+  const prompt = ctx.prompt || parsed.meta.title || "";
+  const result = {
+    platform,
+    capturedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    prompt,
+    meta: parsed.meta,
+    subQueries: parsed.subQueries,
+    browsedUrls,
+    citedUrls,
+    browsedOnly,
+    snippets: parsed.snippets,
+    counts: {
+      subQueries: parsed.subQueries.length,
+      browsed: browsedUrls.length,
+      cited: citedUrls.length,
+      browsedOnly: browsedOnly.length
+    }
+  };
+  if (parsed.subQueries.length === 0 && browsedUrls.length === 0) {
+    result.debug = {
+      interceptorReady: ready,
+      rawBytes,
+      unmappedKeys: parsed.unmappedKeys,
+      note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call browser_capture_fanout again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page. The session must be opened with MCP_SCRAPER_BROWSER_MODE=local and navigated to chatgpt.com or claude.ai after opening, so the hook installs before the site scripts run."
+    };
+  }
+  return result;
+}
+
+// src/services/fanout/classify.ts
+var SOCIAL = /(^|\.)(youtube\.com|youtu\.be|instagram\.com|tiktok\.com|facebook\.com|fb\.com|x\.com|twitter\.com|linkedin\.com|pinterest\.com|threads\.net|vimeo\.com)$/i;
+var ENCYCLOPEDIA = /(^|\.)(wikipedia\.org|wikimedia\.org|britannica\.com|merriam-webster\.com|dictionary\.com|fandom\.com|investopedia\.com)$/i;
+var REVIEW = /(^|\.)(yelp\.com|trustpilot\.com|g2\.com|capterra\.com|angi\.com|angieslist\.com|bbb\.org|consumeraffairs\.com|bestcompany\.com|sitejabber\.com|clutch\.co|bestpickreports\.com|homeguide\.com|porch\.com|ontoplist\.com|thumbtack\.com)$/i;
+var NEWS = /(^|\.)(reuters\.com|apnews\.com|nytimes\.com|wsj\.com|washingtonpost\.com|bloomberg\.com|cnbc\.com|forbes\.com|bbc\.com|theguardian\.com|cnn\.com|npr\.org|usatoday\.com|expressnews\.com|mysanantonio\.com|beaumontenterprise\.com)$/i;
+function classifyDomain(domain, firstPartyDomain) {
+  const d = domain.toLowerCase().replace(/^www\./, "");
+  if (firstPartyDomain) {
+    const fp = firstPartyDomain.toLowerCase().replace(/^www\./, "");
+    if (fp && (d === fp || d.endsWith("." + fp))) return "First-party/vendor";
+  }
+  if (/(^|\.)reddit\.com$/i.test(d)) return "Reddit";
+  if (SOCIAL.test(d)) return "Social/video";
+  if (ENCYCLOPEDIA.test(d)) return "Encyclopedia";
+  if (REVIEW.test(d)) return "Review site";
+  if (NEWS.test(d) || /(^|\.)news\./i.test(d)) return "News/media";
+  if (/^docs?\.|^developer\.|^developers\.|readthedocs|\.dev$/i.test(d)) return "Docs";
+  return "Blog";
+}
+function emptyCategoryCounts() {
+  return { "First-party/vendor": 0, "News/media": 0, Reddit: 0, "Social/video": 0, Encyclopedia: 0, "Review site": 0, Docs: 0, Blog: 0 };
+}
+function enrichFanout(result, opts = {}) {
+  const addSiteType = (s) => ({ ...s, siteType: classifyDomain(s.domain, opts.firstPartyDomain) });
+  const browsedUrls = result.browsedUrls.map(addSiteType);
+  const citedUrls = result.citedUrls.map(addSiteType);
+  const browsedOnly = result.browsedOnly.map(addSiteType);
+  const byCategory = emptyCategoryCounts();
+  const domainAgg = /* @__PURE__ */ new Map();
+  for (const s of browsedUrls) {
+    byCategory[s.siteType]++;
+    const cur = domainAgg.get(s.domain);
+    if (!cur) domainAgg.set(s.domain, { domain: s.domain, count: 1, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType });
+    else {
+      cur.count++;
+      cur.cited = cur.cited || s.cited;
+      cur.timesCited += s.timesCited;
+    }
+  }
+  const topSites = [...domainAgg.values()].sort((a, b) => b.count - a.count || b.timesCited - a.timesCited);
+  const citationOrder = citedUrls.slice().sort((a, b) => b.timesCited - a.timesCited).map((s, i) => ({ rank: i + 1, domain: s.domain, url: s.url, timesCited: s.timesCited }));
+  return {
+    platform: result.platform,
+    capturedAt: result.capturedAt,
+    prompt: result.prompt,
+    meta: result.meta,
+    queries: result.subQueries,
+    browsedUrls,
+    citedUrls,
+    browsedOnly,
+    snippets: result.snippets,
+    counts: result.counts,
+    aggregates: { topSites, citationOrder, byCategory },
+    firstPartyDomain: opts.firstPartyDomain ? opts.firstPartyDomain.toLowerCase().replace(/^www\./, "") : null,
+    ...result.debug ? { debug: result.debug } : {}
+  };
+}
+
+// src/services/fanout/export.ts
+import { mkdirSync, writeFileSync } from "fs";
+import { homedir } from "os";
+import { join } from "path";
+import Papa from "papaparse";
+function outputBaseDir() {
+  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join(homedir(), "Downloads", "mcp-scraper");
+}
+function safe(value) {
+  return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
+}
+function writeTable(path, rows, delimiter) {
+  writeFileSync(path, Papa.unparse(rows.length ? rows : [{}], { delimiter }));
+}
+function exportFanout(enriched) {
+  const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
+  const dir = join(outputBaseDir(), "fanout", `${stamp}-${safe(enriched.platform)}`);
+  mkdirSync(dir, { recursive: true });
+  const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
+  const citationRows = enriched.aggregates.citationOrder.map((c) => {
+    const src = enriched.citedUrls.find((s) => s.url === c.url);
+    return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
+  });
+  const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
+  const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
+  const paths = {
+    dir,
+    json: join(dir, "fanout.json"),
+    queriesCsv: join(dir, "queries.csv"),
+    queriesTsv: join(dir, "queries.tsv"),
+    citationsCsv: join(dir, "citations.csv"),
+    sourcesCsv: join(dir, "sources.csv"),
+    domainsCsv: join(dir, "domains.csv"),
+    report: join(dir, "report.html")
+  };
+  writeFileSync(paths.json, JSON.stringify(enriched, null, 2));
+  writeTable(paths.queriesCsv, queryRows, ",");
+  writeTable(paths.queriesTsv, queryRows, "	");
+  writeTable(paths.citationsCsv, citationRows, ",");
+  writeTable(paths.sourcesCsv, sourceRows, ",");
+  writeTable(paths.domainsCsv, domainRows, ",");
+  writeFileSync(paths.report, renderReportHtml(enriched));
+  return paths;
+}
+function esc(s) {
+  return String(s).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+function renderReportHtml(enriched) {
+  const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
+  return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
+<style>
+:root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
+*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
+.wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
+h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
+.stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
+.cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
+.card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
+.card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
+.bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
+.bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
+.bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
+.tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
+.tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
+.chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
+.chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
+.chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
+.row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
+.row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
+.tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
+.dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
+.pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
+.copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
+.hide{display:none}
+</style>
+</head>
+<body><div class="wrap">
+<h1>AI Search Fan-Out</h1>
+<p class="sub" id="sub"></p>
+<div class="stat" id="stat"></div>
+<div class="cards" id="cards"></div>
+<div id="charts"></div>
+<div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
+<div class="chips" id="chips"></div>
+<div id="list"></div>
+</div>
+<script>
+const D=${data};
+const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
+document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
+document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
+const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
+const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
+function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
+const charts=document.getElementById('charts');
+charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
+let tab='q',filter=null;
+const chips=document.getElementById('chips'),list=document.getElementById('list');
+function render(){
+  chips.innerHTML='';list.innerHTML='';
+  if(tab==='q'){
+    D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
+  } else {
+    const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
+    Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
+    D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
+  }
+}
+function esc(s){return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;')}
+document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
+render();
+</script>
+</body></html>`;
+}
+
+// src/services/fanout/run-capture.ts
+var COMPOSER_SELECTORS = [
+  "#prompt-textarea",
+  "div#prompt-textarea",
+  'textarea[data-testid="prompt-textarea"]',
+  'div.ProseMirror[contenteditable="true"]',
+  'div[contenteditable="true"]',
+  "textarea"
+];
+async function sendPrompt(page, text) {
+  for (const selector of COMPOSER_SELECTORS) {
+    const locator = page.locator(selector).first();
+    try {
+      await locator.waitFor({ state: "visible", timeout: 12e3 });
+    } catch {
+      continue;
+    }
+    await locator.click({ timeout: 4e3 }).catch(() => void 0);
+    await locator.focus().catch(() => void 0);
+    await page.keyboard.type(text, { delay: 12 });
+    await page.waitForTimeout(300).catch(() => void 0);
+    await page.keyboard.press("Enter");
+    return true;
+  }
+  return false;
+}
+async function waitForAnswer(page, cap, baseline, waitMs) {
+  const start = Date.now();
+  while (Date.now() - start < waitMs && cap.answerStreamCount() <= baseline) {
+    await page.waitForTimeout(400).catch(() => void 0);
+  }
+  await page.waitForTimeout(1e3).catch(() => void 0);
+}
+async function runFanoutCapture(page, input) {
+  const cap = new FanoutCdpCapture();
+  await cap.attach(page);
+  try {
+    const baseline = cap.answerStreamCount();
+    if (input.prompt) await sendPrompt(page, input.prompt).catch(() => false);
+    const waitMs = input.wait_ms ?? (input.prompt ? 9e4 : 8e3);
+    await waitForAnswer(page, cap, baseline, waitMs);
+    await cap.drain();
+    let hostname = "";
+    try {
+      hostname = new URL(page.url()).hostname;
+    } catch {
+      hostname = "";
+    }
+    const adapter = adapterForHost(hostname);
+    if (!adapter) {
+      throw new Error(`browser_capture_fanout supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
+    }
+    const text = cap.bestAnswerText();
+    const parsed = adapter.parse(text, input.prompt || "");
+    const result = buildFanoutResult({ platform: adapter.platform, parsed, prompt: input.prompt || "", ready: true, rawBytes: text.length });
+    const enriched = enrichFanout(result, { firstPartyDomain: input.first_party_domain });
+    const exports = input.export ? exportFanout(enriched) : null;
+    return { result: enriched, exports };
+  } finally {
+    await cap.detach().catch(() => void 0);
+  }
+}
+
+export {
+  runFanoutCapture
+};
+//# sourceMappingURL=chunk-NXRWFOEZ.js.map