mcp-scraper 0.3.9 → 0.3.11

package/dist/chunk-RRE7WVHQ.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.3.11";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-RRE7WVHQ.js.map

package/dist/chunk-RRE7WVHQ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.3.11'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{chunk-KDKQ5Y7V.js → chunk-VMH7SRWY.js}

@@ -7,7 +7,7 @@ import {
 } from "./chunk-LFATOGDF.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-7VNTN4Q7.js";
+} from "./chunk-RRE7WVHQ.js";
 
 // src/harvest-timeout.ts
 var VERCEL_FUNCTION_MAX_MS = 3e5;
@@ -3335,4 +3335,4 @@ export {
   registerPaaExtractorMcpTools,
   HttpMcpToolExecutor
 };
-//# sourceMappingURL=chunk-KDKQ5Y7V.js.map
+//# sourceMappingURL=chunk-VMH7SRWY.js.map

package/dist/{server-ZSHE4R5Z.js → server-3SA5Q4OF.js}

@@ -99,7 +99,7 @@ import {
   harvestTimeoutBudget,
   liveWebToolAnnotations,
   outputBaseDir
-} from "./chunk-KDKQ5Y7V.js";
+} from "./chunk-VMH7SRWY.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -132,7 +132,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-LFATOGDF.js";
-import "./chunk-7VNTN4Q7.js";
+import "./chunk-RRE7WVHQ.js";
 
 // src/api/outbound-sanitize.ts
 var KEY_RENAMES = {
@@ -14114,7 +14114,9 @@ function writeTable(path5, rows, delimiter) {
 }
 function exportFanout(enriched) {
   const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
-  const dir = join6(outputBaseDir3(), "fanout", `${stamp}-${safe(enriched.platform)}`);
+  const outputDir = outputBaseDir3();
+  const relativeDir = join6("fanout", `${stamp}-${safe(enriched.platform)}`);
+  const dir = join6(outputDir, relativeDir);
   mkdirSync3(dir, { recursive: true });
   const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
   const citationRows = enriched.aggregates.citationOrder.map((c) => {
@@ -14122,25 +14124,32 @@ function exportFanout(enriched) {
     return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
   });
   const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
+  const browsedOnlyRows = enriched.browsedOnly.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
+  const snippetRows = enriched.snippets.map((s) => ({ domain: s.domain, url: s.url, title: s.title, text: s.text }));
   const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
-  const paths = {
-    dir,
-    json: join6(dir, "fanout.json"),
-    queriesCsv: join6(dir, "queries.csv"),
-    queriesTsv: join6(dir, "queries.tsv"),
-    citationsCsv: join6(dir, "citations.csv"),
-    sourcesCsv: join6(dir, "sources.csv"),
-    domainsCsv: join6(dir, "domains.csv"),
-    report: join6(dir, "report.html")
+  const relativePaths = {
+    relativeTo: "MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper",
+    dir: relativeDir,
+    json: join6(relativeDir, "fanout.json"),
+    queriesCsv: join6(relativeDir, "queries.csv"),
+    queriesTsv: join6(relativeDir, "queries.tsv"),
+    citationsCsv: join6(relativeDir, "citations.csv"),
+    sourcesCsv: join6(relativeDir, "sources.csv"),
+    browsedOnlyCsv: join6(relativeDir, "browsed-only.csv"),
+    snippetsCsv: join6(relativeDir, "snippets.csv"),
+    domainsCsv: join6(relativeDir, "domains.csv"),
+    report: join6(relativeDir, "report.html")
   };
-  writeFileSync2(paths.json, JSON.stringify(enriched, null, 2));
-  writeTable(paths.queriesCsv, queryRows, ",");
-  writeTable(paths.queriesTsv, queryRows, "	");
-  writeTable(paths.citationsCsv, citationRows, ",");
-  writeTable(paths.sourcesCsv, sourceRows, ",");
-  writeTable(paths.domainsCsv, domainRows, ",");
-  writeFileSync2(paths.report, renderReportHtml(enriched));
-  return paths;
+  writeFileSync2(join6(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2));
+  writeTable(join6(outputDir, relativePaths.queriesCsv), queryRows, ",");
+  writeTable(join6(outputDir, relativePaths.queriesTsv), queryRows, "	");
+  writeTable(join6(outputDir, relativePaths.citationsCsv), citationRows, ",");
+  writeTable(join6(outputDir, relativePaths.sourcesCsv), sourceRows, ",");
+  writeTable(join6(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ",");
+  writeTable(join6(outputDir, relativePaths.snippetsCsv), snippetRows, ",");
+  writeTable(join6(outputDir, relativePaths.domainsCsv), domainRows, ",");
+  writeFileSync2(join6(outputDir, relativePaths.report), renderReportHtml(enriched));
+  return relativePaths;
 }
 function esc(s) {
   return String(s).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
@@ -14255,6 +14264,7 @@ async function runFanoutCapture(page, input) {
   const cap = new FanoutCdpCapture();
   await cap.attach(page);
   try {
+    if (input.reset) cap.reset();
     const baseline = cap.answerStreamCount();
     if (input.prompt) await sendPrompt(page, input.prompt).catch(() => false);
     const waitMs = input.wait_ms ?? (input.prompt ? 9e4 : 8e3);
@@ -14301,10 +14311,68 @@ async function ensureProfile(k, name) {
     if (!isProfileConflict(err)) throw err;
   }
 }
+function profileConnectionMatches(connection, profileName, domain) {
+  if (!connection || typeof connection !== "object") return false;
+  const data = connection;
+  return data.profile_name === profileName && data.domain === domain;
+}
+async function findProfileConnection(k, profileName, domain) {
+  for await (const connection of k.auth.connections.list({ profile_name: profileName, domain })) {
+    if (profileConnectionMatches(connection, profileName, domain)) return connection;
+  }
+  return null;
+}
+async function findOrCreateProfileConnection(k, opts) {
+  try {
+    return await k.auth.connections.create({
+      domain: opts.domain,
+      profile_name: opts.profileName,
+      ...opts.loginUrl ? { login_url: opts.loginUrl } : {}
+    });
+  } catch (err) {
+    if (!isProfileConflict(err)) throw err;
+    const existing = await findProfileConnection(k, opts.profileName, opts.domain);
+    if (existing) return existing;
+    throw err;
+  }
+}
+function profileOnboardResult(connection, hostedUrl = null, liveViewUrl = null) {
+  return {
+    connectionId: String(connection.id),
+    profileName: String(connection.profile_name),
+    domain: String(connection.domain),
+    status: String(connection.status),
+    flowStatus: typeof connection.flow_status === "string" ? connection.flow_status : null,
+    flowStep: typeof connection.flow_step === "string" ? connection.flow_step : null,
+    flowExpiresAt: typeof connection.flow_expires_at === "string" ? connection.flow_expires_at : null,
+    hostedUrl: hostedUrl ?? (typeof connection.hosted_url === "string" ? connection.hosted_url : null),
+    liveViewUrl: liveViewUrl ?? (typeof connection.live_view_url === "string" ? connection.live_view_url : null),
+    browserSessionId: typeof connection.browser_session_id === "string" ? connection.browser_session_id : null,
+    postLoginUrl: typeof connection.post_login_url === "string" ? connection.post_login_url : null
+  };
+}
 function explicitProxyId(value) {
   const trimmed = value?.trim();
   return trimmed || void 0;
 }
+async function startProfileOnboarding(opts) {
+  const k = client();
+  const connection = await findOrCreateProfileConnection(k, opts);
+  const login = await k.auth.connections.login(connection.id);
+  const state = await k.auth.connections.retrieve(connection.id);
+  return profileOnboardResult(state, login.hosted_url ?? null, login.live_view_url ?? null);
+}
+async function getProfileOnboardingStatus(input) {
+  const k = client();
+  let connection = null;
+  if (input.connectionId) {
+    connection = await k.auth.connections.retrieve(input.connectionId);
+  } else if (input.profileName && input.domain) {
+    connection = await findProfileConnection(k, input.profileName, input.domain);
+  }
+  if (!connection) throw new Error("profile auth connection not found");
+  return profileOnboardResult(connection);
+}
 async function createSession(opts = {}) {
   const k = client();
   const resolvedProxyId = explicitProxyId(opts.proxyId);
@@ -14698,6 +14766,56 @@ function buildBrowserAgentRoutes() {
     return next();
   });
   app2.use("*", auth);
+  app2.post("/profiles/onboard", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const profileName = typeof body.profile === "string" ? body.profile.trim() : "";
+    const domain = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : "chatgpt.com";
+    const loginUrl = typeof body.login_url === "string" ? body.login_url.trim() : void 0;
+    if (!profileName) return c.json({ error: "profile is required" }, 400);
+    if (!domain) return c.json({ error: "domain is required" }, 400);
+    try {
+      const result = await startProfileOnboarding({ profileName, domain, loginUrl });
+      return c.json({
+        connection_id: result.connectionId,
+        profile: result.profileName,
+        domain: result.domain,
+        status: result.status,
+        flow_status: result.flowStatus,
+        flow_step: result.flowStep,
+        flow_expires_at: result.flowExpiresAt,
+        hosted_url: result.hostedUrl,
+        live_view_url: result.liveViewUrl,
+        browser_session_id: result.browserSessionId,
+        post_login_url: result.postLoginUrl
+      });
+    } catch (err) {
+      return c.json(failure(err), 502);
+    }
+  });
+  app2.post("/profiles/status", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const connectionId = typeof body.connection_id === "string" ? body.connection_id.trim() : void 0;
+    const profileName = typeof body.profile === "string" ? body.profile.trim() : void 0;
+    const domain = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : void 0;
+    try {
+      const result = await getProfileOnboardingStatus({ connectionId, profileName, domain });
+      return c.json({
+        connection_id: result.connectionId,
+        profile: result.profileName,
+        domain: result.domain,
+        status: result.status,
+        flow_status: result.flowStatus,
+        flow_step: result.flowStep,
+        flow_expires_at: result.flowExpiresAt,
+        hosted_url: result.hostedUrl,
+        live_view_url: result.liveViewUrl,
+        browser_session_id: result.browserSessionId,
+        post_login_url: result.postLoginUrl
+      });
+    } catch (err) {
+      return c.json(failure(err), 404);
+    }
+  });
   app2.post("/sessions", async (c) => {
     const user = c.get("user");
     if (Number(user.balance_mc ?? 0) < BROWSER_OPEN_MIN_BALANCE_MC) {
@@ -14801,6 +14919,7 @@ function buildBrowserAgentRoutes() {
       prompt: typeof body.prompt === "string" ? body.prompt : void 0,
       wait_ms: typeof body.wait_ms === "number" ? body.wait_ms : void 0,
       first_party_domain: typeof body.first_party_domain === "string" ? body.first_party_domain : void 0,
+      reset: body.reset === true,
       export: body.export === true
     };
     const t0 = Date.now();
@@ -16452,4 +16571,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-ZSHE4R5Z.js.map
+//# sourceMappingURL=server-3SA5Q4OF.js.map