mcp-scraper 0.3.17 → 0.3.19

package/dist/{server-TKWY47JA.js → server-EYNMXWUJ.js}

@@ -1,3 +1,39 @@
+import {
+  CaptureSerpPageSnapshotsInputSchema,
+  CaptureSerpPageSnapshotsOutputSchema,
+  CaptureSerpSnapshotInputSchema,
+  CaptureSerpSnapshotOutputSchema,
+  HttpMcpToolExecutor,
+  auditImages,
+  buildLinkGraph,
+  buildLinkReport,
+  buildPaaExtractorMcpServer,
+  computeIssues,
+  configureReportSaving,
+  exportFanout,
+  formatCaptureSerpPageSnapshots,
+  formatCaptureSerpSnapshot,
+  harvestTimeoutBudget,
+  liveWebToolAnnotations,
+  outputBaseDir,
+  renderImageSection,
+  renderIssueReport,
+  renderLinkReport,
+  sanitizeAttempts,
+  sanitizeHarvestResult
+} from "./chunk-A46NKAAM.js";
+import {
+  csvRecords,
+  listWorkflowDefinitions,
+  rowsToCsv,
+  runWorkflow,
+  runWorkflowStep,
+  workflowDefinition,
+  workflowOutputBaseDir,
+  workflowStepCount,
+  workflowSupportsSteps
+} from "./chunk-H2R232HK.js";
+import "./chunk-AUKOY2IV.js";
 import {
   BALANCE_PACK_LABELS,
   BALANCE_PRICE_IDS,
@@ -16,7 +52,34 @@ import {
   harvestProblemResponse,
   insufficientBalanceResponse,
   serializeHarvestProblem
-} from "./chunk-ZAUMSBV3.js";
+} from "./chunk-T337IGVE.js";
+import {
+  BrowserDriver,
+  MapsSelectors,
+  buildYouTubeChannelVideosUrl,
+  deleteKernelProxyId,
+  harvest,
+  resolveKernelProxyId
+} from "./chunk-IGOEAKC7.js";
+import {
+  CaptchaError,
+  RECAPTCHA_INSTRUCTIONS,
+  browserServiceApiKey,
+  browserServiceProfileName,
+  browserServiceProfileSaveChanges,
+  browserServiceProxyId,
+  sanitizeVendorName
+} from "./chunk-ZAP7MYVO.js";
+import {
+  DEFAULT_MAPS_PROXY_MODE,
+  DEFAULT_PROXY_MODE,
+  MapsPlaceOptionsSchema,
+  MapsSearchOptionsSchema,
+  RawMapsAboutAttributeSchema,
+  RawMapsHoursRowSchema,
+  RawMapsOverviewSchema,
+  RawMapsReviewStatsSchema
+} from "./chunk-LFATOGDF.js";
 import {
   SiteAuditJobRowSchema,
   advanceWorkflowRunStep,
@@ -78,118 +141,6 @@ import {
   setStripeCustomerId,
   verifyPassword
 } from "./chunk-5H22TOXQ.js";
-import {
-  BrowserDriver,
-  MapsSelectors,
-  buildYouTubeChannelVideosUrl,
-  deleteKernelProxyId,
-  harvest,
-  resolveKernelProxyId
-} from "./chunk-4OPKIDON.js";
-import {
-  CaptureSerpPageSnapshotsInputSchema,
-  CaptureSerpPageSnapshotsOutputSchema,
-  CaptureSerpSnapshotInputSchema,
-  CaptureSerpSnapshotOutputSchema,
-  HttpMcpToolExecutor,
-  auditImages,
-  buildLinkGraph,
-  buildLinkReport,
-  buildPaaExtractorMcpServer,
-  computeIssues,
-  configureReportSaving,
-  formatCaptureSerpPageSnapshots,
-  formatCaptureSerpSnapshot,
-  harvestTimeoutBudget,
-  liveWebToolAnnotations,
-  outputBaseDir,
-  renderImageSection,
-  renderIssueReport,
-  renderLinkReport
-} from "./chunk-N33YEY6W.js";
-import {
-  CaptchaError,
-  RECAPTCHA_INSTRUCTIONS,
-  sanitizeVendorName
-} from "./chunk-M2S27J6Z.js";
-import {
-  exportFanout
-} from "./chunk-3YGKXXUG.js";
-import {
-  browserServiceApiKey,
-  browserServiceProfileName,
-  browserServiceProfileSaveChanges,
-  browserServiceProxyId
-} from "./chunk-WN7PBKMV.js";
-import {
-  csvRecords,
-  listWorkflowDefinitions,
-  rowsToCsv,
-  runWorkflow,
-  runWorkflowStep,
-  workflowDefinition,
-  workflowOutputBaseDir,
-  workflowStepCount,
-  workflowSupportsSteps
-} from "./chunk-H2R232HK.js";
-import {
-  DEFAULT_MAPS_PROXY_MODE,
-  DEFAULT_PROXY_MODE,
-  MapsPlaceOptionsSchema,
-  MapsSearchOptionsSchema,
-  RawMapsAboutAttributeSchema,
-  RawMapsHoursRowSchema,
-  RawMapsOverviewSchema,
-  RawMapsReviewStatsSchema
-} from "./chunk-LFATOGDF.js";
-import "./chunk-PVWWY5NV.js";
-
-// src/api/outbound-sanitize.ts
-var KEY_RENAMES = {
-  kernel: "browserRuntime",
-  kernel_session_id: "browser_session_id",
-  kernel_delete_started: "session_cleanup_started",
-  kernel_delete_succeeded: "session_cleanup_succeeded",
-  kernel_delete_error: "session_cleanup_error",
-  kernelSessionId: "browserSessionId",
-  kernelDeleteStarted: "sessionCleanupStarted",
-  kernelDeleteSucceeded: "sessionCleanupSucceeded",
-  kernelDeleteError: "sessionCleanupError",
-  kernelProxyId: "proxyId"
-};
-var SANITIZED_VALUE_KEYS = /error|message/i;
-function sanitizeOutboundDiagnostics(value, parentKey = "") {
-  if (typeof value === "string") {
-    if (SANITIZED_VALUE_KEYS.test(parentKey) && /kernel/i.test(value)) {
-      return sanitizeVendorName(value);
-    }
-    return value;
-  }
-  if (Array.isArray(value)) return value.map((v) => sanitizeOutboundDiagnostics(v, parentKey));
-  if (value !== null && typeof value === "object") {
-    const out = {};
-    for (const [key, val] of Object.entries(value)) {
-      const renamed = KEY_RENAMES[key] ?? key;
-      out[renamed] = sanitizeOutboundDiagnostics(val, key);
-    }
-    return out;
-  }
-  return value;
-}
-function sanitizeAttempts(attempts) {
-  return attempts.map((a) => sanitizeOutboundDiagnostics(a));
-}
-function sanitizeHarvestResult(result) {
-  const diagnostics = result?.diagnostics;
-  if (!diagnostics?.debug) return result;
-  return {
-    ...result,
-    diagnostics: {
-      ...diagnostics,
-      debug: sanitizeOutboundDiagnostics(diagnostics.debug)
-    }
-  };
-}
 
 // src/blog/registry.ts
 var posts = [
@@ -7586,11 +7537,6 @@ async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference)
 
 // src/inngest/functions/site-extract.ts
 var BATCH = 30;
-function chunk(items, size) {
-  const out = [];
-  for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
-  return out;
-}
 function buildArtifacts(job, pages, branding, imageAudit) {
   const { edges, metrics } = buildLinkGraph(pages, job.startUrl);
   const issues = computeIssues(pages, metrics);
@@ -7649,18 +7595,62 @@ var siteExtractFn = inngest.createFunction(
     const maxPages = Number(job.options.maxPages ?? 1e4);
     const concurrency = Number(job.options.concurrency ?? 1);
     const urlsPerBrowser = Number(job.options.urlsPerBrowser ?? job.options.rotateProxyEvery ?? 10);
-    const urls = await step.run("discover", async () => {
+    const seed = await step.run("discover", async () => {
       const discovered = await discoverUrls(job.startUrl, maxPages, key);
-      await setExtractJobTotal(jobId, discovered.length);
-      return discovered;
+      const out = discovered.length ? discovered : [job.startUrl];
+      await setExtractJobTotal(jobId, out.length);
+      return out;
     });
-    const batches = chunk(urls, BATCH);
-    for (let i = 0; i < batches.length; i++) {
-      await step.run(`crawl-batch-${i}`, async () => {
-        const pages = await extractPagesRotating(batches[i], { kernelApiKey: key, concurrency, urlsPerBrowser });
+    const siteHost = (() => {
+      try {
+        return new URL(job.startUrl).hostname.replace(/^www\./, "");
+      } catch {
+        return "";
+      }
+    })();
+    const norm = (u) => {
+      try {
+        const x = new URL(u);
+        x.hash = "";
+        return (x.origin + x.pathname).replace(/\/+$/, "") + x.search;
+      } catch {
+        return u;
+      }
+    };
+    const sameSite = (u) => {
+      try {
+        return new URL(u).hostname.replace(/^www\./, "") === siteHost;
+      } catch {
+        return false;
+      }
+    };
+    const queue = [...seed];
+    const seen = new Set(seed.map(norm));
+    let crawled = 0;
+    const MAX_BATCHES = Math.ceil(maxPages / BATCH) + 30;
+    for (let i = 0; i < MAX_BATCHES && crawled < maxPages && crawled < queue.length; i++) {
+      const batch = queue.slice(crawled, Math.min(crawled + BATCH, maxPages));
+      if (!batch.length) break;
+      const newLinks = await step.run(`crawl-batch-${i}`, async () => {
+        const pages = await extractPagesRotating(batch, { kernelApiKey: key, concurrency, urlsPerBrowser });
         await saveExtractPages(jobId, pages);
-        return pages.length;
+        const found = [];
+        for (const p of pages) for (const l of p.outlinks ?? []) {
+          if (l.internal && l.href && sameSite(l.href)) found.push(l.href);
+        }
+        return [...new Set(found)];
       });
+      crawled += batch.length;
+      let added = false;
+      for (const href of newLinks) {
+        const n = norm(href);
+        if (!seen.has(n) && seen.size < maxPages) {
+          seen.add(n);
+          queue.push(href);
+          added = true;
+        }
+      }
+      if (added) await setExtractJobTotal(jobId, Math.min(seen.size, maxPages));
     }
     const branding = Array.isArray(job.options.formats) && job.options.formats.includes("branding") ? await step.run("branding", () => extractBranding(job.startUrl, key).catch(() => null)) : null;
     const imageAudit = Array.isArray(job.options.formats) && job.options.formats.includes("images") ? await step.run("image-audit", async () => {
@@ -9343,10 +9333,10 @@ function buildTranscriptMarkdown(result) {
   if (result.chunks?.length) {
     lines.push("## Timestamped Segments");
     lines.push("");
-    for (const chunk2 of result.chunks) {
-      const start = fmtTs(chunk2.timestamp[0]);
-      const end = fmtTs(chunk2.timestamp[1]);
-      lines.push(`**[${start} \u2192 ${end}]** ${chunk2.text.trim()}`);
+    for (const chunk of result.chunks) {
+      const start = fmtTs(chunk.timestamp[0]);
+      const end = fmtTs(chunk.timestamp[1]);
+      lines.push(`**[${start} \u2192 ${end}]** ${chunk.text.trim()}`);
       lines.push("");
     }
   }
@@ -9420,7 +9410,11 @@ screenshotApp.post("/", async (c) => {
     metadata: { url: targetUrl, device }
   });
   if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
+  let debited = false;
   try {
+    const { ok, balance_mc } = await debitMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, targetUrl);
+    if (!ok) return c.json(insufficientBalanceResponse(balance_mc, MC_COSTS.page_scrape), 402);
+    debited = true;
     const buf = await captureScreenshot(targetUrl, browserServiceApiKey(), device);
     return new Response(new Uint8Array(buf), {
       status: 200,
@@ -9431,6 +9425,7 @@ screenshotApp.post("/", async (c) => {
       }
     });
   } catch (err) {
+    if (debited) await creditMc(user.id, MC_COSTS.page_scrape, LedgerOperation.EXTRACT_URL, "failed screenshot refund");
     const msg = err instanceof Error ? err.message : String(err);
     return c.json({ error: msg }, 502);
   } finally {
@@ -9847,20 +9842,20 @@ var FacebookAdExtractor = class {
       if (splitRe.lastIndex === m.index) splitRe.lastIndex++;
     }
     if (last < bodyText.length) adChunks.push(bodyText.slice(last));
-    const ads = adChunks.filter((c) => /Library ID[:\s]+\d{10,20}/i.test(c)).slice(0, maxAds).map((chunk2) => {
-      const lidM = chunk2.match(/Library ID[:\s]+([0-9]{10,20})/i);
+    const ads = adChunks.filter((c) => /Library ID[:\s]+\d{10,20}/i.test(c)).slice(0, maxAds).map((chunk) => {
+      const lidM = chunk.match(/Library ID[:\s]+([0-9]{10,20})/i);
       const libraryId = lidM ? lidM[1] : null;
-      const statusM = chunk2.match(/^(Active|Inactive)/i);
+      const statusM = chunk.match(/^(Active|Inactive)/i);
       const status = statusM ? statusM[1] : null;
-      const startM = chunk2.match(/Started running on\s+([A-Za-z]+\.?\s+\d{1,2},?\s*\d{4})/i);
+      const startM = chunk.match(/Started running on\s+([A-Za-z]+\.?\s+\d{1,2},?\s*\d{4})/i);
       const started = startM ? startM[1].replace(/\s+/g, " ").trim() : null;
       const visual = adVisuals[libraryId ?? ""] ?? { imageSrc: null, videoSrc: null, videoPoster: null };
-      const hasVideoText = /0:00\s*\/\s*0:00|\d+:\d+\s*\/\s*\d+:\d+/.test(chunk2);
+      const hasVideoText = /0:00\s*\/\s*0:00|\d+:\d+\s*\/\s*\d+:\d+/.test(chunk);
       const creativeType = visual.videoSrc || hasVideoText ? "video" : visual.imageSrc ? "image" : "unknown";
       let domain = null;
       const domainRe = /\b([A-Z0-9]{2,}(?:\.[A-Z]{2,})+)\b/g;
       let dm;
-      while ((dm = domainRe.exec(chunk2)) !== null) {
+      while ((dm = domainRe.exec(chunk)) !== null) {
         const c = dm[1];
         if (c.length >= 6 && c !== "LIBRARY") {
           domain = c;
@@ -9869,9 +9864,9 @@ var FacebookAdExtractor = class {
       }
       let primaryText = null;
       let headline = null;
-      const sponsoredIdx = chunk2.search(/\bSponsored\b/i);
+      const sponsoredIdx = chunk.search(/\bSponsored\b/i);
       if (sponsoredIdx >= 0) {
-        let afterSponsored = chunk2.slice(sponsoredIdx + "Sponsored".length).trim();
+        let afterSponsored = chunk.slice(sponsoredIdx + "Sponsored".length).trim();
         afterSponsored = afterSponsored.replace(/\s*\d*:?\d+\s*\/\s*\d*:?\d+.*$/, "");
         if (domain) {
           const di = afterSponsored.indexOf(domain);
@@ -9882,18 +9877,18 @@ var FacebookAdExtractor = class {
       }
       const ctaRe2 = new RegExp("\\b(" + CTA_LABELS.map((l) => l.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|") + ")\\b", "i");
       if (domain) {
-        const di = chunk2.indexOf(domain);
+        const di = chunk.indexOf(domain);
         if (di >= 0) {
-          let afterDomain = chunk2.slice(di + domain.length).trim();
+          let afterDomain = chunk.slice(di + domain.length).trim();
           const ctaMatch = afterDomain.search(ctaRe2);
           if (ctaMatch >= 0) afterDomain = afterDomain.slice(0, ctaMatch).trim();
           if (afterDomain.length > 3 && afterDomain.length < 120) headline = afterDomain;
         }
       }
       if (!headline) {
-        const ctaIdx = chunk2.search(ctaRe2);
+        const ctaIdx = chunk.search(ctaRe2);
         if (ctaIdx > 0) {
-          const beforeCta = chunk2.slice(0, ctaIdx).trimEnd();
+          const beforeCta = chunk.slice(0, ctaIdx).trimEnd();
           const sentenceEnd = Math.max(beforeCta.lastIndexOf(". "), beforeCta.lastIndexOf("! "), beforeCta.lastIndexOf("? "));
           if (sentenceEnd >= 0) {
             const candidate = beforeCta.slice(sentenceEnd + 2).trim();
@@ -9905,12 +9900,12 @@ var FacebookAdExtractor = class {
       }
       let cta = null;
       for (const lbl of CTA_LABELS) {
-        if (chunk2.toLowerCase().includes(lbl.toLowerCase())) {
+        if (chunk.toLowerCase().includes(lbl.toLowerCase())) {
           cta = lbl;
           break;
         }
       }
-      const multiM = chunk2.match(/(\d+)\s+ad[s]?\s+use[s]?\s+this/i);
+      const multiM = chunk.match(/(\d+)\s+ad[s]?\s+use[s]?\s+this/i);
       const clusterCount = multiM ? parseInt(multiM[1]) : null;
       return {
         libraryId,
@@ -9933,8 +9928,8 @@ var FacebookAdExtractor = class {
     const unknownCreativeCount = ads.filter((a) => a.creativeType === "unknown").length;
     const advertiserPageId = new URL(page.url()).searchParams.get("view_all_page_id") ?? null;
     const nameFreq = /* @__PURE__ */ new Map();
-    for (const chunk2 of adChunks) {
-      const detailsParts = chunk2.split("See ad details ");
+    for (const chunk of adChunks) {
+      const detailsParts = chunk.split("See ad details ");
       if (detailsParts.length < 2) continue;
       const sponsoredIdx = detailsParts[1].indexOf(" Sponsored");
       if (sponsoredIdx < 0) continue;
@@ -10674,11 +10669,11 @@ facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
     }
     if (last < bodyText.length) adChunks.push(bodyText.slice(last));
     const advertiserMap = /* @__PURE__ */ new Map();
-    for (const chunk2 of adChunks) {
-      const lidM = chunk2.match(/Library ID[:\s]+([0-9]{10,20})/i);
+    for (const chunk of adChunks) {
+      const lidM = chunk.match(/Library ID[:\s]+([0-9]{10,20})/i);
       if (!lidM) continue;
       const libraryId = lidM[1];
-      const detailsParts = chunk2.split("See ad details ");
+      const detailsParts = chunk.split("See ad details ");
       if (detailsParts.length < 2) continue;
       const afterDetails = detailsParts[1];
       const sponsoredIdx = afterDetails.indexOf(" Sponsored");
@@ -11230,8 +11225,8 @@ function runFfmpegMux(videoPath, audioPath, outPath) {
       stdio: ["ignore", "ignore", "pipe"]
     });
     let stderr = "";
-    child.stderr.on("data", (chunk2) => {
-      stderr += String(chunk2);
+    child.stderr.on("data", (chunk) => {
+      stderr += String(chunk);
     });
     child.on("error", (err) => resolve({ ok: false, error: err.message }));
     child.on("close", (code) => resolve({ ok: code === 0, error: code === 0 ? null : stderr.trim() || `ffmpeg exited ${code}` }));
@@ -14363,8 +14358,64 @@ async function migrateBrowserAgent() {
     )
   `);
   await db.execute(`CREATE INDEX IF NOT EXISTS browser_agent_replays_session ON browser_agent_replays(session_id)`);
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS browser_auth_connections (
+      connection_id TEXT PRIMARY KEY,
+      hosted_url    TEXT,
+      domain        TEXT,
+      profile       TEXT,
+      account_email TEXT,
+      note          TEXT,
+      created_at    TEXT NOT NULL DEFAULT (datetime('now')),
+      updated_at    TEXT
+    )
+  `);
+  try {
+    await db.execute(`ALTER TABLE browser_auth_connections ADD COLUMN account_email TEXT`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE browser_auth_connections ADD COLUMN note TEXT`);
+  } catch {
+  }
+  try {
+    await db.execute(`ALTER TABLE browser_auth_connections ADD COLUMN updated_at TEXT`);
+  } catch {
+  }
+  await db.execute(`CREATE INDEX IF NOT EXISTS browser_auth_connections_profile ON browser_auth_connections(profile)`);
   _ready = true;
 }
+async function saveAuthConnection(input) {
+  if (!input.connectionId) return;
+  const db = getDb();
+  await db.execute({
+    sql: `INSERT INTO browser_auth_connections (connection_id, hosted_url, domain, profile, account_email, note, updated_at)
+          VALUES (?, ?, ?, ?, ?, ?, datetime('now'))
+          ON CONFLICT(connection_id) DO UPDATE SET
+            hosted_url    = COALESCE(excluded.hosted_url, browser_auth_connections.hosted_url),
+            domain        = COALESCE(excluded.domain, browser_auth_connections.domain),
+            profile       = COALESCE(excluded.profile, browser_auth_connections.profile),
+            account_email = COALESCE(excluded.account_email, browser_auth_connections.account_email),
+            note          = COALESCE(excluded.note, browser_auth_connections.note),
+            updated_at    = datetime('now')`,
+    args: [input.connectionId, input.hostedUrl, input.domain, input.profile, input.accountEmail ?? null, input.note ?? null]
+  });
+}
+async function getAuthConnectionRow(connectionId) {
+  if (!connectionId) return null;
+  const db = getDb();
+  const res = await db.execute({ sql: `SELECT * FROM browser_auth_connections WHERE connection_id = ?`, args: [connectionId] });
+  return res.rows[0] ?? null;
+}
+async function listAuthConnectionsByProfile(profile) {
+  if (!profile) return [];
+  const db = getDb();
+  const res = await db.execute({
+    sql: `SELECT * FROM browser_auth_connections WHERE profile = ? ORDER BY COALESCE(updated_at, created_at) DESC`,
+    args: [profile]
+  });
+  return res.rows;
+}
 async function createSessionRow(input) {
   const db = getDb();
   const id = `bas_${randomUUID2().replace(/-/g, "").slice(0, 20)}`;
@@ -14398,6 +14449,14 @@ async function listSessionRows(userId, includeClosed = false) {
   });
   return res.rows;
 }
+async function listIdleOpenSessionRows(idleSeconds) {
+  const db = getDb();
+  const res = await db.execute({
+    sql: `SELECT * FROM browser_agent_sessions WHERE status = 'open' AND COALESCE(last_action_at, created_at) <= datetime('now', ?) ORDER BY created_at LIMIT 200`,
+    args: [`-${Math.max(1, Math.round(idleSeconds))} seconds`]
+  });
+  return res.rows;
+}
 async function addActiveMs(id, deltaMs) {
   const db = getDb();
   await db.execute({
@@ -15197,6 +15256,17 @@ async function findProfileConnection(k, profileName, domain) {
   }
   return null;
 }
+async function listProfileConnections(profileName) {
+  if (!profileName) return [];
+  const k = client();
+  const out = [];
+  for await (const connection of k.auth.connections.list({ profile_name: profileName })) {
+    const data = connection;
+    if (data.profile_name && data.profile_name !== profileName) continue;
+    out.push(profileOnboardResult(connection));
+  }
+  return out;
+}
 async function findOrCreateProfileConnection(k, opts) {
   try {
     return await k.auth.connections.create({
@@ -15575,29 +15645,61 @@ function browserSessionLockTtlSeconds(timeoutSeconds) {
   }
   return DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS;
 }
-async function charge(sessionId, userId, startedAtMs) {
-  const elapsedMs = Date.now() - startedAtMs;
-  const { active_ms, billed_mc } = await addActiveMs(sessionId, elapsedMs);
-  const owed = browserActiveCostMc(active_ms);
-  const delta = owed - billed_mc;
+async function charge(sessionId, userId, _startedAtMs) {
+  const row = await getSessionRow(sessionId);
+  if (!row) return;
+  const createdMs = parseAgentDate(row.created_at);
+  if (createdMs == null) return;
+  const uptimeMs = Math.max(0, Date.now() - createdMs);
+  const owed = browserActiveCostMc(uptimeMs);
+  await addActiveMs(sessionId, 0);
+  const delta = owed - Number(row.billed_mc ?? 0);
   if (delta > 0) {
     const res = await debitMc(userId, delta, LedgerOperation.BROWSER_SESSION, sessionId);
     if (res.ok) await setBilledMc(sessionId, owed);
   }
 }
+async function reapIdleBrowserSessions(idleSeconds = 120) {
+  const rows = await listIdleOpenSessionRows(idleSeconds);
+  let reaped = 0;
+  for (const row of rows) {
+    if (row.user_id != null) {
+      try {
+        await charge(row.id, row.user_id);
+      } catch {
+      }
+    }
+    try {
+      await closeSession(row.runtime_session_id);
+    } catch {
+    }
+    await markSessionClosed(row.id).catch(() => {
+    });
+    if (row.concurrency_lock_id) await releaseConcurrencyGate(row.concurrency_lock_id).catch(() => {
+    });
+    reaped++;
+  }
+  return { reaped };
+}
 function publicSession(row) {
   return {
     session_id: row.id,
     status: row.status,
     label: row.label,
-    live_view_url: row.live_view_url,
+    watch_url: `/console/${row.id}`,
     created_at: row.created_at,
     last_action_at: row.last_action_at,
     closed_at: row.closed_at,
-    active_seconds: Math.round((row.active_ms ?? 0) / 1e3),
+    uptime_seconds: uptimeSeconds(row),
     credits_used: Math.round((row.billed_mc ?? 0) / 10) / 100
   };
 }
+function uptimeSeconds(row) {
+  const createdMs = parseAgentDate(row.created_at);
+  const endMs = row.closed_at ? parseAgentDate(row.closed_at) : Date.now();
+  if (createdMs == null || endMs == null) return 0;
+  return Math.max(0, Math.round((endMs - createdMs) / 1e3));
+}
 function failure(err) {
   const msg = err instanceof Error ? err.message : String(err);
   return { error: sanitizeVendorName(msg) };
@@ -15646,10 +15748,23 @@ function buildBrowserAgentRoutes() {
     const profileName = typeof body.profile === "string" ? body.profile.trim() : "";
     const domain = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : "chatgpt.com";
     const loginUrl = typeof body.login_url === "string" ? body.login_url.trim() : void 0;
+    const accountEmail = typeof body.account_email === "string" ? body.account_email.trim() || null : null;
+    const note = typeof body.note === "string" ? body.note.trim() || null : null;
     if (!profileName) return c.json({ error: "profile is required" }, 400);
     if (!domain) return c.json({ error: "domain is required" }, 400);
     try {
       const result = await startProfileOnboarding({ profileName, domain, loginUrl });
+      try {
+        await saveAuthConnection({
+          connectionId: result.connectionId,
+          hostedUrl: result.hostedUrl,
+          domain: result.domain,
+          profile: result.profileName,
+          accountEmail,
+          note
+        });
+      } catch {
+      }
       return c.json({
         connection_id: result.connectionId,
         profile: result.profileName,
@@ -15658,8 +15773,6 @@ function buildBrowserAgentRoutes() {
         flow_status: result.flowStatus,
         flow_step: result.flowStep,
         flow_expires_at: result.flowExpiresAt,
-        hosted_url: result.hostedUrl,
-        live_view_url: result.liveViewUrl,
         browser_session_id: result.browserSessionId,
         post_login_url: result.postLoginUrl
       });
@@ -15674,6 +15787,17 @@ function buildBrowserAgentRoutes() {
     const domain = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : void 0;
     try {
       const result = await getProfileOnboardingStatus({ connectionId, profileName, domain });
+      if (result.hostedUrl) {
+        try {
+          await saveAuthConnection({
+            connectionId: result.connectionId,
+            hostedUrl: result.hostedUrl,
+            domain: result.domain,
+            profile: result.profileName
+          });
+        } catch {
+        }
+      }
       return c.json({
         connection_id: result.connectionId,
         profile: result.profileName,
@@ -15682,8 +15806,6 @@ function buildBrowserAgentRoutes() {
         flow_status: result.flowStatus,
         flow_step: result.flowStep,
         flow_expires_at: result.flowExpiresAt,
-        hosted_url: result.hostedUrl,
-        live_view_url: result.liveViewUrl,
         browser_session_id: result.browserSessionId,
         post_login_url: result.postLoginUrl
       });
@@ -15691,6 +15813,37 @@ function buildBrowserAgentRoutes() {
       return c.json(failure(err), 404);
     }
   });
+  app2.post("/profiles/list", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const profileName = typeof body.profile === "string" ? body.profile.trim() : "";
+    if (!profileName) return c.json({ error: "profile is required" }, 400);
+    const domainFilter = typeof body.domain === "string" ? body.domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "") : void 0;
+    const connectionIdFilter = typeof body.connection_id === "string" ? body.connection_id.trim() : void 0;
+    try {
+      const [live, stored] = await Promise.all([
+        listProfileConnections(profileName),
+        listAuthConnectionsByProfile(profileName).catch(() => [])
+      ]);
+      const notesByDomain = new Map(stored.map((row) => [row.domain ?? "", row]));
+      const notesByConnection = new Map(stored.map((row) => [row.connection_id, row]));
+      let connections = live.map((conn) => {
+        const record = notesByConnection.get(conn.connectionId) ?? notesByDomain.get(conn.domain);
+        return {
+          connection_id: conn.connectionId,
+          domain: conn.domain,
+          status: conn.status,
+          account_email: record?.account_email ?? null,
+          note: record?.note ?? null,
+          last_connected_at: record?.updated_at ?? record?.created_at ?? null
+        };
+      });
+      if (connectionIdFilter) connections = connections.filter((c2) => c2.connection_id === connectionIdFilter);
+      if (domainFilter) connections = connections.filter((c2) => c2.domain === domainFilter);
+      return c.json({ profile: profileName, connections });
+    } catch (err) {
+      return c.json(failure(err), 502);
+    }
+  });
   app2.post("/sessions", async (c) => {
     const user = c.get("user");
     if (Number(user.balance_mc ?? 0) < BROWSER_OPEN_MIN_BALANCE_MC) {
@@ -15758,6 +15911,7 @@ function buildBrowserAgentRoutes() {
     const user = c.get("user");
     const row = await loadOpenSession(c.req.param("id"), user.id);
     if (!row) return c.json({ error: "not found" }, 404);
+    await charge(row.id, user.id);
     try {
       await closeSession(row.runtime_session_id);
     } catch {
@@ -16047,6 +16201,50 @@ function buildBrowserAgentRoutes() {
 }
 
 // src/api/browser-agent-console.ts
+function renderAuthConsoleHtml(connectionId, hostedUrl) {
+  const safeId = String(connectionId).replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c]);
+  const goUrl = `/console/auth/${encodeURIComponent(connectionId)}/go`;
+  const body = hostedUrl ? `<iframe class="stage" allow="clipboard-read; clipboard-write" src="${goUrl}"></iframe>
+       <div class="fallback">
+         <span>Sign-in window not showing?</span>
+         <a class="btn" href="${goUrl}" target="_blank" rel="noopener">Open sign-in in a new tab</a>
+       </div>` : `<div class="empty">
+         <div>This sign-in link is no longer active.</div>
+         <div class="muted">Ask the assistant to reconnect the profile, then open the new link.</div>
+       </div>`;
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+<title>MCP Scraper \u2014 Secure Sign-in</title>
+<style>
+  :root { color-scheme: dark; }
+  :where(*) { box-sizing: border-box; }
+  body { margin: 0; height: 100vh; display: flex; flex-direction: column; font: 14px/1.5 ui-sans-serif, system-ui, -apple-system, sans-serif; background: #0b0e14; color: #d7dce5; }
+  header { display: flex; align-items: center; gap: 10px; padding: 12px 18px; border-bottom: 1px solid #1c2230; background: #0f131c; }
+  header h1 { font-size: 14px; margin: 0; font-weight: 600; color: #fff; }
+  header .muted { color: #6b7689; font-size: 12px; }
+  .spacer { flex: 1; }
+  .stage { flex: 1; width: 100%; border: 0; display: block; background: #05070b; }
+  .fallback { display: flex; align-items: center; gap: 12px; padding: 10px 18px; border-top: 1px solid #1c2230; background: #0f131c; font-size: 13px; color: #8893a7; }
+  .btn { background: #2b6cff; color: #fff; border-radius: 7px; padding: 7px 12px; text-decoration: none; font-weight: 500; }
+  .empty { flex: 1; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 8px; color: #8893a7; }
+  .muted { color: #6b7689; font-size: 12px; }
+</style>
+${ATTRIBUTION_PIXEL_SCRIPT}
+</head>
+<body>
+<header>
+  <h1>MCP Scraper</h1>
+  <span class="muted">Secure sign-in \xB7 ${safeId}</span>
+  <span class="spacer"></span>
+  <span class="muted">Complete the login below \u2014 your session is saved to your profile.</span>
+</header>
+${body}
+</body>
+</html>`;
+}
 function renderConsoleHtml(initialSessionId) {
   const initial = JSON.stringify(initialSessionId ?? "");
   return `<!DOCTYPE html>
@@ -17378,14 +17576,15 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-2WVKKCC7.js");
+  const { drainQueue } = await import("./worker-YZ2ZJE4F.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
-  const [results, sweepResult] = await Promise.all([
+  const [results, sweepResult, reapResult] = await Promise.all([
     drainQueue(budget),
-    runMonthlyRefreshSweep()
+    runMonthlyRefreshSweep(),
+    reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 }))
   ]);
-  return c.json({ drained: results.length, results, sweepResult, workflowDispatch: workflowDispatchResult });
+  return c.json({ drained: results.length, results, sweepResult, reaped: reapResult, workflowDispatch: workflowDispatchResult });
 });
 app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({ client: inngest, functions: [siteAuditFn, siteExtractFn] }));
 app.route("/api/internal/site-architecture-auditor", siteAuditApp);
@@ -17400,6 +17599,19 @@ app.route("/serp-intelligence", serpIntelligenceApp);
 app.route("/mcp", mcpApp);
 app.route("/agent", buildBrowserAgentRoutes());
 app.get("/console", (c) => c.html(renderConsoleHtml()));
+app.get("/console/auth/:id", async (c) => {
+  const id = c.req.param("id");
+  await migrateBrowserAgent();
+  const row = await getAuthConnectionRow(id);
+  return c.html(renderAuthConsoleHtml(id, row?.hosted_url ?? null));
+});
+app.get("/console/auth/:id/go", async (c) => {
+  const id = c.req.param("id");
+  await migrateBrowserAgent();
+  const row = await getAuthConnectionRow(id);
+  if (!row?.hosted_url) return c.text("This sign-in link is no longer active.", 404);
+  return c.redirect(row.hosted_url, 302);
+});
 app.get("/console/:id", (c) => c.html(renderConsoleHtml(c.req.param("id"))));
 app.route("/stripe", stripeApp);
 if (!process.env.INNGEST_EVENT_KEY) {
@@ -17508,4 +17720,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-TKWY47JA.js.map
+//# sourceMappingURL=server-EYNMXWUJ.js.map