mcp-scraper 0.38.1 → 0.38.2

package/dist/bin/api-server.cjs

@@ -3622,8 +3622,10 @@ __export(db_exports, {
   claimMonthlyFreeRefresh: () => claimMonthlyFreeRefresh,
   claimPendingJob: () => claimPendingJob,
   claimPendingKpoJob: () => claimPendingKpoJob,
+  claimStripeEvent: () => claimStripeEvent,
   completeJob: () => completeJob,
   completeKpoJob: () => completeKpoJob,
+  completeStripeEvent: () => completeStripeEvent,
   completeWorkflowRunRecord: () => completeWorkflowRunRecord,
   consumeCode: () => consumeCode,
   consumePasswordResetToken: () => consumePasswordResetToken,
@@ -3645,6 +3647,7 @@ __export(db_exports, {
   expireOldLots: () => expireOldLots,
   failJob: () => failJob,
   failKpoJob: () => failKpoJob,
+  failStripeEvent: () => failStripeEvent,
   failWorkflowRunRecord: () => failWorkflowRunRecord,
   finishHarvestAttempt: () => finishHarvestAttempt,
   generateApiKey: () => generateApiKey,
@@ -3699,7 +3702,6 @@ __export(db_exports, {
   putCode: () => putCode,
   putRefresh: () => putRefresh,
   reconcileBalanceMc: () => reconcileBalanceMc,
-  recordStripeEvent: () => recordStripeEvent,
   recordWorkflowWebhookDelivery: () => recordWorkflowWebhookDelivery,
   registerClient: () => registerClient,
   replaceWorkflowArtifacts: () => replaceWorkflowArtifacts,
@@ -3733,6 +3735,7 @@ async function hasCurrentCoreSchema(db) {
   const checks = [
     `(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN (${sqlStringList(CORE_SCHEMA_TABLES)})) = ${CORE_SCHEMA_TABLES.length}`,
     `(SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name IN (${sqlStringList(CORE_SCHEMA_INDEXES)})) = ${CORE_SCHEMA_INDEXES.length}`,
+    `(SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger' AND name IN (${sqlStringList(CORE_SCHEMA_TRIGGERS)})) = ${CORE_SCHEMA_TRIGGERS.length}`,
     ...Object.entries(CORE_SCHEMA_COLUMNS).map(
       ([table, columns]) => `(SELECT COUNT(*) FROM pragma_table_info('${table}') WHERE name IN (${sqlStringList(columns)})) = ${columns.length}`
     )
@@ -4008,6 +4011,61 @@ async function migrate() {
   } catch {
   }
   await db.execute(`CREATE INDEX IF NOT EXISTS credit_lots_user_active ON credit_lots(user_id, expires_at, remaining_mc)`);
+  await db.execute(`
+    CREATE TABLE IF NOT EXISTS credit_lot_expirations (
+      lot_id       INTEGER PRIMARY KEY REFERENCES credit_lots(id),
+      user_id      INTEGER NOT NULL REFERENCES users(id),
+      amount_mc    INTEGER NOT NULL,
+      claim_token  TEXT NOT NULL UNIQUE,
+      expired_at   TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
+  await db.execute(`
+    CREATE TRIGGER IF NOT EXISTS credit_lots_bounds_insert
+    BEFORE INSERT ON credit_lots
+    WHEN NEW.amount_mc < 0
+      OR NEW.remaining_mc < 0
+      OR NEW.remaining_mc > NEW.amount_mc
+    BEGIN
+      SELECT RAISE(ABORT, 'invalid credit lot bounds');
+    END
+  `);
+  await db.execute(`
+    CREATE TRIGGER IF NOT EXISTS credit_lots_bounds_update
+    BEFORE UPDATE OF amount_mc, remaining_mc ON credit_lots
+    WHEN NEW.amount_mc < 0
+      OR NEW.remaining_mc < 0
+      OR NEW.remaining_mc > NEW.amount_mc
+    BEGIN
+      SELECT RAISE(ABORT, 'invalid credit lot bounds');
+    END
+  `);
+  await db.execute(`
+    CREATE TRIGGER IF NOT EXISTS credit_lots_one_migration_insert
+    BEFORE INSERT ON credit_lots
+    WHEN NEW.source = 'migration'
+      AND EXISTS (
+        SELECT 1 FROM credit_lots
+        WHERE user_id = NEW.user_id AND source = 'migration'
+      )
+    BEGIN
+      SELECT RAISE(ABORT, 'duplicate migration lot');
+    END
+  `);
+  await db.execute(`
+    CREATE TRIGGER IF NOT EXISTS credit_lots_one_migration_update
+    BEFORE UPDATE OF source, user_id ON credit_lots
+    WHEN NEW.source = 'migration'
+      AND EXISTS (
+        SELECT 1 FROM credit_lots
+        WHERE user_id = NEW.user_id
+          AND source = 'migration'
+          AND id != OLD.id
+      )
+    BEGIN
+      SELECT RAISE(ABORT, 'duplicate migration lot');
+    END
+  `);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS credit_lot_repairs (
       repair_id       TEXT PRIMARY KEY,
@@ -4397,10 +4455,37 @@ async function migrate() {
   await db.execute(`CREATE INDEX IF NOT EXISTS workflow_webhook_deliveries_run ON workflow_webhook_deliveries(run_id)`);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS stripe_events (
-      event_id     TEXT PRIMARY KEY,
-      processed_at TEXT NOT NULL DEFAULT (datetime('now'))
+      event_id      TEXT PRIMARY KEY,
+      status        TEXT NOT NULL DEFAULT 'processed',
+      attempt_count INTEGER NOT NULL DEFAULT 1,
+      claim_token   TEXT,
+      last_error    TEXT,
+      processed_at  TEXT NOT NULL DEFAULT (datetime('now')),
+      updated_at    TEXT NOT NULL DEFAULT (datetime('now'))
     )
   `);
+  for (const col of [
+    `status TEXT NOT NULL DEFAULT 'processed'`,
+    "attempt_count INTEGER NOT NULL DEFAULT 1",
+    "claim_token TEXT",
+    "last_error TEXT",
+    "updated_at TEXT"
+  ]) {
+    try {
+      await db.execute(`ALTER TABLE stripe_events ADD COLUMN ${col}`);
+    } catch {
+    }
+  }
+  await db.execute(`
+    UPDATE stripe_events
+    SET updated_at = COALESCE(updated_at, processed_at),
+        status = COALESCE(status, 'processed'),
+        attempt_count = MAX(1, COALESCE(attempt_count, 1))
+    WHERE updated_at IS NULL
+       OR status IS NULL
+       OR attempt_count IS NULL
+       OR attempt_count < 1
+  `);
   await db.execute(`
     CREATE TABLE IF NOT EXISTS oauth_clients (
       client_id     TEXT PRIMARY KEY,
@@ -4566,16 +4651,60 @@ async function rotateRefresh(oldToken, next) {
     args: [next.refresh_token, (/* @__PURE__ */ new Date()).toISOString(), oldToken]
   });
 }
-async function recordStripeEvent(eventId) {
-  const res = await getDb().execute({
-    sql: "INSERT OR IGNORE INTO stripe_events (event_id) VALUES (?)",
+async function claimStripeEvent(eventId) {
+  const token = (0, import_node_crypto.randomUUID)();
+  const claimed = await getDb().execute({
+    sql: `INSERT INTO stripe_events
+            (event_id, status, attempt_count, claim_token, last_error, processed_at, updated_at)
+          VALUES (?, 'processing', 1, ?, NULL, datetime('now'), datetime('now'))
+          ON CONFLICT(event_id) DO UPDATE SET
+            status = 'processing',
+            attempt_count = stripe_events.attempt_count + 1,
+            claim_token = excluded.claim_token,
+            last_error = NULL,
+            updated_at = datetime('now')
+          WHERE stripe_events.status = 'failed'
+             OR (
+               stripe_events.status = 'processing'
+               AND stripe_events.updated_at <= datetime('now', '-10 minutes')
+             )`,
+    args: [eventId, token]
+  });
+  if (claimed.rowsAffected > 0) return { state: "claimed", token };
+  const current = await getDb().execute({
+    sql: "SELECT status FROM stripe_events WHERE event_id = ? LIMIT 1",
     args: [eventId]
   });
-  return res.rowsAffected > 0;
+  return { state: String(current.rows[0]?.status) === "processed" ? "processed" : "processing" };
+}
+async function completeStripeEvent(eventId, token) {
+  const result = await getDb().execute({
+    sql: `UPDATE stripe_events
+          SET status = 'processed',
+              claim_token = NULL,
+              last_error = NULL,
+              processed_at = datetime('now'),
+              updated_at = datetime('now')
+          WHERE event_id = ? AND status = 'processing' AND claim_token = ?`,
+    args: [eventId, token]
+  });
+  return result.rowsAffected === 1;
+}
+async function failStripeEvent(eventId, token, error) {
+  const result = await getDb().execute({
+    sql: `UPDATE stripe_events
+          SET status = 'failed',
+              claim_token = NULL,
+              last_error = ?,
+              updated_at = datetime('now')
+          WHERE event_id = ? AND status = 'processing' AND claim_token = ?`,
+    args: [error.slice(0, 500), eventId, token]
+  });
+  return result.rowsAffected === 1;
 }
 async function stripeEventAlreadyProcessed(eventId) {
   const res = await getDb().execute({
-    sql: "SELECT 1 FROM stripe_events WHERE event_id = ? LIMIT 1",
+    sql: `SELECT 1 FROM stripe_events WHERE event_id = ? AND status = 'processed' LIMIT 1`,
     args: [eventId]
   });
   return res.rows.length > 0;
@@ -5578,23 +5707,64 @@ async function ensureMigrated(userId) {
 async function expireOldLots() {
   const db = getDb();
   const lots = await db.execute({
-    sql: `SELECT id, user_id, remaining_mc FROM credit_lots WHERE remaining_mc > 0 AND never_expires = 0 AND expires_at <= datetime('now') LIMIT 500`
+    sql: `SELECT id
+          FROM credit_lots
+          WHERE remaining_mc > 0
+            AND never_expires = 0
+            AND expires_at <= datetime('now')
+          LIMIT 500`
   });
   if (lots.rows.length === 0) return { expired_lots: 0, expired_mc: 0 };
-  let totalMc = 0;
-  const users = /* @__PURE__ */ new Set();
+  const claims = [];
   const stmts = [];
   for (const lot of lots.rows) {
-    const rem = Number(lot.remaining_mc);
-    const uid = Number(lot.user_id);
-    totalMc += rem;
-    users.add(uid);
-    stmts.push({ sql: "UPDATE credit_lots SET remaining_mc = 0 WHERE id = ?", args: [Number(lot.id)] });
-    stmts.push({ sql: `INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, 'credit_expired', 'credits expired')`, args: [uid, -rem] });
-  }
-  await db.batch(stmts, "write");
+    const lotId = Number(lot.id);
+    const token = (0, import_node_crypto.randomUUID)();
+    claims.push({ token, insertResultIndex: stmts.length });
+    stmts.push({
+      sql: `INSERT OR IGNORE INTO credit_lot_expirations (lot_id, user_id, amount_mc, claim_token)
+            SELECT id, user_id, remaining_mc, ?
+            FROM credit_lots
+            WHERE id = ?
+              AND remaining_mc > 0
+              AND never_expires = 0
+              AND expires_at <= datetime('now')`,
+      args: [token, lotId]
+    });
+    stmts.push({
+      sql: `UPDATE credit_lots
+            SET remaining_mc = 0
+            WHERE id = ?
+              AND EXISTS (
+                SELECT 1 FROM credit_lot_expirations
+                WHERE lot_id = ? AND claim_token = ?
+              )`,
+      args: [lotId, lotId, token]
+    });
+    stmts.push({
+      sql: `INSERT INTO ledger (user_id, amount_mc, operation, description)
+            SELECT user_id, -amount_mc, 'credit_expired', 'credits expired'
+            FROM credit_lot_expirations
+            WHERE lot_id = ? AND claim_token = ?`,
+      args: [lotId, token]
+    });
+  }
+  const results = await db.batch(stmts, "write");
+  const claimedTokens = claims.filter((claim) => results[claim.insertResultIndex]?.rowsAffected === 1).map((claim) => claim.token);
+  if (claimedTokens.length === 0) return { expired_lots: 0, expired_mc: 0 };
+  const placeholders = claimedTokens.map(() => "?").join(", ");
+  const expired = await db.execute({
+    sql: `SELECT user_id, amount_mc
+          FROM credit_lot_expirations
+          WHERE claim_token IN (${placeholders})`,
+    args: claimedTokens
+  });
+  const users = new Set(expired.rows.map((row) => Number(row.user_id)));
   for (const uid of users) await reconcileBalanceMc(uid);
-  return { expired_lots: lots.rows.length, expired_mc: totalMc };
+  return {
+    expired_lots: expired.rows.length,
+    expired_mc: expired.rows.reduce((sum, row) => sum + Number(row.amount_mc), 0)
+  };
 }
 async function migrateExistingBalancesToLots() {
   const db = getDb();
@@ -6184,29 +6354,46 @@ async function ledgerExistsForStripePI(stripePI) {
 }
 async function reconcileBalanceMc(userId, knownBalanceMc) {
   const db = getDb();
-  const res = await db.execute({
-    sql: `SELECT
-      COUNT(*) AS lot_count,
-      COALESCE(SUM(CASE WHEN remaining_mc > 0 AND (never_expires = 1 OR expires_at > datetime('now')) THEN remaining_mc ELSE 0 END), 0) AS balance_mc
-      FROM credit_lots WHERE user_id = ?`,
-    args: [userId]
-  });
-  let balanceMc = Number(res.rows[0]?.balance_mc ?? 0);
-  if (Number(res.rows[0]?.lot_count ?? 0) === 0) {
-    await ensureMigrated(Number(userId));
-    const migrated = await db.execute({
-      sql: `SELECT COALESCE(SUM(remaining_mc), 0) AS balance_mc FROM credit_lots WHERE user_id = ? AND remaining_mc > 0 AND (never_expires = 1 OR expires_at > datetime('now'))`,
+  void knownBalanceMc;
+  for (let attempt = 0; attempt < 8; attempt += 1) {
+    const snapshot = await db.execute({
+      sql: `SELECT
+              u.balance_mc AS cached_balance_mc,
+              COUNT(l.id) AS lot_count,
+              COALESCE(SUM(CASE
+                WHEN l.remaining_mc > 0
+                  AND (l.never_expires = 1 OR l.expires_at > datetime('now'))
+                THEN l.remaining_mc ELSE 0 END), 0) AS projected_balance_mc
+            FROM users u
+            LEFT JOIN credit_lots l ON l.user_id = u.id
+            WHERE u.id = ?
+            GROUP BY u.id, u.balance_mc`,
       args: [userId]
     });
-    balanceMc = Number(migrated.rows[0]?.balance_mc ?? 0);
-  }
-  if (knownBalanceMc !== balanceMc) {
-    await db.execute({
-      sql: "UPDATE users SET balance_mc = ? WHERE id = ? AND balance_mc != ?",
-      args: [balanceMc, userId, balanceMc]
+    const row = snapshot.rows[0];
+    if (!row) return 0;
+    if (Number(row.lot_count ?? 0) === 0) {
+      await ensureMigrated(Number(userId));
+      const nowHasLots = await db.execute({
+        sql: "SELECT 1 FROM credit_lots WHERE user_id = ? LIMIT 1",
+        args: [userId]
+      });
+      if (nowHasLots.rows.length > 0) continue;
+    }
+    const cachedBalanceMc = Number(row.cached_balance_mc ?? 0);
+    const projectedBalanceMc = Number(row.projected_balance_mc ?? 0);
+    if (cachedBalanceMc === projectedBalanceMc) return projectedBalanceMc;
+    const updated = await db.execute({
+      sql: "UPDATE users SET balance_mc = ? WHERE id = ? AND balance_mc = ?",
+      args: [projectedBalanceMc, userId, cachedBalanceMc]
     });
+    if (updated.rowsAffected === 1) return projectedBalanceMc;
   }
-  return balanceMc;
+  const current = await db.execute({
+    sql: "SELECT balance_mc FROM users WHERE id = ? LIMIT 1",
+    args: [userId]
+  });
+  return Number(current.rows[0]?.balance_mc ?? 0);
 }
 async function listInboxMessages(userId) {
   const res = await getDb().execute({
@@ -6269,7 +6456,7 @@ async function listActiveUserIds() {
   const res = await getDb().execute("SELECT id FROM users WHERE active = 1");
   return res.rows.map((r) => Number(r.id));
 }
-var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CORE_SCHEMA_VERSION, CORE_SCHEMA_TABLES, CORE_SCHEMA_INDEXES, CORE_SCHEMA_COLUMNS, REQUEST_EVENT_RESULT_MAX_BYTES, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
+var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CORE_SCHEMA_VERSION, CORE_SCHEMA_TABLES, CORE_SCHEMA_INDEXES, CORE_SCHEMA_TRIGGERS, CORE_SCHEMA_COLUMNS, REQUEST_EVENT_RESULT_MAX_BYTES, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
 var init_db = __esm({
   "src/api/db.ts"() {
     "use strict";
@@ -6282,7 +6469,7 @@ var init_db = __esm({
     DB_TOKEN = process.env.TURSO_AUTH_TOKEN;
     _db = null;
     _rateLimitSchemaReady = false;
-    CORE_SCHEMA_VERSION = "2026-07-28.2";
+    CORE_SCHEMA_VERSION = "2026-07-28.3";
     CORE_SCHEMA_TABLES = [
       "admin_credit_adjustment_events",
       "admin_credit_adjustments",
@@ -6294,6 +6481,7 @@ var init_db = __esm({
       "browser_profile_labels",
       "connected_account_billing",
       "concurrency_locks",
+      "credit_lot_expirations",
       "credit_lots",
       "credit_lot_repairs",
       "free_credit_refreshes",
@@ -6362,6 +6550,12 @@ var init_db = __esm({
       "workflow_schedules_user_status",
       "workflow_webhook_deliveries_run"
     ];
+    CORE_SCHEMA_TRIGGERS = [
+      "credit_lots_bounds_insert",
+      "credit_lots_bounds_update",
+      "credit_lots_one_migration_insert",
+      "credit_lots_one_migration_update"
+    ];
     CORE_SCHEMA_COLUMNS = {
       credit_lots: ["never_expires"],
       harvest_attempts: ["debug_json"],
@@ -6379,6 +6573,7 @@ var init_db = __esm({
         "next_settlement_at",
         "settlement_error"
       ],
+      stripe_events: ["attempt_count", "claim_token", "last_error", "status", "updated_at"],
       users: [
         "balance_mc",
         "concurrency_stripe_sub_id",
@@ -6470,7 +6665,7 @@ var init_cost_rates = __esm({
     "use strict";
     KERNEL_HEADLESS_USD_PER_SEC = 166667e-10;
     KERNEL_HEADFUL_USD_PER_SEC = 1333336e-10;
-    HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "maps_search", "yt_channel", "yt_transcription", "fb_search", "fb_ad", "instagram"]);
+    HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "fb_search", "fb_ad", "instagram"]);
     FAL_WIZPER_USD_PER_COMPUTE_SEC = 111e-5;
     FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR = 4;
     DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS = 4e-4;
@@ -8066,12 +8261,15 @@ async function spiderSite(opts) {
         const discovered = { url, status, linkedFrom: from, via: "fetch" };
         const children = [];
         if (html) {
-          for (const link of extractLinks(html, url)) {
-            if (baseDomain(link) !== startDomain) continue;
-            if (!visited.has(link)) {
-              visited.add(link);
-              children.push({ url: link, from: url });
+          try {
+            for (const link of extractLinks(html, url)) {
+              if (baseDomain(link) !== startDomain) continue;
+              if (!visited.has(link)) {
+                visited.add(link);
+                children.push({ url: link, from: url });
+              }
             }
+          } catch {
           }
         }
         return { discovered, children };
@@ -10051,67 +10249,78 @@ async function extractSite(opts) {
   let truncated = false;
   let spider;
   let sitemapUrls = [];
-  if (rotating) {
-    const browserConcurrency = Math.max(1, Math.min(opts.parallelism ?? 1, 8));
-    const urlsPerBrowser = opts.rotateProxyEvery ?? 10;
-    const followLinks = !seeded;
-    const seen = /* @__PURE__ */ new Set();
-    const frontier = [];
-    const enqueue = (u) => {
-      const n = normalizeUrl(u, opts.startUrl);
-      if (n && !seen.has(n)) {
-        seen.add(n);
-        frontier.push(n);
-      }
-    };
-    if (seeded) {
-      for (const u of opts.seedUrls) enqueue(u);
-    } else {
-      enqueue(opts.startUrl);
-      sitemapUrls = await discoverSitemapUrls2(opts.startUrl, maxPages);
-      for (const u of sitemapUrls) enqueue(u);
-    }
-    const waveCap = Math.max(browserConcurrency * urlsPerBrowser, 30);
-    let done = 0;
-    while (frontier.length > 0 && pages.length < maxPages) {
-      const capacity = maxPages - pages.length;
-      const wave = frontier.splice(0, Math.min(capacity, waveCap));
-      const fetched = await crawlWithRotation(wave, {
-        apiKey: opts.kernelApiKey,
-        concurrency: browserConcurrency,
-        urlsPerBrowser,
-        onPage: (p) => {
-          done++;
-          opts.onProgress?.({ done, discovered: seen.size, frontier: frontier.length, lastOk: p.ok ? 1 : 0, lastFail: p.ok ? 0 : 1, phase: p.phase });
+  let stoppedEarly = null;
+  try {
+    if (rotating) {
+      const browserConcurrency = Math.max(1, Math.min(opts.parallelism ?? 1, 8));
+      const urlsPerBrowser = opts.rotateProxyEvery ?? 10;
+      const followLinks = !seeded;
+      const seen = /* @__PURE__ */ new Set();
+      const frontier = [];
+      const enqueue = (u) => {
+        const n = normalizeUrl(u, opts.startUrl);
+        if (n && !seen.has(n)) {
+          seen.add(n);
+          frontier.push(n);
         }
-      });
-      for (const r of fetched) {
-        const page = r.html && r.status >= 200 && r.status < 300 ? parsePageData(r.url, r.html, r.status, "browser", { headers: r.headers, responseTimeMs: r.responseTimeMs, redirectUrl: r.redirectUrl }, [successfulAttempt("browser", r.status, r.responseTimeMs ?? null)]) : emptyPageData(r.url, r.status || null, "browser", {
-          code: r.errorCode ?? "browser_empty_response",
-          message: r.error ?? "Browser fetch returned no usable response.",
-          attempts: [failedAttempt("browser", r.status || null, r.responseTimeMs ?? null, r.errorCode ?? "browser_empty_response", r.error ?? "Browser fetch returned no usable response.")]
+      };
+      if (seeded) {
+        for (const u of opts.seedUrls) enqueue(u);
+      } else {
+        enqueue(opts.startUrl);
+        sitemapUrls = await discoverSitemapUrls2(opts.startUrl, maxPages);
+        for (const u of sitemapUrls) enqueue(u);
+      }
+      const waveCap = Math.max(browserConcurrency * urlsPerBrowser, 30);
+      let done = 0;
+      while (frontier.length > 0 && pages.length < maxPages) {
+        const capacity = maxPages - pages.length;
+        const wave = frontier.splice(0, Math.min(capacity, waveCap));
+        const fetched = await crawlWithRotation(wave, {
+          apiKey: opts.kernelApiKey,
+          concurrency: browserConcurrency,
+          urlsPerBrowser,
+          onPage: (p) => {
+            done++;
+            opts.onProgress?.({ done, discovered: seen.size, frontier: frontier.length, lastOk: p.ok ? 1 : 0, lastFail: p.ok ? 0 : 1, phase: p.phase });
+          }
         });
-        pages.push(page);
-        browserRetries++;
-        if (followLinks && isPageExtractionSuccessful(page)) {
-          for (const href of page.discoveryLinks ?? page.outlinks.filter((link) => link.internal).map((link) => link.href)) enqueue(href);
+        for (const r of fetched) {
+          const page = r.html && r.status >= 200 && r.status < 300 ? parsePageData(r.url, r.html, r.status, "browser", { headers: r.headers, responseTimeMs: r.responseTimeMs, redirectUrl: r.redirectUrl }, [successfulAttempt("browser", r.status, r.responseTimeMs ?? null)]) : emptyPageData(r.url, r.status || null, "browser", {
+            code: r.errorCode ?? "browser_empty_response",
+            message: r.error ?? "Browser fetch returned no usable response.",
+            attempts: [failedAttempt("browser", r.status || null, r.responseTimeMs ?? null, r.errorCode ?? "browser_empty_response", r.error ?? "Browser fetch returned no usable response.")]
+          });
+          pages.push(page);
+          browserRetries++;
+          if (followLinks && isPageExtractionSuccessful(page)) {
+            for (const href of page.discoveryLinks ?? page.outlinks.filter((link) => link.internal).map((link) => link.href)) enqueue(href);
+          }
         }
       }
+      truncated = frontier.length > 0;
+      spider = makeSeedSpider(opts.startUrl, [...seen]);
+    } else {
+      spider = seeded ? makeSeedSpider(opts.startUrl, opts.seedUrls) : await spiderSite({ startUrl: opts.startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey: opts.kernelApiKey });
+      sitemapUrls = spider.sitemapUrls;
+      const urlsToExtract = spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
+      browserRetries = spider.browserRetries;
+      await runWithConcurrency(urlsToExtract, concurrency, async (url) => {
+        const data = await fetchAndParse(url, opts.kernelApiKey);
+        if (data.via === "browser") browserRetries++;
+        pages.push(data);
+      });
+      truncated = spider.truncated || pages.length >= maxPages;
     }
-    truncated = frontier.length > 0;
-    spider = makeSeedSpider(opts.startUrl, [...seen]);
-  } else {
-    spider = seeded ? makeSeedSpider(opts.startUrl, opts.seedUrls) : await spiderSite({ startUrl: opts.startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey: opts.kernelApiKey });
-    sitemapUrls = spider.sitemapUrls;
-    const urlsToExtract = spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
-    browserRetries = spider.browserRetries;
-    await runWithConcurrency(urlsToExtract, concurrency, async (url) => {
-      const data = await fetchAndParse(url, opts.kernelApiKey);
-      if (data.via === "browser") browserRetries++;
-      pages.push(data);
-    });
-    truncated = spider.truncated || pages.length >= maxPages;
+  } catch (err) {
+    if (pages.length === 0) throw err;
+    stoppedEarly = {
+      code: "crawl_stopped_early",
+      message: err instanceof Error ? err.message : String(err)
+    };
+    truncated = true;
   }
+  if (!spider) spider = makeSeedSpider(opts.startUrl, pages.map((p) => p.url));
   const sitemapSet = sitemapUrls.length > 0 ? new Set(sitemapUrls.map((u) => normalizeUrl(u, opts.startUrl) ?? u)) : null;
   for (const p of pages) p.inSitemap = sitemapSet ? sitemapSet.has(normalizeUrl(p.url, opts.startUrl) ?? p.url) : null;
   let branding = null;
@@ -10121,10 +10330,11 @@ async function extractSite(opts) {
   const seoAudit = opts.formats?.includes("issues") ? buildSeoAudit(pages, opts.startUrl) : null;
   let imageAudit = null;
   if (opts.formats?.includes("images")) {
-    imageAudit = await auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
+    imageAudit = await auditImages(pages, { concurrency: 12, timeoutMs: 12e3 }).catch(() => null);
   }
   return {
     startUrl: opts.startUrl,
+    stoppedEarly,
     pages,
     totalPages: pages.length,
     spider,
@@ -10928,11 +11138,14 @@ var init_rates = __esm({
       url_map: 500,
       yt_channel: 200,
       yt_transcription: 200,
-      fb_ad: 200,
+      fb_ad: 700,
       maps_search: 500,
       maps_place: 6e3,
       maps_review: 100,
-      fb_search: 600,
+      fb_search: 800,
+      fb_search_result: 100,
+      fb_page_intel: 700,
+      fb_page_intel_ad: 50,
       fb_transcribe: 33300,
       google_ads_search: 600,
       google_ads_intel: 200,
@@ -11311,6 +11524,12 @@ var init_rates = __esm({
       FB_SEARCH: "fb_search",
       FB_TRANSCRIBE: "fb_transcribe",
       FB_SEARCH_REFUND: "fb_search_refund",
+      FB_SEARCH_RESULT: "fb_search_result",
+      FB_SEARCH_RESULT_REFUND: "fb_search_result_refund",
+      FB_PAGE_INTEL: "fb_page_intel",
+      FB_PAGE_INTEL_REFUND: "fb_page_intel_refund",
+      FB_PAGE_INTEL_AD: "fb_page_intel_ad",
+      FB_PAGE_INTEL_AD_REFUND: "fb_page_intel_ad_refund",
       FB_TRANSCRIBE_REFUND: "fb_transcribe_refund",
       GOOGLE_ADS_SEARCH: "google_ads_search",
       GOOGLE_ADS_SEARCH_REFUND: "google_ads_search_refund",
@@ -16262,8 +16481,12 @@ async function formatExtractSite(raw, input, ctx) {
       schemaTypes: schemaTypesOf(p)
     })));
     const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
+    const stoppedEarly = d.stoppedEarly;
     const summaryHeader = [
       `# Site Extract: ${input.url}`,
+      stoppedEarly ? `
+> **Partial crawl.** The crawl stopped before finishing, so this covers ${pages.length} page(s) rather than the whole site. You were only charged for the pages returned. Reason: ${stoppedEarly.message ?? "unknown"}
+` : "",
       durationLine,
       `
 ## Pages (first ${Math.min(BULK_PAGE_THRESHOLD, pages.length)} of ${pages.length})
@@ -18805,8 +19028,10 @@ var init_BrowserDriver = __esm({
         return this.kernelClient ? "Google returned a CAPTCHA on this session \u2014 retrying with a fresh session." : RECAPTCHA_INSTRUCTIONS;
       }
       async navigateTo(url) {
+        const arrivalBase = process.env.ADS_ARRIVAL_REDIRECT_BASE?.trim();
+        const navUrl = arrivalBase ? serpArrivalUrl(url, arrivalBase) : url;
         try {
-          await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+          await this.page.goto(navUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
           if (/google\.[^/]+\/search\?/i.test(url) && await this.isSerpChallengeVisible()) {
             throw new CaptchaError(this.captchaMessage());
           }
@@ -19022,6 +19247,30 @@ function closeOpenBusinessPanelDialogInBrowser(expectedHeading) {
   close?.click();
   return Boolean(close);
 }
+function readExpandedPanelRowInBrowser(input) {
+  const normalize4 = (value) => (value ?? "").replace(/\s+/g, " ").trim();
+  const expectedBusiness = normalize4(input.businessName).toLowerCase();
+  const visible = (element) => element.getAttribute("aria-hidden") !== "true" && (element.offsetParent !== null || element.getClientRects().length > 0);
+  const readText = (element) => normalize4(element.innerText || element.textContent);
+  const dialogs = Array.from(document.querySelectorAll('dialog, [role="dialog"]')).filter((dialog) => visible(dialog) && readText(dialog).toLowerCase().includes(expectedBusiness)).reverse();
+  for (const dialog of dialogs) {
+    const row = Array.from(
+      dialog.querySelectorAll('button, a, [role="button"]')
+    ).find((element) => visible(element) && readText(element).startsWith(input.prefix));
+    if (!row) continue;
+    const body = readText(row).slice(input.prefix.length);
+    const seen = /* @__PURE__ */ new Set();
+    const items = [];
+    for (const part of body.split(/\s*[,\u00b7]\s*/)) {
+      const value = part.trim();
+      if (!value || seen.has(value)) continue;
+      seen.add(value);
+      items.push(value);
+    }
+    if (items.length > 0) return items;
+  }
+  return [];
+}
 async function clickPanelRowWithRetry(page, businessName, prefix) {
   const input = { businessName, prefix };
   if (await page.evaluate(clickBusinessPanelRowInBrowser, input)) return true;
@@ -19035,9 +19284,12 @@ async function extractExpandedList(page, businessName, rowPrefix, dialogHeading)
     extractOpenBusinessPanelDialogItemsInBrowser,
     dialogHeading
   );
-  await page.evaluate(closeOpenBusinessPanelDialogInBrowser, dialogHeading);
-  await page.waitForTimeout(250);
-  return items;
+  if (items.length > 0) {
+    await page.evaluate(closeOpenBusinessPanelDialogInBrowser, dialogHeading);
+    await page.waitForTimeout(250);
+    return items;
+  }
+  return page.evaluate(readExpandedPanelRowInBrowser, { businessName, prefix: rowPrefix });
 }
 async function extractExpandedGoogleBusinessPanelServices(page, businessName) {
   const services = await extractExpandedList(page, businessName, "Services:", "Services");
@@ -19114,7 +19366,7 @@ var init_MapsSearchExtractor = __esm({
           await page.waitForTimeout(LOCAL_RESULTS_WAIT_MS);
           if (await this.detectBlock(page)) throw new CaptchaError(RECAPTCHA_INSTRUCTIONS);
           const reachedFinder = await this.clickIntoLocalFinder(page);
-          if (!reachedFinder) throw new ExtractionError('Google "More businesses" control was not found on the organic results page');
+          if (!reachedFinder) throw new MapsSearchIncompletePageError('Google "More businesses" control was not found on the organic results page');
           await page.waitForTimeout(LOCAL_RESULTS_WAIT_MS);
           if (await this.detectBlock(page)) throw new CaptchaError(RECAPTCHA_INSTRUCTIONS);
           const results = await this.collectResults(page, options);
@@ -20100,7 +20352,8 @@ function shouldRetryMapsSearchError(input) {
   if (input.attemptIndex >= input.maxAttempts - 1) return false;
   if (!retryableMapsSearchError(input.err, input.proxyMode)) return false;
   if (input.err instanceof MapsSearchIncompletePageError) {
-    return !input.priorAttempts.some((attempt) => attempt.outcome === "results_unconfirmed");
+    const unconfirmed = input.priorAttempts.filter((attempt) => attempt.outcome === "results_unconfirmed").length;
+    return unconfirmed < UNCONFIRMED_PAGE_MAX_RETRIES;
   }
   return true;
 }
@@ -20281,7 +20534,7 @@ async function runMapsSearchWithRotation(options) {
     true
   );
 }
-var MAPS_SEARCH_MAX_ATTEMPTS, MAPS_SEARCH_NONE_MAX_ATTEMPTS, MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS, MapsSearchRotationError;
+var UNCONFIRMED_PAGE_MAX_RETRIES, MAPS_SEARCH_MAX_ATTEMPTS, MAPS_SEARCH_NONE_MAX_ATTEMPTS, MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS, MapsSearchRotationError;
 var init_maps_search_rotation = __esm({
   "src/maps-search-rotation.ts"() {
     "use strict";
@@ -20291,6 +20544,7 @@ var init_maps_search_rotation = __esm({
     init_kernel_proxy_resolver();
     init_backup_proxy();
     init_cost_context();
+    UNCONFIRMED_PAGE_MAX_RETRIES = 5;
     MAPS_SEARCH_MAX_ATTEMPTS = 3;
     MAPS_SEARCH_NONE_MAX_ATTEMPTS = 7;
     MAPS_SEARCH_LOCATION_PROXY_MAX_ATTEMPTS = 5;
@@ -25823,6 +26077,7 @@ async function resolveYouTubeAudio(videoId) {
   if (!base || !secret2) return null;
   const controller = new AbortController();
   const timer = setTimeout(() => controller.abort(), RESOLVE_TIMEOUT_MS);
+  const startedMs = Date.now();
   try {
     const resp = await fetch(`${base.replace(/\/+$/, "")}/resolve-audio`, {
       method: "POST",
@@ -25840,6 +26095,12 @@ async function resolveYouTubeAudio(videoId) {
       console.warn(`[audio-resolver] ${videoId}: response missing mediaUrl`);
       return null;
     }
+    void recordVendorUsage({
+      vendor: "frames_audio_resolve",
+      model: null,
+      units: (Date.now() - startedMs) / 1e3,
+      unitType: "compute_sec"
+    });
     return {
       mediaUrl: data.mediaUrl,
       durationS: Number(data.durationS) || 0,
@@ -25858,6 +26119,7 @@ var RESOLVE_TIMEOUT_MS;
 var init_audio_resolver = __esm({
   "src/youtube/audio-resolver.ts"() {
     "use strict";
+    init_cost_telemetry();
     RESOLVE_TIMEOUT_MS = 24e4;
   }
 });
@@ -27430,10 +27692,20 @@ function isAllowedFacebookOrganicUrl(url) {
 async function transcribeFacebookVideoUrl(videoUrl, markdownTitle = "# Facebook Video Transcript") {
   return transcribeMediaUrl(videoUrl, markdownTitle);
 }
+function classifyAdLibraryPage(bodyText, finalUrl) {
+  if (AUTH_WALL_URL.test(finalUrl)) return "blocked";
+  if (bodyText.length < MIN_RENDERED_BODY_CHARS) return "blocked";
+  return AD_LIBRARY_APP_CHROME.test(bodyText) ? "reached_app" : "blocked";
+}
+async function refundEmptyPageIntel(c, userId, subject, blocked) {
+  await creditMc(userId, MC_COSTS.fb_page_intel, LedgerOperation.FB_PAGE_INTEL_REFUND, blocked ? "soft-block empty result" : "empty result");
+  await logRequestEvent({ userId, source: "facebook_page_intel", status: "failed", query: subject, error: blocked ? "soft-block: empty result refunded" : "empty result refunded" });
+  return blocked ? c.json({ error: "soft-block: no ads returned (refunded)" }, 503) : c.json({ error: `No ads in Meta's Ad Library match "${subject}" (refunded). This is a real empty result, not a block.` }, 404);
+}
 async function detectSoftBlock(driver) {
   const page = driver.getPage();
   const bodyText = await page.evaluate(() => document.body?.innerText ?? "").catch(() => "");
-  return bodyText.length < 200 || /Log in|log in|Create new account|You must log in/.test(bodyText);
+  return classifyAdLibraryPage(bodyText, page.url()) === "blocked";
 }
 function buildPageIntelUrl(body, country) {
   if (body.libraryId?.trim()) return `https://www.facebook.com/ads/library/?id=${FacebookAdExtractor.resolveLibraryId(body.libraryId.trim()) ?? body.libraryId.trim()}`;
@@ -27509,7 +27781,7 @@ async function kernelLaunchOptsResidential() {
   }
   return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
 }
-var import_hono7, import_zod21, import_client5, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, facebookAdApp, ALLOWED_MEDIA_HOSTS;
+var import_hono7, import_zod21, import_client5, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, AD_LIBRARY_APP_CHROME, AUTH_WALL_URL, MIN_RENDERED_BODY_CHARS, facebookAdApp, ALLOWED_MEDIA_HOSTS;
 var init_facebook_ad_routes = __esm({
   "src/api/facebook-ad-routes.ts"() {
     "use strict";
@@ -27559,6 +27831,9 @@ var init_facebook_ad_routes = __esm({
       url: import_zod21.z.string().trim().min(1, "url is required"),
       filename: import_zod21.z.string().trim().optional()
     });
+    AD_LIBRARY_APP_CHROME = /Ad Library|Search by keyword|Branded Content/i;
+    AUTH_WALL_URL = /facebook\.com\/(login|checkpoint|authentication)/i;
+    MIN_RENDERED_BODY_CHARS = 200;
     facebookAdApp = new import_hono7.Hono();
     facebookAdApp.post("/ad", createApiKeyAuth(), async (c) => {
       const raw = await c.req.json().catch(() => ({}));
@@ -27619,26 +27894,27 @@ var init_facebook_ad_routes = __esm({
       const driver = new BrowserDriver();
       let refunded = false, debited = false;
       try {
-        const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD, subject);
-        if (!fbOk) return c.json(insufficientBalanceResponse(fbBal, MC_COSTS.fb_ad), 402);
+        const { ok: fbOk, balance_mc: fbBal } = await debitMc(fbUser.id, MC_COSTS.fb_page_intel, LedgerOperation.FB_PAGE_INTEL, subject);
+        if (!fbOk) return c.json(insufficientBalanceResponse(fbBal, MC_COSTS.fb_page_intel), 402);
         debited = true;
         await driver.launch(await kernelLaunchOptsResidential());
         await driver.navigateTo(listingUrl);
         const extractor = new FacebookAdExtractor(driver);
         const result = await extractor.extractPageIntel(listingUrl, maxAds);
-        if (result.ads.length === 0 && await detectSoftBlock(driver)) {
-          await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "soft-block empty result");
+        if (result.ads.length === 0) {
           refunded = true;
-          await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: subject, error: "soft-block: empty result refunded" });
-          return c.json({ error: "soft-block: no ads returned (refunded)" }, 503);
+          return await refundEmptyPageIntel(c, fbUser.id, subject, await detectSoftBlock(driver));
         }
+        const perAdMc = result.ads.length * MC_COSTS.fb_page_intel_ad;
+        const adDebit = await debitMc(fbUser.id, perAdMc, LedgerOperation.FB_PAGE_INTEL_AD, subject);
+        if (!adDebit.ok) return c.json(insufficientBalanceResponse(adDebit.balance_mc, perAdMc), 402);
         const match = assessPageIntelMatch(body, result);
         const output = { ...result, ...match };
         await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "done", query: subject, resultCount: result.ads.length, result: output });
         return c.json(output);
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
-        if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_ad, LedgerOperation.FB_AD_REFUND, "failed call");
+        if (debited && !refunded) await creditMc(fbUser.id, MC_COSTS.fb_page_intel, LedgerOperation.FB_PAGE_INTEL_REFUND, "failed call");
         await logRequestEvent({ userId: fbUser.id, source: "facebook_page_intel", status: "failed", query: subject, error: msg });
         if (msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha")) {
           return c.json({ error: msg }, 503);
@@ -27779,10 +28055,16 @@ var init_facebook_ad_routes = __esm({
         debited = true;
         await driver.launch(await kernelLaunchOptsResidential());
         const page = driver.getPage();
-        const collated = await collectAdLibraryResults(page, searchUrl, Math.max(maxResults * 4, 40));
+        let collated = await collectAdLibraryResults(page, searchUrl, Math.max(maxResults * 4, 40));
+        if (collated.length === 0) {
+          collated = await collectAdLibraryResults(page, searchUrl, Math.max(maxResults * 4, 40));
+        }
         const gqlAdvertisers = advertisersFromResults(collated, maxResults);
         if (gqlAdvertisers.length > 0) {
           const results2 = gqlAdvertisers.map((a) => ({ name: a.pageName, pageName: a.pageName, pageId: a.pageId, libraryId: a.sampleLibraryId, sampleLibraryId: a.sampleLibraryId, adCount: a.adCount }));
+          const gqlPerResultMc = results2.length * MC_COSTS.fb_search_result;
+          const gqlResultDebit = await debitMc(fbUser.id, gqlPerResultMc, LedgerOperation.FB_SEARCH_RESULT, body.query.trim());
+          if (!gqlResultDebit.ok) return c.json(insufficientBalanceResponse(gqlResultDebit.balance_mc, gqlPerResultMc), 402);
           const searchResult2 = { query: body.query.trim(), searchUrl, results: results2, via: "graphql" };
           await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "done", query: body.query.trim(), resultCount: results2.length, result: searchResult2 });
           return c.json(searchResult2);
@@ -27795,7 +28077,7 @@ var init_facebook_ad_routes = __esm({
           await page.waitForTimeout(1e3);
         }
         const rawBodyText = await page.evaluate(() => document.body?.innerText ?? "");
-        if (rawBodyText.length < 200 || /Log in|You must log in|Create new account/.test(rawBodyText)) {
+        if (classifyAdLibraryPage(rawBodyText, page.url()) === "blocked") {
           await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "soft-block empty result");
           searchRefunded = true;
           await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "failed", query: body.query.trim(), error: "soft-block: empty result refunded" });
@@ -27836,8 +28118,11 @@ var init_facebook_ad_routes = __esm({
           await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "empty result");
           searchRefunded = true;
           await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "failed", query: body.query.trim(), error: "empty result refunded" });
-          return c.json({ error: "no advertisers found (refunded)" }, 503);
+          return c.json({ error: `No advertisers in Meta's Ad Library match "${body.query.trim()}" (refunded). This is a real empty result, not a block \u2014 try a shorter or broader query, or a brand/page name.` }, 404);
         }
+        const perResultMc = results.length * MC_COSTS.fb_search_result;
+        const resultDebit = await debitMc(fbUser.id, perResultMc, LedgerOperation.FB_SEARCH_RESULT, body.query.trim());
+        if (!resultDebit.ok) return c.json(insufficientBalanceResponse(resultDebit.balance_mc, perResultMc), 402);
         const searchResult = { query: body.query.trim(), searchUrl, results };
         await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "done", query: body.query.trim(), resultCount: results.length, result: searchResult });
         return c.json(searchResult);
@@ -27903,9 +28188,6 @@ var init_facebook_ad_routes = __esm({
 });
 
 // src/extractor/GoogleAdsExtractor.ts
-function looksLikeDomain(q) {
-  return /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(q.trim()) && !q.includes(" ");
-}
 var ORIGIN, GoogleAdsExtractor;
 var init_GoogleAdsExtractor = __esm({
   "src/extractor/GoogleAdsExtractor.ts"() {
@@ -27921,50 +28203,108 @@ var init_GoogleAdsExtractor = __esm({
       }
       async search(query, region, maxResults) {
         const q = query.trim();
-        const url = looksLikeDomain(q) ? `${ORIGIN}/?region=${encodeURIComponent(region)}&domain=${encodeURIComponent(q)}` : `${ORIGIN}/?region=${encodeURIComponent(region)}&query=${encodeURIComponent(q)}`;
-        await this.driver.navigateTo(url);
+        await this.driver.navigateTo(`${ORIGIN}/?region=${encodeURIComponent(region)}`);
         const page = this.page();
-        await page.waitForTimeout(3500);
-        await this.autoScroll(page, 6);
-        const advertisers = await page.evaluate((max) => {
-          const map = /* @__PURE__ */ new Map();
-          const links = Array.from(document.querySelectorAll('a[href*="/advertiser/"]'));
-          for (const a of links) {
-            const m = a.getAttribute("href")?.match(/\/advertiser\/(AR[0-9]+)/);
-            if (!m) continue;
-            const id = m[1];
-            const card = a.closest("creative-preview, advertiser-preview") || a.parentElement;
-            const nameEl = card?.querySelector('[class*="advertiser"], [class*="name"]');
-            const name = (nameEl?.innerText || "").trim() || null;
-            const cur = map.get(id);
-            if (cur) cur.count++;
-            else map.set(id, { advertiserId: id, name, count: 1 });
-          }
-          return Array.from(map.values()).slice(0, max).map((v) => ({
-            advertiserId: v.advertiserId,
-            name: v.name,
-            approxAdCount: v.count
-          }));
-        }, maxResults);
+        await page.waitForTimeout(2500);
+        const box = await page.waitForSelector('input[type="text"], input[aria-label*="Search" i], input', { timeout: 2e4 });
+        await box.click();
+        await box.fill("");
+        await box.type(q, { delay: 110 });
+        await page.waitForSelector('[role="option"]', { timeout: 2e4 }).catch(() => null);
+        await page.waitForTimeout(2200);
+        const raw = await page.evaluate(() => {
+          const rows = Array.from(document.querySelectorAll('[role="option"]')).map((el2) => {
+            const parts = Array.from(el2.querySelectorAll("*")).filter((n) => n.children.length === 0).map((n) => (n.textContent ?? "").replace(/\s+/g, " ").trim()).filter(Boolean);
+            const flat = el2.innerText.replace(/\s+/g, " ").trim();
+            return { parts: parts.length ? parts : [flat], flat };
+          }).filter((r) => r.flat);
+          return { rows };
+        });
+        const domainRe = /^[a-z0-9][a-z0-9-]*(\.[a-z0-9-]+)+$/i;
+        const websiteRows = raw.rows.filter((r) => domainRe.test(r.flat));
+        const advertiserRows = raw.rows.filter((r) => !domainRe.test(r.flat));
+        const advertisers = advertiserRows.slice(0, maxResults).map((row) => {
+          const segments = row.parts.filter((p) => !/^Verified$/i.test(p));
+          const adSeg = segments.find((p) => /~\s*[\d,]+\s*ads?/i.test(p));
+          const adCount = adSeg?.match(/~\s*([\d,]+)\s*ads?/i);
+          const rest = segments.filter((p) => p !== adSeg);
+          return {
+            advertiserId: null,
+            name: rest[0] ?? null,
+            basedIn: rest.length > 1 ? rest[rest.length - 1] : null,
+            domain: null,
+            approxAdCount: adCount ? Number(adCount[1].replace(/,/g, "")) : null,
+            detailUrl: null
+          };
+        });
+        const websites = websiteRows.slice(0, maxResults).map((row) => ({
+          domain: row.flat,
+          detailUrl: `${ORIGIN}/?region=${encodeURIComponent(region)}&domain=${encodeURIComponent(row.flat)}`
+        }));
         return {
           query: q,
           region,
           advertiserCount: advertisers.length,
-          advertisers: advertisers.map((a) => ({
-            advertiserId: a.advertiserId,
-            name: a.name,
-            domain: looksLikeDomain(q) ? q : null,
-            approxAdCount: a.approxAdCount,
-            detailUrl: `${ORIGIN}/advertiser/${a.advertiserId}?region=${region}`
-          }))
+          advertisers,
+          websiteCount: websites.length,
+          websites
         };
       }
+      async advertiserIdOnPage() {
+        const page = this.page();
+        const fromUrl = page.url().match(/\/advertiser\/(AR[0-9]+)/);
+        if (fromUrl) return fromUrl[1];
+        return await page.evaluate(() => {
+          for (const a of Array.from(document.querySelectorAll('a[href*="/advertiser/"]'))) {
+            const m = a.getAttribute("href")?.match(/\/advertiser\/(AR[0-9]+)/);
+            if (m) return m[1];
+          }
+          return null;
+        });
+      }
+      async resolveAdvertiserIdByDomain(domain, region) {
+        await this.driver.navigateTo(`${ORIGIN}/?region=${encodeURIComponent(region)}&domain=${encodeURIComponent(domain.trim())}`);
+        await this.page().waitForTimeout(3500);
+        return await this.advertiserIdOnPage();
+      }
+      async resolveAdvertiserIdByTypeahead(term, region) {
+        const page = this.page();
+        await this.driver.navigateTo(`${ORIGIN}/?region=${encodeURIComponent(region)}`);
+        await page.waitForTimeout(2500);
+        const box = await page.waitForSelector('input[type="text"], input[aria-label*="Search" i], input', { timeout: 2e4 });
+        await box.click();
+        await box.fill("");
+        await box.type(term.trim(), { delay: 110 });
+        await page.waitForSelector('[role="option"]', { timeout: 2e4 }).catch(() => null);
+        await page.waitForTimeout(2e3);
+        const optionCount = (await page.$$('[role="option"]')).length;
+        for (let i = 0; i < Math.min(optionCount, 3); i++) {
+          const options = await page.$$('[role="option"]');
+          if (!options[i]) break;
+          await options[i].click().catch(() => null);
+          await page.waitForTimeout(3e3);
+          const found = await this.advertiserIdOnPage();
+          if (found) return found;
+          await this.driver.navigateTo(`${ORIGIN}/?region=${encodeURIComponent(region)}`);
+          await page.waitForTimeout(1500);
+          const retryBox = await page.waitForSelector('input[type="text"], input', { timeout: 15e3 }).catch(() => null);
+          if (!retryBox) break;
+          await retryBox.click();
+          await retryBox.fill("");
+          await retryBox.type(term.trim(), { delay: 110 });
+          await page.waitForSelector('[role="option"]', { timeout: 15e3 }).catch(() => null);
+          await page.waitForTimeout(1800);
+        }
+        return null;
+      }
       async pageIntel(opts) {
         const { region, maxAds } = opts;
         let advertiserId = opts.advertiserId?.trim() || null;
-        if (!advertiserId && opts.domain) {
-          const s = await this.search(opts.domain, region, 1);
-          advertiserId = s.advertisers[0]?.advertiserId ?? null;
+        if (!advertiserId && opts.domain?.trim()) {
+          advertiserId = await this.resolveAdvertiserIdByDomain(opts.domain, region);
+        }
+        if (!advertiserId && opts.advertiserName?.trim()) {
+          advertiserId = await this.resolveAdvertiserIdByTypeahead(opts.advertiserName, region);
         }
         if (!advertiserId) {
           return { advertiserId: null, advertiserName: null, domain: opts.domain ?? null, region, totalCreatives: 0, videoCount: 0, imageCount: 0, textCount: 0, ads: [] };
@@ -28125,10 +28465,11 @@ var init_google_ads_routes = __esm({
     });
     GoogleAdsPageIntelBodySchema = import_zod22.z.object({
       advertiserId: import_zod22.z.string().trim().optional(),
+      advertiserName: import_zod22.z.string().trim().optional(),
       domain: import_zod22.z.string().trim().optional(),
       region: import_zod22.z.string().trim().toUpperCase().length(2).optional(),
       maxAds: import_zod22.z.number().int().min(1).max(200).optional()
-    }).refine((d) => !!d.advertiserId || !!d.domain, { message: "advertiserId or domain is required" });
+    }).refine((d) => !!d.advertiserId || !!d.domain || !!d.advertiserName, { message: "advertiserId, advertiserName, or domain is required (advertiserName and domain both come straight from google_ads_search)" });
     GoogleAdsTranscribeBodySchema = import_zod22.z.object({
       videoUrl: import_zod22.z.string().trim().min(1, "videoUrl is required")
     });
@@ -28154,13 +28495,13 @@ var init_google_ads_routes = __esm({
           const extractor = new GoogleAdsExtractor(driver);
           return extractor.search(body.query.trim(), region, maxResults);
         });
-        if (result.advertisers.length === 0) {
+        if (result.advertisers.length === 0 && result.websites.length === 0) {
           await creditMc(user.id, MC_COSTS.google_ads_search, LedgerOperation.GOOGLE_ADS_SEARCH_REFUND, "empty result");
           refunded = true;
           await logRequestEvent({ userId: user.id, source: "google_ads_search", status: "failed", query: body.query.trim(), error: "empty result refunded" });
           return c.json({ error: "no advertisers found (refunded)" }, 503);
         }
-        await logRequestEvent({ userId: user.id, source: "google_ads_search", status: "done", query: body.query.trim(), resultCount: result.advertisers.length, result });
+        await logRequestEvent({ userId: user.id, source: "google_ads_search", status: "done", query: body.query.trim(), resultCount: result.advertisers.length + result.websites.length, result });
         return c.json(result);
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
@@ -28179,7 +28520,7 @@ var init_google_ads_routes = __esm({
       const body = parsed.data;
       const region = body.region ?? "US";
       const maxAds = Math.min(200, Math.max(1, body.maxAds ?? 50));
-      const queryLabel = body.advertiserId ?? body.domain ?? "";
+      const queryLabel = body.advertiserId ?? body.domain ?? body.advertiserName ?? "";
       const user = c.get("user");
       const gate = await acquireConcurrencyGate(user, "google_ads_page_intel", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { advertiserId: body.advertiserId, domain: body.domain, region } });
       if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
@@ -28192,7 +28533,7 @@ var init_google_ads_routes = __esm({
         const result = await runWithCostContext({ ...currentCostContext(), op: "google_ads_intel", userId: user.id }, async () => {
           await driver.launch(await kernelLaunchOptsResidential2());
           const extractor = new GoogleAdsExtractor(driver);
-          return extractor.pageIntel({ advertiserId: body.advertiserId, domain: body.domain, region, maxAds });
+          return extractor.pageIntel({ advertiserId: body.advertiserId, advertiserName: body.advertiserName, domain: body.domain, region, maxAds });
         });
         if (result.ads.length === 0) {
           await creditMc(user.id, MC_COSTS.google_ads_intel, LedgerOperation.GOOGLE_ADS_INTEL_REFUND, "empty result");
@@ -29140,6 +29481,7 @@ var init_instagram_routes = __esm({
     init_db();
     init_rates();
     init_concurrency_gates();
+    init_cost_context();
     init_blob_store();
     init_InstagramContentExtractor();
     init_media_transcription();
@@ -29203,6 +29545,12 @@ var init_instagram_routes = __esm({
           scrollDelayMs: body.scrollDelayMs,
           stableScrollLimit: body.stableScrollLimit
         });
+        if (result.items.length === 0) {
+          await creditMc(user.id, MC_COSTS.instagram_profile, LedgerOperation.INSTAGRAM_PROFILE_REFUND, "empty result");
+          debited = false;
+          await logRequestEvent({ userId: user.id, source: "instagram_profile_content", status: "failed", query: target.profileUrl, error: "empty result refunded" });
+          return c.json({ error: `No public posts were readable on ${target.profileUrl} (refunded). The profile may be private, empty, or require a logged-in session.` }, 404);
+        }
         const response = { ...result, browser: launch.browser };
         await logRequestEvent({ userId: user.id, source: "instagram_profile_content", status: "done", query: target.profileUrl, resultCount: result.items.length, result: response });
         return c.json(response);
@@ -29326,7 +29674,10 @@ var init_instagram_routes = __esm({
             const tx = await debitMc(user.id, holdMc, LedgerOperation.INSTAGRAM_TRANSCRIBE, sourceUrl.href);
             if (!tx.ok) return c.json(insufficientBalanceResponse(tx.balance_mc, holdMc), 402);
             transcriptDebited = true;
-            transcript = await transcribeMediaUrl(audioUrl, "# Instagram Media Transcript");
+            transcript = await runWithCostContext(
+              { ...currentCostContext(), op: "instagram_transcribe", userId: user.id },
+              () => transcribeMediaUrl(audioUrl, "# Instagram Media Transcript")
+            );
             const settleMc = mediaTranscribeChargeMc(transcript.durationMs / 1e3);
             const diff = holdMc - settleMc;
             if (diff > 0) await creditMc(user.id, diff, LedgerOperation.INSTAGRAM_TRANSCRIBE_REFUND, "real-cost reconcile");
@@ -30028,6 +30379,8 @@ var init_PAAExtractor = __esm({
         aioShareCaptured: null
       };
       aioShareUrlEarly = null;
+      collectedRows = [];
+      partialContext = null;
       normalizeQuestion(q) {
         return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
       }
@@ -30132,11 +30485,50 @@ var init_PAAExtractor = __esm({
           extracted_at: (/* @__PURE__ */ new Date()).toISOString()
         };
       }
+      getPartialResult() {
+        const ctx = this.partialContext;
+        if (!ctx || this.collectedRows.length === 0) return null;
+        const flat = [...this.collectedRows];
+        const stats = {
+          seed: ctx.seed,
+          totalQuestions: flat.length,
+          maxDepthReached: flat.reduce((m, r) => Math.max(m, r.depth), 0),
+          durationMs: Date.now() - ctx.startMs,
+          errorCount: 0
+        };
+        return {
+          seed: ctx.seed,
+          location: ctx.location,
+          extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          diagnostics: {
+            completionStatus: "paa_found",
+            problem: null,
+            completeness: { ...this.completeness },
+            warnings: [{
+              code: "paa_partial",
+              surface: "paa",
+              message: `Harvest stopped before finishing; returning the ${flat.length} question(s) collected up to that point.`,
+              retryable: true
+            }]
+          },
+          totalQuestions: flat.length,
+          whatPeopleSaying: [],
+          tree: this.buildTree(flat, ctx.seed),
+          flat,
+          videos: [],
+          forums: [],
+          organicResults: [],
+          localPack: [],
+          entityIds: {},
+          stats
+        };
+      }
       async runBFS(page, options, signal) {
         const seenKeys = /* @__PURE__ */ new Set();
         const seenQs = /* @__PURE__ */ new Set();
         const orderedQs = [];
-        const results = [];
+        this.collectedRows = [];
+        const results = this.collectedRows;
         const abandonedNeverReclick = /* @__PURE__ */ new Set();
         const clickedOnceEver = /* @__PURE__ */ new Set();
         const itemArgs = {
@@ -30748,6 +31140,8 @@ var init_PAAExtractor = __esm({
         const startMs = Date.now();
         this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
         this.aioShareUrlEarly = null;
+        this.collectedRows = [];
+        this.partialContext = { seed: options.query, location: options.location ?? null, startMs };
         const isMobile = options.device === "mobile";
         const config = {
           headless: options.headless,
@@ -31295,7 +31689,23 @@ async function extractOnce2(options, signal) {
     } : null);
     cleanup = await driver.close();
   }
-  return error ? { result: null, error, cleanup, debug } : { result, error: null, cleanup, debug };
+  if (error) {
+    const outcome = classifyAttemptError(error);
+    const budgetExhausted = outcome === "timeout" || outcome === "request_aborted";
+    let salvaged = null;
+    if (budgetExhausted) {
+      try {
+        salvaged = typeof extractor.getPartialResult === "function" ? extractor.getPartialResult() : null;
+      } catch {
+        salvaged = null;
+      }
+    }
+    if (salvaged) {
+      return { result: salvaged, error: null, cleanup, debug, salvagedFrom: error };
+    }
+    return { result: null, error, cleanup, debug };
+  }
+  return { result, error: null, cleanup, debug };
 }
 async function harvest(rawOptions) {
   const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};
@@ -32913,6 +33323,7 @@ var init_trustpilot_routes = __esm({
         const driver = new BrowserDriver();
         const allReviews = [];
         let pagesFetched = 0;
+        let stoppedEarly = null;
         try {
           await driver.launch({
             headless: false,
@@ -32930,6 +33341,12 @@ var init_trustpilot_routes = __esm({
             if (cards.length === 0) break;
             allReviews.push(...cards);
           }
+        } catch (err) {
+          if (allReviews.length === 0) throw err;
+          stoppedEarly = {
+            code: "review_scrape_stopped_early",
+            message: err instanceof Error ? err.message : String(err)
+          };
         } finally {
           await driver.close();
         }
@@ -32955,6 +33372,7 @@ var init_trustpilot_routes = __esm({
           pagesFetched,
           reviewCount: allReviews.length,
           reviews: allReviews,
+          stoppedEarly,
           durationMs: Date.now() - startMs
         };
         await logRequestEvent({ userId: user.id, source: "trustpilot_reviews", status: "done", query: domain, resultCount: allReviews.length, result });
@@ -37500,7 +37918,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.38.1";
+    PACKAGE_VERSION = "0.38.2";
   }
 });
 
@@ -37997,9 +38415,9 @@ var init_mcp_tool_schemas = __esm({
           const hostname = parsed.hostname.replace(/^\[|\]$/g, "");
           const looksLikeIpv4 = /^\d{1,3}(?:\.\d{1,3}){3}$/.test(hostname);
           const looksLikeIpv6 = hostname.includes(":");
-          const looksLikeDomain2 = hostname.includes(".");
+          const looksLikeDomain = hostname.includes(".");
           const looksLikeLocalhost = hostname === "localhost";
-          if (!looksLikeIpv4 && !looksLikeIpv6 && !looksLikeDomain2 && !looksLikeLocalhost) {
+          if (!looksLikeIpv4 && !looksLikeIpv6 && !looksLikeDomain && !looksLikeLocalhost) {
             ctx.addIssue({ code: import_zod38.z.ZodIssueCode.custom, message: WEBSITE_URL_OR_DOMAIN_ERROR });
             return import_zod38.z.NEVER;
           }
@@ -38135,8 +38553,9 @@ var init_mcp_tool_schemas = __esm({
       maxResults: import_zod38.z.number().int().min(1).max(20).default(10).describe("Maximum advertisers to return. Default 10, maximum 20.")
     };
     GoogleAdsPageIntelInputSchema = {
-      advertiserId: import_zod38.z.string().optional().describe("Google Ads Transparency advertiser ID (starts with AR...). Use one returned by google_ads_search; do not construct one yourself."),
-      domain: import_zod38.z.string().optional().describe("A domain (e.g. getviktor.com) whose primary advertiser to inspect when advertiserId is unknown. One of advertiserId or domain is required."),
+      domain: import_zod38.z.string().optional().describe("A website from google_ads_search.websites (e.g. getviktor.com). Preferred input: people know the website, not the advertiser's registered name. Resolves to whichever advertiser account runs ads for that domain."),
+      advertiserName: import_zod38.z.string().optional().describe('An advertiser name exactly as returned in google_ads_search.advertisers[].name (e.g. "PPS Plumbing Services"). Use when the user picked an advertiser rather than a website.'),
+      advertiserId: import_zod38.z.string().optional().describe("Google Ads Transparency advertiser ID (starts with AR...). Only available if you already have one from a prior page-intel result \u2014 google_ads_search does NOT return advertiser IDs. Do not construct one yourself."),
       region: import_zod38.z.string().length(2).default("US").describe("Two-letter region code for where the ads are shown. Default US."),
       maxAds: import_zod38.z.number().int().min(1).max(200).default(50).describe("Maximum creatives to inspect and hydrate. Default 50, maximum 200. Prefer 25-50 for focused scans.")
     };
@@ -38591,6 +39010,10 @@ var init_mcp_tool_schemas = __esm({
     };
     ExtractSiteOutputSchema = {
       url: import_zod38.z.string(),
+      stoppedEarly: import_zod38.z.object({
+        code: import_zod38.z.string(),
+        message: import_zod38.z.string()
+      }).nullish().describe("Present when the crawl stopped before finishing. The returned pages are everything that was captured, and billing covers only those pages - treat the result as partial, not as the whole site."),
       pageCount: import_zod38.z.number().int().min(0).optional().describe("Absent when background is true \u2014 the crawl has not finished yet."),
       pages: import_zod38.z.array(import_zod38.z.object({
         url: import_zod38.z.string(),
@@ -38970,9 +39393,15 @@ var init_mcp_tool_schemas = __esm({
       advertisers: import_zod38.z.array(import_zod38.z.object({
         advertiserId: NullableString,
         name: NullableString,
+        basedIn: NullableString,
         domain: NullableString,
         approxAdCount: import_zod38.z.number().int().nullable(),
         detailUrl: NullableString
+      })),
+      websiteCount: import_zod38.z.number().int().min(0),
+      websites: import_zod38.z.array(import_zod38.z.object({
+        domain: import_zod38.z.string(),
+        detailUrl: import_zod38.z.string()
       }))
     };
     GoogleAdsPageIntelOutputSchema = {
@@ -40289,14 +40718,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
   }, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
   server.registerTool("google_ads_search", {
     title: "Google Ads Transparency Search",
-    description: "Search the Google Ads Transparency Center to find advertisers by domain or brand name. Returns advertisers with advertiser ID and approximate ad count, to pass to google_ads_page_intel.",
+    description: "Search the Google Ads Transparency Center by brand, person, or website. Returns two lists: advertisers (name, country, approximate ad count) and websites (domains). Neither carries an advertiser ID. Pass a websites[].domain (preferred - users know the website, not the registered advertiser name) or an advertisers[].name to google_ads_page_intel to pull that advertiser's actual ads.",
     inputSchema: GoogleAdsSearchInputSchema,
     outputSchema: recordOutputSchema("google_ads_search", GoogleAdsSearchOutputSchema),
     annotations: liveWebToolAnnotations("Google Ads Transparency Search")
   }, async (input) => formatGoogleAdsSearch(await executor.googleAdsSearch(input), input));
   server.registerTool("google_ads_page_intel", {
     title: "Google Ads Advertiser Intel",
-    description: "Harvest an advertiser's ad creatives from the Google Ads Transparency Center: format, image URLs, and \u2014 for video ads \u2014 a YouTube video ID or direct video URL. Accepts advertiserId (from google_ads_search) or a domain.",
+    description: "Harvest an advertiser's ad creatives from the Google Ads Transparency Center: format, image URLs, and \u2014 for video ads \u2014 a YouTube video ID or direct video URL. Accepts a domain or advertiserName from google_ads_search (google_ads_search does not return advertiser IDs). A domain may map to several advertiser accounts; this returns the primary one.",
     inputSchema: GoogleAdsPageIntelInputSchema,
     outputSchema: recordOutputSchema("google_ads_page_intel", GoogleAdsPageIntelOutputSchema),
     annotations: liveWebToolAnnotations("Google Ads Advertiser Intel")
@@ -48394,6 +48823,106 @@ async function cancelStandaloneMemorySub(memSubId, baseSubId) {
     console.error("[stripe] could not wind down standalone memory subscription", memSubId, err instanceof Error ? err.message : err);
   }
 }
+async function processStripeEvent(event) {
+  if (event.type === "invoice.paid" || event.type === "invoice.payment_succeeded") {
+    const invoice = event.data.object;
+    const memLineId = invoice.lines.data.map(linePriceId).find((id) => id && id in MEMORY_PLANS);
+    if (memLineId) return;
+    const invoiceHasTierLine = invoice.lines.data.some((l) => {
+      const id = linePriceId(l);
+      return id && id in SUBSCRIPTION_TIERS;
+    });
+    if (shouldGrantScraperSubscriptionCredits(invoiceHasTierLine, invoice.total)) {
+      const subId = invoice.subscription;
+      const liveBasePlanPriceId = subId ? findBasePlanItem(await stripe.subscriptions.retrieve(subId))?.price?.id : void 0;
+      const tierPriceId = liveBasePlanPriceId ?? invoice.lines.data.map(linePriceId).find((id) => id && id in SUBSCRIPTION_TIERS);
+      const tier = tierPriceId ? SUBSCRIPTION_TIERS[tierPriceId] : void 0;
+      if (tier) {
+        const user = await resolveUser3(invoice.customer, invoice.customer_email ?? void 0);
+        if (user && invoice.id && !await ledgerExistsForStripePI(invoice.id)) {
+          await creditMc(user.id, tier.credits_mc, LedgerOperation.SUBSCRIPTION, `${tier.label} subscription credits`, invoice.id, { neverExpires: tier.credits_never_expire });
+          await setSubscriptionTier(user.id, tier.tier, tier.concurrency, subId ?? user.subscription_id);
+          if (tier.includes_memory) {
+            await cancelStandaloneMemorySub(user.memory_subscription_id, subId ?? user.subscription_id);
+            await setMemoryPlan(user.id, "pro", null);
+            await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: "pro", subscription_tier: tier.tier }));
+          }
+          const credentials = await syncScheduledActionCredentials(user);
+          if (!credentials.ok) console.warn("[stripe] scheduled-action credential sync failed:", credentials.error ?? "unknown error");
+        }
+      }
+    }
+  }
+  if (event.type === "customer.subscription.created" || event.type === "customer.subscription.updated") {
+    const sub = event.data.object;
+    const priceId = findBasePlanItem(sub)?.price?.id ?? sub.items.data[0]?.price?.id;
+    if (priceId && priceId in MEMORY_PLANS) {
+      const mem = MEMORY_PLANS[priceId];
+      const user2 = await resolveUser3(sub.customer, sub.customer_email);
+      if (user2) {
+        const live = sub.status === "active" || sub.status === "trialing";
+        const newMemoryPlan = live ? mem.plan : "free";
+        await setMemoryPlan(user2.id, newMemoryPlan, live ? sub.id : null);
+        await syncMemoryKeyPlan(user2, resolveEffectiveMemoryPlan({ ...user2, memory_plan: newMemoryPlan }));
+      }
+      return;
+    }
+    const user = await getUserByStripeCustomerId(sub.customer);
+    if (!user) return;
+    if (priceId && priceId in SUBSCRIPTION_TIERS) {
+      const tier = SUBSCRIPTION_TIERS[priceId];
+      const live = sub.status === "active" || sub.status === "trialing";
+      const newTier = live ? tier.tier : null;
+      await setSubscriptionTier(user.id, newTier, live ? tier.concurrency : 0, live ? sub.id : null);
+      if (tier.includes_memory) {
+        const newMemoryPlan = live ? "pro" : "free";
+        if (live) await cancelStandaloneMemorySub(user.memory_subscription_id, sub.id);
+        await setMemoryPlan(user.id, newMemoryPlan, live ? sub.id : null);
+        await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: newMemoryPlan }));
+      } else if (user.memory_subscription_id === sub.id) {
+        await setMemoryPlan(user.id, "free", null);
+        await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: "free" }));
+      }
+      if (live) {
+        const credentials = await syncScheduledActionCredentials(user);
+        if (!credentials.ok) console.warn("[stripe] scheduled-action credential sync failed:", credentials.error ?? "unknown error");
+      }
+    } else if (priceId === CONCURRENCY_PRICE_ID && event.type === "customer.subscription.created") {
+      await setExtraConcurrencySlots(user.id, 1);
+      await setConcurrencySubId(user.id, sub.id);
+    }
+  }
+  if (event.type === "customer.subscription.deleted") {
+    const sub = event.data.object;
+    const priceId = findBasePlanItem(sub)?.price?.id ?? sub.items.data[0]?.price?.id;
+    if (priceId && priceId in SUBSCRIPTION_TIERS) {
+      const tier = SUBSCRIPTION_TIERS[priceId];
+      const user2 = await getUserByStripeCustomerId(sub.customer);
+      if (user2) {
+        await setSubscriptionTier(user2.id, null, 0, null);
+        if (tier.includes_memory || user2.memory_subscription_id === sub.id) {
+          await setMemoryPlan(user2.id, "free", null);
+          await syncMemoryKeyPlan(user2, resolveEffectiveMemoryPlan({ ...user2, memory_plan: "free" }));
+        }
+      }
+      return;
+    }
+    if (priceId && priceId in MEMORY_PLANS || await resolveUserByMemorySubscription(sub.id)) {
+      const user2 = await resolveUserByMemorySubscription(sub.id) ?? await getUserByStripeCustomerId(sub.customer);
+      if (user2) {
+        await setMemoryPlan(user2.id, "free", null);
+        await syncMemoryKeyPlan(user2, resolveEffectiveMemoryPlan({ ...user2, memory_plan: "free" }));
+      }
+      return;
+    }
+    const user = await getUserByStripeCustomerId(sub.customer);
+    if (!user) return;
+    if (priceId === CONCURRENCY_PRICE_ID) {
+      await setExtraConcurrencySlots(user.id, 0);
+      await setConcurrencySubId(user.id, null);
+    }
+  }
+}
 function linePriceId(line) {
   const l = line;
   return l?.price?.id ?? l?.pricing?.price_details?.price ?? l?.plan?.id;
@@ -48428,107 +48957,19 @@ var init_stripe_routes = __esm({
       } catch {
         return c.json({ error: "Invalid signature" }, 400);
       }
-      const isNew = await recordStripeEvent(event.id);
-      if (!isNew) return c.json({ received: true });
-      if (event.type === "invoice.paid" || event.type === "invoice.payment_succeeded") {
-        const invoice = event.data.object;
-        const memLineId = invoice.lines.data.map(linePriceId).find((id) => id && id in MEMORY_PLANS);
-        if (memLineId) return c.json({ received: true });
-        const invoiceHasTierLine = invoice.lines.data.some((l) => {
-          const id = linePriceId(l);
-          return id && id in SUBSCRIPTION_TIERS;
-        });
-        if (shouldGrantScraperSubscriptionCredits(invoiceHasTierLine, invoice.total)) {
-          const subId = invoice.subscription;
-          const liveBasePlanPriceId = subId ? findBasePlanItem(await stripe.subscriptions.retrieve(subId))?.price?.id : void 0;
-          const tierPriceId = liveBasePlanPriceId ?? invoice.lines.data.map(linePriceId).find((id) => id && id in SUBSCRIPTION_TIERS);
-          const tier = tierPriceId ? SUBSCRIPTION_TIERS[tierPriceId] : void 0;
-          if (tier) {
-            const user = await resolveUser3(invoice.customer, invoice.customer_email ?? void 0);
-            if (user && invoice.id && !await ledgerExistsForStripePI(invoice.id)) {
-              await creditMc(user.id, tier.credits_mc, LedgerOperation.SUBSCRIPTION, `${tier.label} subscription credits`, invoice.id, { neverExpires: tier.credits_never_expire });
-              await setSubscriptionTier(user.id, tier.tier, tier.concurrency, subId ?? user.subscription_id);
-              if (tier.includes_memory) {
-                await cancelStandaloneMemorySub(user.memory_subscription_id, subId ?? user.subscription_id);
-                await setMemoryPlan(user.id, "pro", null);
-                await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: "pro", subscription_tier: tier.tier }));
-              }
-              const credentials = await syncScheduledActionCredentials(user);
-              if (!credentials.ok) console.warn("[stripe] scheduled-action credential sync failed:", credentials.error ?? "unknown error");
-            }
-          }
-        }
-      }
-      if (event.type === "customer.subscription.created" || event.type === "customer.subscription.updated") {
-        const sub = event.data.object;
-        const priceId = findBasePlanItem(sub)?.price?.id ?? sub.items.data[0]?.price?.id;
-        if (priceId && priceId in MEMORY_PLANS) {
-          const mem = MEMORY_PLANS[priceId];
-          const user2 = await resolveUser3(sub.customer, sub.customer_email);
-          if (user2) {
-            const live = sub.status === "active" || sub.status === "trialing";
-            const newMemoryPlan = live ? mem.plan : "free";
-            await setMemoryPlan(user2.id, newMemoryPlan, live ? sub.id : null);
-            await syncMemoryKeyPlan(user2, resolveEffectiveMemoryPlan({ ...user2, memory_plan: newMemoryPlan }));
-          }
-          return c.json({ received: true });
-        }
-        const user = await getUserByStripeCustomerId(sub.customer);
-        if (!user) return c.json({ received: true });
-        if (priceId && priceId in SUBSCRIPTION_TIERS) {
-          const tier = SUBSCRIPTION_TIERS[priceId];
-          const live = sub.status === "active" || sub.status === "trialing";
-          const newTier = live ? tier.tier : null;
-          await setSubscriptionTier(user.id, newTier, live ? tier.concurrency : 0, live ? sub.id : null);
-          if (tier.includes_memory) {
-            const newMemoryPlan = live ? "pro" : "free";
-            if (live) await cancelStandaloneMemorySub(user.memory_subscription_id, sub.id);
-            await setMemoryPlan(user.id, newMemoryPlan, live ? sub.id : null);
-            await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: newMemoryPlan }));
-          } else if (user.memory_subscription_id === sub.id) {
-            await setMemoryPlan(user.id, "free", null);
-            await syncMemoryKeyPlan(user, resolveEffectiveMemoryPlan({ ...user, memory_plan: "free" }));
-          }
-          if (live) {
-            const credentials = await syncScheduledActionCredentials(user);
-            if (!credentials.ok) console.warn("[stripe] scheduled-action credential sync failed:", credentials.error ?? "unknown error");
-          }
-        } else if (priceId === CONCURRENCY_PRICE_ID && event.type === "customer.subscription.created") {
-          await setExtraConcurrencySlots(user.id, user.extra_concurrency_slots + 1);
-          await setConcurrencySubId(user.id, sub.id);
-        }
-      }
-      if (event.type === "customer.subscription.deleted") {
-        const sub = event.data.object;
-        const priceId = findBasePlanItem(sub)?.price?.id ?? sub.items.data[0]?.price?.id;
-        if (priceId && priceId in SUBSCRIPTION_TIERS) {
-          const tier = SUBSCRIPTION_TIERS[priceId];
-          const user2 = await getUserByStripeCustomerId(sub.customer);
-          if (user2) {
-            await setSubscriptionTier(user2.id, null, 0, null);
-            if (tier.includes_memory || user2.memory_subscription_id === sub.id) {
-              await setMemoryPlan(user2.id, "free", null);
-              await syncMemoryKeyPlan(user2, resolveEffectiveMemoryPlan({ ...user2, memory_plan: "free" }));
-            }
-          }
-          return c.json({ received: true });
-        }
-        if (priceId && priceId in MEMORY_PLANS || await resolveUserByMemorySubscription(sub.id)) {
-          const user2 = await resolveUserByMemorySubscription(sub.id) ?? await getUserByStripeCustomerId(sub.customer);
-          if (user2) {
-            await setMemoryPlan(user2.id, "free", null);
-            await syncMemoryKeyPlan(user2, resolveEffectiveMemoryPlan({ ...user2, memory_plan: "free" }));
-          }
-          return c.json({ received: true });
-        }
-        const user = await getUserByStripeCustomerId(sub.customer);
-        if (!user) return c.json({ received: true });
-        if (priceId === CONCURRENCY_PRICE_ID) {
-          await setExtraConcurrencySlots(user.id, Math.max(0, user.extra_concurrency_slots - 1));
-          await setConcurrencySubId(user.id, null);
-        }
+      const claim = await claimStripeEvent(event.id);
+      if (claim.state !== "claimed") return c.json({ received: true });
+      try {
+        await processStripeEvent(event);
+        const completed = await completeStripeEvent(event.id, claim.token);
+        if (!completed) throw new Error("Stripe event claim was superseded before completion");
+        return c.json({ received: true });
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        await failStripeEvent(event.id, claim.token, message).catch(() => false);
+        console.error("[stripe] webhook processing failed", event.id, message);
+        return c.json({ error: "Webhook processing failed" }, 500);
       }
-      return c.json({ received: true });
     });
   }
 });
@@ -55201,6 +55642,7 @@ function opForCostPath(p) {
   if (p.startsWith("/facebook/")) {
     if (p.endsWith("transcribe")) return "fb_transcribe";
     if (p.endsWith("/search")) return "fb_search";
+    if (p.endsWith("/page-intel")) return "fb_page_intel";
     return "fb_ad";
   }
   if (p.startsWith("/instagram/")) return "instagram";