mcp-scraper 0.43.3 → 0.43.5

package/dist/index.d.cts

@@ -150,7 +150,7 @@ interface BrowserContextDebug {
     };
     locale: string;
     device: SerpDevice;
-    userAgent: string;
+    userAgent: string | null;
     deviceScaleFactor: number | null;
     isMobile: boolean;
     hasTouch: boolean;

package/dist/index.d.ts

@@ -150,7 +150,7 @@ interface BrowserContextDebug {
     };
     locale: string;
     device: SerpDevice;
-    userAgent: string;
+    userAgent: string | null;
     deviceScaleFactor: number | null;
     isMobile: boolean;
     hasTouch: boolean;

package/dist/index.js

@@ -1,12 +1,12 @@
 import {
   harvest
-} from "./chunk-H7LQ6PTI.js";
+} from "./chunk-RJMOOII7.js";
 import {
   recordVendorUsage
-} from "./chunk-DAS7L4SF.js";
+} from "./chunk-T5AFM4G5.js";
 import "./chunk-3ZUBQQPQ.js";
 import "./chunk-5UN33CGU.js";
-import "./chunk-FCFQ634B.js";
+import "./chunk-5X7CJEK3.js";
 
 // src/video/VideoGenerator.ts
 import { execSync as execSync2 } from "child_process";

package/dist/{location-data-repository-SHEOIBTL.js → location-data-repository-HDNCU4Z4.js}

@@ -13,9 +13,9 @@ import {
   parseHostedCensusPlacesCsv,
   parseHostedZipGroupsCsv,
   queryHostedLocationMarkets
-} from "./chunk-QOWBJ4YY.js";
+} from "./chunk-WT2UHDSE.js";
 import "./chunk-G3P3ZDB4.js";
-import "./chunk-FCFQ634B.js";
+import "./chunk-5X7CJEK3.js";
 export {
   HOSTED_CENSUS_DATASET_PREFIX,
   HOSTED_ZIP_DATASET_KIND,
@@ -32,4 +32,4 @@ export {
   parseHostedZipGroupsCsv,
   queryHostedLocationMarkets
 };
-//# sourceMappingURL=location-data-repository-SHEOIBTL.js.map
+//# sourceMappingURL=location-data-repository-HDNCU4Z4.js.map

package/dist/{server-7CLMCX42.js → server-BTDXZA5U.js}

@@ -10,14 +10,14 @@ import {
   importHostedCensusPlacesCsv,
   importHostedZipGroupsCsv,
   listActiveHostedLocationDatasets
-} from "./chunk-QOWBJ4YY.js";
+} from "./chunk-WT2UHDSE.js";
 import {
   classifyHarvestProblem,
   createHarvestAttemptRecorder,
   deliverWebhook,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-ZBQ6EXZW.js";
+} from "./chunk-OVD4E4AP.js";
 import {
   AIModeSelectors,
   AIOverviewSelectors,
@@ -35,13 +35,16 @@ import {
   cleanupBackupProxyId,
   createBackupProxyIdSafe,
   deleteKernelProxyId,
+  detectPageUserAgent,
+  detectRuntimeUserAgent,
   encodeUule,
   harvest,
   normalizeLocation,
   resolveBackupKernelProxyId,
   resolveKernelProxyId,
-  serpArrivalUrl
-} from "./chunk-H7LQ6PTI.js";
+  serpArrivalUrl,
+  withRuntimeUserAgent
+} from "./chunk-RJMOOII7.js";
 import {
   ApiErrorResponseSchema,
   ArtifactTemplateConfigSchema,
@@ -83,7 +86,7 @@ import {
   renewDirectoryArtifactDownload,
   resolveDeploymentProfile,
   transcribeMediaUrl
-} from "./chunk-CHC3RCFA.js";
+} from "./chunk-RSGUS5V6.js";
 import {
   auditImageUrls,
   auditImages,
@@ -111,7 +114,7 @@ import {
   recordKernelSession,
   recordVendorUsage,
   runWithCostContext
-} from "./chunk-DAS7L4SF.js";
+} from "./chunk-T5AFM4G5.js";
 import {
   listWorkflowDefinitions,
   runWorkflow,
@@ -139,7 +142,7 @@ import {
 } from "./chunk-3ZUBQQPQ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-4LQVW2EK.js";
+} from "./chunk-H4RPUFRR.js";
 import {
   abandonExtractSettlement,
   countSuccessfulPages,
@@ -162,7 +165,7 @@ import {
   setExtractJobTotal,
   settleExtractJob,
   terminalExtractJobStatus
-} from "./chunk-GUI33Q27.js";
+} from "./chunk-WWIJ2NID.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -318,7 +321,7 @@ import {
   startHarvestAttempt,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-FCFQ634B.js";
+} from "./chunk-5X7CJEK3.js";
 
 // src/blog/registry.ts
 var posts = [
@@ -3749,10 +3752,10 @@ async function fetchWithKernel(url) {
   const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
   const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
   try {
-    const context = await browser.newContext({
-      userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+    const context = await browser.newContext(withRuntimeUserAgent({
       serviceWorkers: "block"
-    });
+    }, runtimeUserAgent));
     const page = context.pages()[0] ?? await context.newPage();
     const guard = await installBrowserRequestGuard(page, auditedUrl);
     await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
@@ -4296,8 +4299,6 @@ async function extractBrandingFromPage(page) {
 // src/lib/screenshot.ts
 var DESKTOP_VIEWPORT = { width: 1440, height: 900 };
 var MOBILE_VIEWPORT = { width: 390, height: 844 };
-var DESKTOP_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-var MOBILE_UA = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
 var SCROLL_STEP = 900;
 var SETTLE_MS = 400;
 async function scrollAndStitch(page, device = "desktop") {
@@ -4348,14 +4349,13 @@ async function scrollAndStitch(page, device = "desktop") {
 }
 async function captureLocal(url, device, wantScreenshot, wantBranding) {
   const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
-  const ua = device === "mobile" ? MOBILE_UA : DESKTOP_UA;
   const browser = await chromium2.launch({ headless: true });
-  const context = await browser.newContext({
+  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+  const context = await browser.newContext(withRuntimeUserAgent({
     viewport: vp,
     locale: "en-US",
-    userAgent: ua,
     isMobile: device === "mobile"
-  });
+  }, runtimeUserAgent));
   const page = await context.newPage();
   try {
     await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
@@ -4374,12 +4374,11 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
   const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
   const browser = await chromium2.connectOverCDP(session.cdp_ws_url);
   const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
-  const ua = device === "mobile" ? MOBILE_UA : DESKTOP_UA;
-  const context = browser.contexts()[0] ?? await browser.newContext({
+  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+  const context = browser.contexts()[0] ?? await browser.newContext(withRuntimeUserAgent({
     viewport: vp,
-    userAgent: ua,
     isMobile: device === "mobile"
-  });
+  }, runtimeUserAgent));
   const page = context.pages()[0] ?? await context.newPage();
   try {
     await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
@@ -4885,6 +4884,7 @@ async function crawlWithRotation(urls, opts) {
     const worker = async () => {
       let kb = null;
       let browser = null;
+      let runtimeUserAgent = null;
       let inBrowser = 0;
       let kbOpenedAtMs = 0;
       const retire = async () => {
@@ -4892,6 +4892,7 @@ async function crawlWithRotation(urls, opts) {
         browser = null;
         kb = null;
         kbOpenedAtMs = 0;
+        runtimeUserAgent = null;
         if (b) await b.close().catch(() => {
         });
         if (k) {
@@ -4913,9 +4914,10 @@ async function crawlWithRotation(urls, opts) {
               kb = await createBrowserWithRetry(client2, headless);
               kbOpenedAtMs = Date.now();
               browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
+              runtimeUserAgent = await detectRuntimeUserAgent(browser);
               inBrowser = 0;
             }
-            const context = await browser.newContext({ serviceWorkers: "block" });
+            const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
             try {
               r = await crawlPage(context, url);
             } finally {
@@ -8664,7 +8666,7 @@ var siteExtractFn = inngest.createFunction(
       });
     }) : null;
     const finalized = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-IOM45SFL.js");
+      const { assembleExtractArtifacts } = await import("./extract-bundle-ITPFWRNI.js");
       const current = await getExtractJob(jobId2);
       if (!current) throw new Error("extract job disappeared before finalization");
       const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -9606,7 +9608,7 @@ async function resolveUsZipsPath(requestedPath) {
 async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
   const mode = resolvedLocationDataMode({ zipSource: sourceMode });
   if (mode === "hosted") {
-    const { getHostedZipGroups } = await import("./location-data-repository-SHEOIBTL.js");
+    const { getHostedZipGroups } = await import("./location-data-repository-HDNCU4Z4.js");
     const hosted = await getHostedZipGroups(stateAbbr);
     if (!hosted.dataset) {
       throw new DirectoryLocationError(
@@ -9669,7 +9671,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
   const sourceUrl = censusStateUrl(state.fips);
   const warnings = [];
   if (resolvedLocationDataMode(config) === "hosted") {
-    const { queryHostedLocationMarkets } = await import("./location-data-repository-SHEOIBTL.js");
+    const { queryHostedLocationMarkets } = await import("./location-data-repository-HDNCU4Z4.js");
     const hosted = await queryHostedLocationMarkets({
       state: state.abbr,
       minPopulation: options.minPopulation,
@@ -16681,7 +16683,6 @@ facebookAdApp.post("/media", createApiKeyAuth(), async (c) => {
   try {
     res = await fetch(parsedUrl.href, {
       headers: {
-        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
         "Referer": "https://www.facebook.com/"
       },
       signal: AbortSignal.timeout(6e4)
@@ -17998,10 +17999,10 @@ function blobKeyPrefix(shortcode, sourceUrl) {
   const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
   return `instagram/${stamp}-${slug}`;
 }
-async function fetchMediaBytes(url, referer) {
+async function fetchMediaBytes(url, referer, runtimeUserAgent) {
   const res = await fetch(url, {
     headers: {
-      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      ...runtimeUserAgent ? { "User-Agent": runtimeUserAgent } : {},
       "Referer": referer
     },
     signal: AbortSignal.timeout(9e4)
@@ -18121,6 +18122,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
     const launch = await resolveInstagramLaunch(body);
     await driver.launch(launch.config);
     const page = driver.getPage();
+    const runtimeUserAgent = await detectPageUserAgent(page);
     const capturedMediaUrls = [];
     page.on("response", (response) => {
       const url = normalizeInstagramMediaUrl(response.url());
@@ -18149,7 +18151,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
         downloads.push({ kind: "text", url: null, savedPath: storedText.url, sizeBytes: storedText.bytes, mimeType: "text/plain", error: null });
         if (mediaTypes.has("image") && extraction.imageUrl) {
           try {
-            const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl);
+            const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl, runtimeUserAgent);
             const ext = extFromMime(mimeType, "jpg");
             const stored = await store.put(`${prefix}/image.${ext}`, bytes, mimeType ?? "application/octet-stream");
             downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
@@ -18165,7 +18167,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
           if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
           const kind = track.streamType === "audio" ? "audio" : "video";
           try {
-            const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl);
+            const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl, runtimeUserAgent);
             const stored = await store.put(`${prefix}/${trackFilename(track, index, extraction.shortcode)}`, bytes, mimeType ?? "video/mp4");
             downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
             const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
@@ -24041,6 +24043,12 @@ async function runFanoutCapture(page, input) {
 // src/services/browser-agent/browser-agent-service.ts
 function keepHostedBrowserAlive(_browser) {
 }
+async function currentOrRuntimeContext(browser) {
+  const current = browser.contexts()[0];
+  if (current) return current;
+  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+  return browser.newContext(withRuntimeUserAgent({}, runtimeUserAgent));
+}
 var DEFAULT_TIMEOUT_SECONDS = 600;
 function client() {
   const apiKey = browserServiceApiKey();
@@ -24165,7 +24173,7 @@ async function pressKeys(runtimeSessionId, keys) {
 async function goto(cdpWsUrl, url) {
   const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
     return { url: page.url(), title: await page.title() };
@@ -24176,7 +24184,7 @@ async function goto(cdpWsUrl, url) {
 async function captureFanout(cdpWsUrl, input) {
   const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     return await runFanoutCapture(page, input);
   } finally {
@@ -24186,7 +24194,7 @@ async function captureFanout(cdpWsUrl, input) {
 async function readPage(cdpWsUrl) {
   const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     const url = page.url();
     const title = await page.title().catch(() => "");
@@ -24241,7 +24249,7 @@ async function readPage(cdpWsUrl) {
 async function locatePageTargets(cdpWsUrl, targets) {
   const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     const url = page.url();
     const title = await page.title().catch(() => "");
@@ -25541,6 +25549,31 @@ var ACCESS_TTL_SECONDS = 3600;
 var REFRESH_TTL_SECONDS = 60 * 60 * 24 * 30;
 var CODE_TTL_SECONDS = 60;
 var ROTATION_GRACE_SECONDS = 120;
+var OAUTH_DATABASE_TIMEOUT_MS = 5e3;
+var OAuthDatabaseUnavailableError = class extends Error {
+  stage;
+  constructor(stage) {
+    super("The authorization service database is temporarily unavailable.");
+    this.name = "OAuthDatabaseUnavailableError";
+    this.stage = stage;
+  }
+};
+async function withOAuthDatabaseDeadline(stage, operation, timeoutMs = OAUTH_DATABASE_TIMEOUT_MS) {
+  const startedAt = Date.now();
+  let timeout;
+  try {
+    const value = await Promise.race([
+      operation,
+      new Promise((_, reject) => {
+        timeout = setTimeout(() => reject(new OAuthDatabaseUnavailableError(stage)), timeoutMs);
+      })
+    ]);
+    console.log("[oauth-db] stage=%s status=ok duration_ms=%d", stage, Date.now() - startedAt);
+    return value;
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+}
 var secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
 var sessionCookieOptions = {
   httpOnly: true,
@@ -25603,7 +25636,7 @@ async function sessionUser(token4) {
   if (!token4) return null;
   const userId = verifySession(token4);
   if (!userId) return null;
-  const user = await getUserById(userId);
+  const user = await withOAuthDatabaseDeadline("session-user", getUserById(userId));
   return user ?? null;
 }
 function escapeHtml(value) {
@@ -25693,6 +25726,18 @@ function redirectWithError(redirectUri, state, error) {
   return new Response(null, { status: 302, headers: { Location: u.toString() } });
 }
 var oauthApp = new Hono23();
+oauthApp.onError((error, c) => {
+  if (error instanceof OAuthDatabaseUnavailableError) {
+    console.error("[oauth-db] stage=%s status=timeout timeout_ms=%d", error.stage, OAUTH_DATABASE_TIMEOUT_MS);
+    c.header("Retry-After", "5");
+    return c.json({
+      error: "temporarily_unavailable",
+      error_description: "Authorization is temporarily unavailable. Please retry shortly."
+    }, 503);
+  }
+  console.error("[oauth] unhandled error:", error);
+  return c.json({ error: "server_error", error_description: "Authorization could not be completed." }, 500);
+});
 oauthApp.use("*", async (c, next) => {
   c.header("Access-Control-Allow-Origin", "*");
   c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
@@ -25775,7 +25820,7 @@ oauthApp.post("/oauth/register", async (c) => {
   }
   const clientName = typeof body.client_name === "string" ? body.client_name : null;
   const clientId = `client_${randomBytes2(16).toString("hex")}`;
-  await registerClient(clientId, redirectUris, clientName);
+  await withOAuthDatabaseDeadline("register-client", registerClient(clientId, redirectUris, clientName));
   console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
   return c.json({
     client_id: clientId,
@@ -25802,7 +25847,7 @@ function parseAuthParams(get) {
 async function validateAuthRequest(p) {
   if (p.response_type !== "code") return { ok: false, status: 400, message: "response_type must be code" };
   if (!p.client_id) return { ok: false, status: 400, message: "client_id is required" };
-  const client2 = await getClient(p.client_id);
+  const client2 = await withOAuthDatabaseDeadline("get-client", getClient(p.client_id));
   if (!client2) return { ok: false, status: 400, message: "unknown client_id" };
   if (!p.redirect_uri || !client2.redirect_uris.includes(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri does not match the registered set" };
   if (!isAllowedRedirect(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri host not allowed" };
@@ -25835,7 +25880,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
   if (action === "login") {
     const email = (get("email") ?? "").trim().toLowerCase();
     const password = get("password") ?? "";
-    const user2 = email ? await getUserByEmail(email) : void 0;
+    const user2 = email ? await withOAuthDatabaseDeadline("login-user", getUserByEmail(email)) : void 0;
     const okPw = !!(user2 && user2.password_hash && verifyPassword(password, user2.password_hash));
     console.log("[oauth-dcr] LOGIN email=%s found=%s hasHash=%s pwLen=%d ok=%s", email, !!user2, !!(user2 && user2.password_hash), password.length, okPw);
     if (!okPw) {
@@ -25851,7 +25896,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
   const scope = negotiateScope(p.scope, user, p.resource);
   const code = `code_${randomBytes2(32).toString("base64url")}`;
   const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
-  await putCode({
+  await withOAuthDatabaseDeadline("put-authorization-code", putCode({
     code,
     client_id: p.client_id,
     redirect_uri: p.redirect_uri,
@@ -25860,7 +25905,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
     scope,
     resource: p.resource || RESOURCE(),
     expires_at: expiresAt
-  });
+  }));
   const u = new URL(p.redirect_uri);
   u.searchParams.set("code", code);
   if (p.state) u.searchParams.set("state", p.state);
@@ -25890,18 +25935,18 @@ oauthApp.post("/oauth/token", async (c) => {
     const clientId = get("client_id") ?? "";
     const redirectUri = get("redirect_uri") ?? "";
     const verifier = get("code_verifier") ?? "";
-    const record = await consumeCode(code);
+    const record = await withOAuthDatabaseDeadline("consume-authorization-code", consumeCode(code));
     if (!record) return tokenErrorResponse(c, "invalid_grant", "Authorization code is invalid or already used", 400);
     if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Authorization code has expired", 400);
     if (record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
     if (record.redirect_uri !== redirectUri) return tokenErrorResponse(c, "invalid_grant", "redirect_uri mismatch", 400);
     if (!pkceMatches(verifier, record.code_challenge)) return tokenErrorResponse(c, "invalid_grant", "PKCE verification failed", 400);
-    const user = await getUserByEmail(record.identity);
+    const user = await withOAuthDatabaseDeadline("token-user", getUserByEmail(record.identity));
     const plan = user ? resolvePlan(user) : "free";
     const audience = record.resource ?? RESOURCE();
     const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
     const refreshToken = `rt_${randomBytes2(40).toString("base64url")}`;
-    await putRefresh({
+    await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
       refresh_token: refreshToken,
       client_id: clientId,
       identity: record.identity,
@@ -25909,15 +25954,15 @@ oauthApp.post("/oauth/token", async (c) => {
       resource: audience,
       expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
       revoked: 0
-    });
+    }));
     return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: refreshToken, scope: record.scope });
   }
   if (grantType === "refresh_token") {
     const refreshToken = get("refresh_token") ?? "";
     const clientId = get("client_id") ?? "";
-    const record = await getRefresh(refreshToken);
+    const record = await withOAuthDatabaseDeadline("get-refresh-token", getRefresh(refreshToken));
     if (record && record.revoked && record.replaced_by && record.rotated_at && Date.now() - new Date(record.rotated_at).getTime() < ROTATION_GRACE_SECONDS * 1e3 && (!clientId || record.client_id === clientId)) {
-      const graceUser = await getUserByEmail(record.identity);
+      const graceUser = await withOAuthDatabaseDeadline("refresh-grace-user", getUserByEmail(record.identity));
       const gracePlan = graceUser ? resolvePlan(graceUser) : "free";
       const graceAudience = record.resource ?? RESOURCE();
       const graceAccess = await mintAccessToken(record.identity, record.scope, gracePlan, graceAudience);
@@ -25926,12 +25971,12 @@ oauthApp.post("/oauth/token", async (c) => {
     if (!record || record.revoked) return tokenErrorResponse(c, "invalid_grant", "Refresh token is invalid or revoked", 400);
     if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Refresh token has expired", 400);
     if (clientId && record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
-    const user = await getUserByEmail(record.identity);
+    const user = await withOAuthDatabaseDeadline("refresh-user", getUserByEmail(record.identity));
     const plan = user ? resolvePlan(user) : "free";
     const audience = record.resource ?? RESOURCE();
     const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
     const nextRefresh = `rt_${randomBytes2(40).toString("base64url")}`;
-    await rotateRefresh(refreshToken, {
+    await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
       refresh_token: nextRefresh,
       client_id: record.client_id,
       identity: record.identity,
@@ -25939,7 +25984,7 @@ oauthApp.post("/oauth/token", async (c) => {
       resource: audience,
       expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
       revoked: 0
-    });
+    }));
     return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: nextRefresh, scope: record.scope });
   }
   return tokenErrorResponse(c, "unsupported_grant_type", "grant_type must be authorization_code or refresh_token", 400);
@@ -34346,8 +34391,7 @@ app.use("*", (c, next) => {
   const op = opForCostPath(new URL(c.req.url).pathname);
   if (!op) return next();
   const probeRunId = c.req.header("x-cost-probe-id") ?? null;
-  const forceHeadless = c.req.header("x-kernel-headless") === "true";
-  return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
+  return runWithCostContext({ op, probeRunId }, () => next());
 });
 app.post("/auth/register", requireAllowedOrigin, async (c) => {
   const { email, password } = await c.req.json();
@@ -37165,7 +37209,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-2LIOH744.js");
+  const { drainQueue } = await import("./worker-73AYU4IK.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation] = await Promise.all([
@@ -37218,8 +37262,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     finishExtractJob: finishExtractJob2,
     terminalExtractJobStatus: terminalExtractJobStatus2,
     extractJobLimitInfo: extractJobLimitInfo2
-  } = await import("./site-extract-repository-XNI4ZK2M.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-IOM45SFL.js");
+  } = await import("./site-extract-repository-CKCJDP7N.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-ITPFWRNI.js");
   const existing = await getExtractJob2(jobId2);
   if (!existing) return c.json({ error: "job not found" }, 404);
   if (existing.status !== "failed") {
@@ -37246,7 +37290,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
   if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-XNI4ZK2M.js");
+    const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-CKCJDP7N.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId2);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -37425,4 +37469,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-7CLMCX42.js.map
+//# sourceMappingURL=server-BTDXZA5U.js.map