mcp-scraper 0.43.4 → 0.43.6

package/dist/index.js

@@ -1,12 +1,12 @@
 import {
   harvest
-} from "./chunk-6TAQ2MXK.js";
+} from "./chunk-RJMOOII7.js";
 import {
   recordVendorUsage
-} from "./chunk-ZDVQARDQ.js";
+} from "./chunk-T5AFM4G5.js";
 import "./chunk-3ZUBQQPQ.js";
 import "./chunk-5UN33CGU.js";
-import "./chunk-FCFQ634B.js";
+import "./chunk-5X7CJEK3.js";
 
 // src/video/VideoGenerator.ts
 import { execSync as execSync2 } from "child_process";

package/dist/{location-data-repository-SHEOIBTL.js → location-data-repository-HDNCU4Z4.js}

@@ -13,9 +13,9 @@ import {
   parseHostedCensusPlacesCsv,
   parseHostedZipGroupsCsv,
   queryHostedLocationMarkets
-} from "./chunk-QOWBJ4YY.js";
+} from "./chunk-WT2UHDSE.js";
 import "./chunk-G3P3ZDB4.js";
-import "./chunk-FCFQ634B.js";
+import "./chunk-5X7CJEK3.js";
 export {
   HOSTED_CENSUS_DATASET_PREFIX,
   HOSTED_ZIP_DATASET_KIND,
@@ -32,4 +32,4 @@ export {
   parseHostedZipGroupsCsv,
   queryHostedLocationMarkets
 };
-//# sourceMappingURL=location-data-repository-SHEOIBTL.js.map
+//# sourceMappingURL=location-data-repository-HDNCU4Z4.js.map

package/dist/{server-K76DL4S7.js → server-ZCX5X3N4.js}

@@ -4,20 +4,20 @@ import {
   harvestPageMedia,
   readOwnedSiteExtractArtifactBuffer,
   renewSiteExtractArtifactDownload
-} from "./chunk-6WLNXYRG.js";
+} from "./chunk-27FMOD6S.js";
 import {
   REQUIRED_LOCATION_STATE_CODES,
   importHostedCensusPlacesCsv,
   importHostedZipGroupsCsv,
   listActiveHostedLocationDatasets
-} from "./chunk-QOWBJ4YY.js";
+} from "./chunk-WT2UHDSE.js";
 import {
   classifyHarvestProblem,
   createHarvestAttemptRecorder,
   deliverWebhook,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-ZBQ6EXZW.js";
+} from "./chunk-OVD4E4AP.js";
 import {
   AIModeSelectors,
   AIOverviewSelectors,
@@ -44,7 +44,7 @@ import {
   resolveKernelProxyId,
   serpArrivalUrl,
   withRuntimeUserAgent
-} from "./chunk-6TAQ2MXK.js";
+} from "./chunk-RJMOOII7.js";
 import {
   ApiErrorResponseSchema,
   ArtifactTemplateConfigSchema,
@@ -86,7 +86,7 @@ import {
   renewDirectoryArtifactDownload,
   resolveDeploymentProfile,
   transcribeMediaUrl
-} from "./chunk-SXLQZKWC.js";
+} from "./chunk-3FKUKMNE.js";
 import {
   auditImageUrls,
   auditImages,
@@ -114,7 +114,7 @@ import {
   recordKernelSession,
   recordVendorUsage,
   runWithCostContext
-} from "./chunk-ZDVQARDQ.js";
+} from "./chunk-T5AFM4G5.js";
 import {
   listWorkflowDefinitions,
   runWorkflow,
@@ -142,7 +142,7 @@ import {
 } from "./chunk-3ZUBQQPQ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-XQIVXIAL.js";
+} from "./chunk-4FQDZ2T7.js";
 import {
   abandonExtractSettlement,
   countSuccessfulPages,
@@ -165,7 +165,7 @@ import {
   setExtractJobTotal,
   settleExtractJob,
   terminalExtractJobStatus
-} from "./chunk-GUI33Q27.js";
+} from "./chunk-WWIJ2NID.js";
 import {
   BROWSER_OPEN_MIN_BALANCE_MC,
   CONCURRENCY_PRICE_ID,
@@ -321,7 +321,7 @@ import {
   startHarvestAttempt,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-FCFQ634B.js";
+} from "./chunk-5X7CJEK3.js";
 
 // src/blog/registry.ts
 var posts = [
@@ -3775,6 +3775,9 @@ async function fetchWithKernel(url) {
   }
 }
 
+// src/lib/http-user-agent.ts
+var MCP_SCRAPER_CRAWLER_USER_AGENT = "MCPScraperBot/1.0 (+https://mcpscraper.dev)";
+
 // src/api/kpo-extractor.ts
 async function extractKpo(opts) {
   const start = Date.now();
@@ -3784,7 +3787,7 @@ async function extractKpo(opts) {
       let target = parsed.href;
       for (let redirects = 0; redirects < 5; redirects++) {
         const res = await fetch(target, {
-          headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
+          headers: { "User-Agent": MCP_SCRAPER_CRAWLER_USER_AGENT },
           signal: AbortSignal.timeout(15e3),
           redirect: "manual"
         });
@@ -4505,7 +4508,6 @@ var SKIP_PATH_PREFIXES = [
 ];
 var SKIP_QUERY_PARAMS = ["replytocom", "p", "preview", "ver", "v"];
 var KERNEL_RETRY_LIMIT = 15;
-var UA = "Mozilla/5.0 (compatible; ThorbitBot/1.0; +https://thorbit.ai)";
 var MAX_DISCOVERED_URL_CHARS = 4096;
 var MAX_DISCOVERY_FRONTIER_BYTES = 4 * 1024 * 1024;
 function boundedDiscoveredUrl(value) {
@@ -4557,7 +4559,7 @@ async function fetchText(url, auditedHost, timeoutMs = 1e4) {
       const checked = await validatePublicHttpUrl(target, { field: "sitemap URL" });
       if (checked.error || !checked.parsed || !sameAuditedHost(checked.parsed, auditedHost)) return null;
       const res = await fetch(checked.parsed.href, {
-        headers: { "User-Agent": UA },
+        headers: { "User-Agent": MCP_SCRAPER_CRAWLER_USER_AGENT },
         signal: AbortSignal.timeout(timeoutMs),
         redirect: "manual"
       });
@@ -4675,7 +4677,7 @@ async function fetchPage(url, timeoutMs) {
       const checked = await validatePublicHttpUrl(target, { field: "page URL" });
       if (checked.error || !checked.parsed || !sameAuditedHost(checked.parsed, auditedHost)) return { html: null, status: null };
       const res = await fetch(checked.parsed.href, {
-        headers: { "User-Agent": UA, "Accept": "text/html,application/xhtml+xml" },
+        headers: { "User-Agent": MCP_SCRAPER_CRAWLER_USER_AGENT, "Accept": "text/html,application/xhtml+xml" },
         signal: AbortSignal.timeout(timeoutMs),
         redirect: "manual"
       });
@@ -5621,7 +5623,6 @@ function makeSeedSpider(startUrl, urls) {
     sitemapUrls: []
   };
 }
-var UA2 = "Mozilla/5.0 (compatible; ThorbitBot/1.0; +https://thorbit.ai)";
 var EXTRACT_CONCURRENCY = 6;
 var MAX_PAGE_MARKDOWN = 4e4;
 var PAGE_DATA_LIMITS = {
@@ -5983,7 +5984,7 @@ async function fetchPagePlain(url) {
   try {
     for (let redirects = 0; redirects <= 5; redirects++) {
       const res = await fetch(target, {
-        headers: { "User-Agent": UA2, "Accept": "text/html,application/xhtml+xml" },
+        headers: { "User-Agent": MCP_SCRAPER_CRAWLER_USER_AGENT, "Accept": "text/html,application/xhtml+xml" },
         signal: AbortSignal.timeout(15e3),
         redirect: "manual"
       });
@@ -8666,7 +8667,7 @@ var siteExtractFn = inngest.createFunction(
       });
     }) : null;
     const finalized = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-IOM45SFL.js");
+      const { assembleExtractArtifacts } = await import("./extract-bundle-U3MNYDKC.js");
       const current = await getExtractJob(jobId2);
       if (!current) throw new Error("extract job disappeared before finalization");
       const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -9608,7 +9609,7 @@ async function resolveUsZipsPath(requestedPath) {
 async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
   const mode = resolvedLocationDataMode({ zipSource: sourceMode });
   if (mode === "hosted") {
-    const { getHostedZipGroups } = await import("./location-data-repository-SHEOIBTL.js");
+    const { getHostedZipGroups } = await import("./location-data-repository-HDNCU4Z4.js");
     const hosted = await getHostedZipGroups(stateAbbr);
     if (!hosted.dataset) {
       throw new DirectoryLocationError(
@@ -9671,7 +9672,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
   const sourceUrl = censusStateUrl(state.fips);
   const warnings = [];
   if (resolvedLocationDataMode(config) === "hosted") {
-    const { queryHostedLocationMarkets } = await import("./location-data-repository-SHEOIBTL.js");
+    const { queryHostedLocationMarkets } = await import("./location-data-repository-HDNCU4Z4.js");
     const hosted = await queryHostedLocationMarkets({
       state: state.abbr,
       minPopulation: options.minPopulation,
@@ -25549,6 +25550,31 @@ var ACCESS_TTL_SECONDS = 3600;
 var REFRESH_TTL_SECONDS = 60 * 60 * 24 * 30;
 var CODE_TTL_SECONDS = 60;
 var ROTATION_GRACE_SECONDS = 120;
+var OAUTH_DATABASE_TIMEOUT_MS = 5e3;
+var OAuthDatabaseUnavailableError = class extends Error {
+  stage;
+  constructor(stage) {
+    super("The authorization service database is temporarily unavailable.");
+    this.name = "OAuthDatabaseUnavailableError";
+    this.stage = stage;
+  }
+};
+async function withOAuthDatabaseDeadline(stage, operation, timeoutMs = OAUTH_DATABASE_TIMEOUT_MS) {
+  const startedAt = Date.now();
+  let timeout;
+  try {
+    const value = await Promise.race([
+      operation,
+      new Promise((_, reject) => {
+        timeout = setTimeout(() => reject(new OAuthDatabaseUnavailableError(stage)), timeoutMs);
+      })
+    ]);
+    console.log("[oauth-db] stage=%s status=ok duration_ms=%d", stage, Date.now() - startedAt);
+    return value;
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+}
 var secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
 var sessionCookieOptions = {
   httpOnly: true,
@@ -25611,7 +25637,7 @@ async function sessionUser(token4) {
   if (!token4) return null;
   const userId = verifySession(token4);
   if (!userId) return null;
-  const user = await getUserById(userId);
+  const user = await withOAuthDatabaseDeadline("session-user", getUserById(userId));
   return user ?? null;
 }
 function escapeHtml(value) {
@@ -25701,6 +25727,18 @@ function redirectWithError(redirectUri, state, error) {
   return new Response(null, { status: 302, headers: { Location: u.toString() } });
 }
 var oauthApp = new Hono23();
+oauthApp.onError((error, c) => {
+  if (error instanceof OAuthDatabaseUnavailableError) {
+    console.error("[oauth-db] stage=%s status=timeout timeout_ms=%d", error.stage, OAUTH_DATABASE_TIMEOUT_MS);
+    c.header("Retry-After", "5");
+    return c.json({
+      error: "temporarily_unavailable",
+      error_description: "Authorization is temporarily unavailable. Please retry shortly."
+    }, 503);
+  }
+  console.error("[oauth] unhandled error:", error);
+  return c.json({ error: "server_error", error_description: "Authorization could not be completed." }, 500);
+});
 oauthApp.use("*", async (c, next) => {
   c.header("Access-Control-Allow-Origin", "*");
   c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
@@ -25783,7 +25821,7 @@ oauthApp.post("/oauth/register", async (c) => {
   }
   const clientName = typeof body.client_name === "string" ? body.client_name : null;
   const clientId = `client_${randomBytes2(16).toString("hex")}`;
-  await registerClient(clientId, redirectUris, clientName);
+  await withOAuthDatabaseDeadline("register-client", registerClient(clientId, redirectUris, clientName));
   console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
   return c.json({
     client_id: clientId,
@@ -25810,7 +25848,7 @@ function parseAuthParams(get) {
 async function validateAuthRequest(p) {
   if (p.response_type !== "code") return { ok: false, status: 400, message: "response_type must be code" };
   if (!p.client_id) return { ok: false, status: 400, message: "client_id is required" };
-  const client2 = await getClient(p.client_id);
+  const client2 = await withOAuthDatabaseDeadline("get-client", getClient(p.client_id));
   if (!client2) return { ok: false, status: 400, message: "unknown client_id" };
   if (!p.redirect_uri || !client2.redirect_uris.includes(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri does not match the registered set" };
   if (!isAllowedRedirect(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri host not allowed" };
@@ -25843,7 +25881,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
   if (action === "login") {
     const email = (get("email") ?? "").trim().toLowerCase();
     const password = get("password") ?? "";
-    const user2 = email ? await getUserByEmail(email) : void 0;
+    const user2 = email ? await withOAuthDatabaseDeadline("login-user", getUserByEmail(email)) : void 0;
     const okPw = !!(user2 && user2.password_hash && verifyPassword(password, user2.password_hash));
     console.log("[oauth-dcr] LOGIN email=%s found=%s hasHash=%s pwLen=%d ok=%s", email, !!user2, !!(user2 && user2.password_hash), password.length, okPw);
     if (!okPw) {
@@ -25859,7 +25897,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
   const scope = negotiateScope(p.scope, user, p.resource);
   const code = `code_${randomBytes2(32).toString("base64url")}`;
   const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
-  await putCode({
+  await withOAuthDatabaseDeadline("put-authorization-code", putCode({
     code,
     client_id: p.client_id,
     redirect_uri: p.redirect_uri,
@@ -25868,7 +25906,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
     scope,
     resource: p.resource || RESOURCE(),
     expires_at: expiresAt
-  });
+  }));
   const u = new URL(p.redirect_uri);
   u.searchParams.set("code", code);
   if (p.state) u.searchParams.set("state", p.state);
@@ -25898,18 +25936,18 @@ oauthApp.post("/oauth/token", async (c) => {
     const clientId = get("client_id") ?? "";
     const redirectUri = get("redirect_uri") ?? "";
     const verifier = get("code_verifier") ?? "";
-    const record = await consumeCode(code);
+    const record = await withOAuthDatabaseDeadline("consume-authorization-code", consumeCode(code));
     if (!record) return tokenErrorResponse(c, "invalid_grant", "Authorization code is invalid or already used", 400);
     if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Authorization code has expired", 400);
     if (record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
     if (record.redirect_uri !== redirectUri) return tokenErrorResponse(c, "invalid_grant", "redirect_uri mismatch", 400);
     if (!pkceMatches(verifier, record.code_challenge)) return tokenErrorResponse(c, "invalid_grant", "PKCE verification failed", 400);
-    const user = await getUserByEmail(record.identity);
+    const user = await withOAuthDatabaseDeadline("token-user", getUserByEmail(record.identity));
     const plan = user ? resolvePlan(user) : "free";
     const audience = record.resource ?? RESOURCE();
     const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
     const refreshToken = `rt_${randomBytes2(40).toString("base64url")}`;
-    await putRefresh({
+    await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
       refresh_token: refreshToken,
       client_id: clientId,
       identity: record.identity,
@@ -25917,15 +25955,15 @@ oauthApp.post("/oauth/token", async (c) => {
       resource: audience,
       expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
       revoked: 0
-    });
+    }));
     return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: refreshToken, scope: record.scope });
   }
   if (grantType === "refresh_token") {
     const refreshToken = get("refresh_token") ?? "";
     const clientId = get("client_id") ?? "";
-    const record = await getRefresh(refreshToken);
+    const record = await withOAuthDatabaseDeadline("get-refresh-token", getRefresh(refreshToken));
     if (record && record.revoked && record.replaced_by && record.rotated_at && Date.now() - new Date(record.rotated_at).getTime() < ROTATION_GRACE_SECONDS * 1e3 && (!clientId || record.client_id === clientId)) {
-      const graceUser = await getUserByEmail(record.identity);
+      const graceUser = await withOAuthDatabaseDeadline("refresh-grace-user", getUserByEmail(record.identity));
       const gracePlan = graceUser ? resolvePlan(graceUser) : "free";
       const graceAudience = record.resource ?? RESOURCE();
       const graceAccess = await mintAccessToken(record.identity, record.scope, gracePlan, graceAudience);
@@ -25934,12 +25972,12 @@ oauthApp.post("/oauth/token", async (c) => {
     if (!record || record.revoked) return tokenErrorResponse(c, "invalid_grant", "Refresh token is invalid or revoked", 400);
     if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Refresh token has expired", 400);
     if (clientId && record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
-    const user = await getUserByEmail(record.identity);
+    const user = await withOAuthDatabaseDeadline("refresh-user", getUserByEmail(record.identity));
     const plan = user ? resolvePlan(user) : "free";
     const audience = record.resource ?? RESOURCE();
     const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
     const nextRefresh = `rt_${randomBytes2(40).toString("base64url")}`;
-    await rotateRefresh(refreshToken, {
+    await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
       refresh_token: nextRefresh,
       client_id: record.client_id,
       identity: record.identity,
@@ -25947,7 +25985,7 @@ oauthApp.post("/oauth/token", async (c) => {
       resource: audience,
       expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
       revoked: 0
-    });
+    }));
     return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: nextRefresh, scope: record.scope });
   }
   return tokenErrorResponse(c, "unsupported_grant_type", "grant_type must be authorization_code or refresh_token", 400);
@@ -37172,7 +37210,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-OB6WSWOD.js");
+  const { drainQueue } = await import("./worker-73AYU4IK.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation] = await Promise.all([
@@ -37225,8 +37263,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     finishExtractJob: finishExtractJob2,
     terminalExtractJobStatus: terminalExtractJobStatus2,
     extractJobLimitInfo: extractJobLimitInfo2
-  } = await import("./site-extract-repository-XNI4ZK2M.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-IOM45SFL.js");
+  } = await import("./site-extract-repository-CKCJDP7N.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-U3MNYDKC.js");
   const existing = await getExtractJob2(jobId2);
   if (!existing) return c.json({ error: "job not found" }, 404);
   if (existing.status !== "failed") {
@@ -37253,7 +37291,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
   if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-XNI4ZK2M.js");
+    const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-CKCJDP7N.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId2);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -37432,4 +37470,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-K76DL4S7.js.map
+//# sourceMappingURL=server-ZCX5X3N4.js.map