mcp-scraper 0.43.4 → 0.43.5

package/README.md

@@ -90,7 +90,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.43.4`, SHA-256 `f6181c189de21204e9c22c6b1e4cff322ee021ab6708c5271f02f8cc6b5f005f`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.43.5`, SHA-256 `2c70cc4490f44d78f6803ab2e2b2e2fcdde72949532be053192744815bebe2aa`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 

package/dist/bin/api-server.cjs

@@ -3671,6 +3671,7 @@ __export(db_exports, {
   CORE_SCHEMA_VERSION: () => CORE_SCHEMA_VERSION,
   CREDIT_LOTS_EPOCH: () => CREDIT_LOTS_EPOCH,
   CREDIT_LOT_TTL: () => CREDIT_LOT_TTL,
+  SITE_EXTRACT_COUNTER_REPAIR_VERSION: () => SITE_EXTRACT_COUNTER_REPAIR_VERSION,
   SiteAuditJobRowSchema: () => SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema: () => SiteAuditPhaseLogRowSchema,
   advanceWorkflowRunStep: () => advanceWorkflowRunStep,
@@ -3808,7 +3809,7 @@ async function hasCurrentCoreSchema(db) {
   const result = await db.execute(`SELECT ${checks.join(" AND ")} AS ready`);
   return Number(result.rows[0]?.ready ?? 0) === 1;
 }
-async function runIdempotentCoreDataBackfills(db) {
+async function runSiteExtractCounterRepair(db) {
   await db.execute(`
     UPDATE site_extract_jobs SET
       attempted_urls = done_urls,
@@ -3848,6 +3849,18 @@ async function runIdempotentCoreDataBackfills(db) {
       AND failed_urls = 0
   `);
 }
+async function runPendingDataRepairs(db) {
+  const applied = await db.execute({
+    sql: "SELECT 1 AS applied FROM schema_migrations WHERE version = ? LIMIT 1",
+    args: [SITE_EXTRACT_COUNTER_REPAIR_VERSION]
+  });
+  if (applied.rows.length > 0) return;
+  await runSiteExtractCounterRepair(db);
+  await db.execute({
+    sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
+    args: [SITE_EXTRACT_COUNTER_REPAIR_VERSION]
+  });
+}
 function getDb() {
   if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
   return _db;
@@ -3855,7 +3868,7 @@ function getDb() {
 function setDbForTesting(client2) {
   _db = client2;
 }
-async function migrate() {
+async function migrate(options = {}) {
   const db = getDb();
   let applied;
   try {
@@ -3874,12 +3887,15 @@ async function migrate() {
     `);
     applied = { rows: [] };
   }
-  if (applied.rows.length > 0 && await hasCurrentCoreSchema(db)) {
-    await runIdempotentCoreDataBackfills(db);
-    return;
+  if (applied.rows.length > 0) {
+    if (!options.verifyCurrentSchema) return;
+    if (await hasCurrentCoreSchema(db)) {
+      if (options.runDataRepairs) await runPendingDataRepairs(db);
+      return;
+    }
   }
   if (await hasCurrentCoreSchema(db)) {
-    await runIdempotentCoreDataBackfills(db);
+    await runPendingDataRepairs(db);
     await db.execute({
       sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
       args: [CORE_SCHEMA_VERSION]
@@ -4692,7 +4708,7 @@ async function migrate() {
   if (!await hasCurrentCoreSchema(db)) {
     throw new Error("core schema migration finished without all required tables, indexes, and columns");
   }
-  await runIdempotentCoreDataBackfills(db);
+  await runPendingDataRepairs(db);
   await db.execute({
     sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
     args: [CORE_SCHEMA_VERSION]
@@ -6666,7 +6682,7 @@ async function listActiveUserIds() {
   const res = await getDb().execute("SELECT id FROM users WHERE active = 1");
   return res.rows.map((r) => Number(r.id));
 }
-var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CORE_SCHEMA_VERSION, CORE_SCHEMA_TABLES, CORE_SCHEMA_INDEXES, CORE_SCHEMA_TRIGGERS, CORE_SCHEMA_COLUMNS, REQUEST_EVENT_RESULT_MAX_BYTES, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
+var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CORE_SCHEMA_VERSION, SITE_EXTRACT_COUNTER_REPAIR_VERSION, CORE_SCHEMA_TABLES, CORE_SCHEMA_INDEXES, CORE_SCHEMA_TRIGGERS, CORE_SCHEMA_COLUMNS, REQUEST_EVENT_RESULT_MAX_BYTES, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
 var init_db = __esm({
   "src/api/db.ts"() {
     "use strict";
@@ -6680,6 +6696,7 @@ var init_db = __esm({
     _db = null;
     _rateLimitSchemaReady = false;
     CORE_SCHEMA_VERSION = "2026-07-30.1";
+    SITE_EXTRACT_COUNTER_REPAIR_VERSION = "data-2026-07-28-site-extract-counters.1";
     CORE_SCHEMA_TABLES = [
       "admin_credit_adjustment_events",
       "admin_credit_adjustments",
@@ -39517,7 +39534,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.43.4";
+    PACKAGE_VERSION = "0.43.5";
   }
 });
 
@@ -52096,6 +52113,22 @@ function isScraperResource(resource) {
   if (!resource) return false;
   return resource.replace(/\/$/, "") === SCRAPER_RESOURCE();
 }
+async function withOAuthDatabaseDeadline(stage, operation, timeoutMs = OAUTH_DATABASE_TIMEOUT_MS) {
+  const startedAt = Date.now();
+  let timeout;
+  try {
+    const value = await Promise.race([
+      operation,
+      new Promise((_, reject) => {
+        timeout = setTimeout(() => reject(new OAuthDatabaseUnavailableError(stage)), timeoutMs);
+      })
+    ]);
+    console.log("[oauth-db] stage=%s status=ok duration_ms=%d", stage, Date.now() - startedAt);
+    return value;
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+}
 async function getKeys() {
   if (cachedKeys) return cachedKeys;
   const raw = process.env.OAUTH_SIGNING_KEY;
@@ -52149,7 +52182,7 @@ async function sessionUser(token4) {
   if (!token4) return null;
   const userId = verifySession(token4);
   if (!userId) return null;
-  const user = await getUserById(userId);
+  const user = await withOAuthDatabaseDeadline("session-user", getUserById(userId));
   return user ?? null;
 }
 function escapeHtml2(value) {
@@ -52274,7 +52307,7 @@ function parseAuthParams(get) {
 async function validateAuthRequest(p) {
   if (p.response_type !== "code") return { ok: false, status: 400, message: "response_type must be code" };
   if (!p.client_id) return { ok: false, status: 400, message: "client_id is required" };
-  const client2 = await getClient(p.client_id);
+  const client2 = await withOAuthDatabaseDeadline("get-client", getClient(p.client_id));
   if (!client2) return { ok: false, status: 400, message: "unknown client_id" };
   if (!p.redirect_uri || !client2.redirect_uris.includes(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri does not match the registered set" };
   if (!isAllowedRedirect(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri host not allowed" };
@@ -52293,7 +52326,7 @@ async function mintAccessToken(identity, scope, plan, audience) {
 function tokenErrorResponse(c, error, description, status) {
   return c.json({ error, error_description: description }, status);
 }
-var import_hono23, import_cookie, import_node_crypto28, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
+var import_hono23, import_cookie, import_node_crypto28, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, OAUTH_DATABASE_TIMEOUT_MS, OAuthDatabaseUnavailableError, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
 var init_oauth_routes = __esm({
   "src/api/oauth-routes.ts"() {
     "use strict";
@@ -52315,6 +52348,15 @@ var init_oauth_routes = __esm({
     REFRESH_TTL_SECONDS = 60 * 60 * 24 * 30;
     CODE_TTL_SECONDS = 60;
     ROTATION_GRACE_SECONDS = 120;
+    OAUTH_DATABASE_TIMEOUT_MS = 5e3;
+    OAuthDatabaseUnavailableError = class extends Error {
+      stage;
+      constructor(stage) {
+        super("The authorization service database is temporarily unavailable.");
+        this.name = "OAuthDatabaseUnavailableError";
+        this.stage = stage;
+      }
+    };
     secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
     sessionCookieOptions = {
       httpOnly: true,
@@ -52325,6 +52367,18 @@ var init_oauth_routes = __esm({
     };
     cachedKeys = null;
     oauthApp = new import_hono23.Hono();
+    oauthApp.onError((error, c) => {
+      if (error instanceof OAuthDatabaseUnavailableError) {
+        console.error("[oauth-db] stage=%s status=timeout timeout_ms=%d", error.stage, OAUTH_DATABASE_TIMEOUT_MS);
+        c.header("Retry-After", "5");
+        return c.json({
+          error: "temporarily_unavailable",
+          error_description: "Authorization is temporarily unavailable. Please retry shortly."
+        }, 503);
+      }
+      console.error("[oauth] unhandled error:", error);
+      return c.json({ error: "server_error", error_description: "Authorization could not be completed." }, 500);
+    });
     oauthApp.use("*", async (c, next) => {
       c.header("Access-Control-Allow-Origin", "*");
       c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
@@ -52386,7 +52440,7 @@ var init_oauth_routes = __esm({
       }
       const clientName = typeof body.client_name === "string" ? body.client_name : null;
       const clientId = `client_${(0, import_node_crypto28.randomBytes)(16).toString("hex")}`;
-      await registerClient(clientId, redirectUris, clientName);
+      await withOAuthDatabaseDeadline("register-client", registerClient(clientId, redirectUris, clientName));
       console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
       return c.json({
         client_id: clientId,
@@ -52424,7 +52478,7 @@ var init_oauth_routes = __esm({
       if (action === "login") {
         const email = (get("email") ?? "").trim().toLowerCase();
         const password = get("password") ?? "";
-        const user2 = email ? await getUserByEmail(email) : void 0;
+        const user2 = email ? await withOAuthDatabaseDeadline("login-user", getUserByEmail(email)) : void 0;
         const okPw = !!(user2 && user2.password_hash && verifyPassword(password, user2.password_hash));
         console.log("[oauth-dcr] LOGIN email=%s found=%s hasHash=%s pwLen=%d ok=%s", email, !!user2, !!(user2 && user2.password_hash), password.length, okPw);
         if (!okPw) {
@@ -52440,7 +52494,7 @@ var init_oauth_routes = __esm({
       const scope = negotiateScope(p.scope, user, p.resource);
       const code = `code_${(0, import_node_crypto28.randomBytes)(32).toString("base64url")}`;
       const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
-      await putCode({
+      await withOAuthDatabaseDeadline("put-authorization-code", putCode({
         code,
         client_id: p.client_id,
         redirect_uri: p.redirect_uri,
@@ -52449,7 +52503,7 @@ var init_oauth_routes = __esm({
         scope,
         resource: p.resource || RESOURCE(),
         expires_at: expiresAt
-      });
+      }));
       const u = new URL(p.redirect_uri);
       u.searchParams.set("code", code);
       if (p.state) u.searchParams.set("state", p.state);
@@ -52467,18 +52521,18 @@ var init_oauth_routes = __esm({
         const clientId = get("client_id") ?? "";
         const redirectUri = get("redirect_uri") ?? "";
         const verifier = get("code_verifier") ?? "";
-        const record = await consumeCode(code);
+        const record = await withOAuthDatabaseDeadline("consume-authorization-code", consumeCode(code));
         if (!record) return tokenErrorResponse(c, "invalid_grant", "Authorization code is invalid or already used", 400);
         if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Authorization code has expired", 400);
         if (record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
         if (record.redirect_uri !== redirectUri) return tokenErrorResponse(c, "invalid_grant", "redirect_uri mismatch", 400);
         if (!pkceMatches(verifier, record.code_challenge)) return tokenErrorResponse(c, "invalid_grant", "PKCE verification failed", 400);
-        const user = await getUserByEmail(record.identity);
+        const user = await withOAuthDatabaseDeadline("token-user", getUserByEmail(record.identity));
         const plan = user ? resolvePlan(user) : "free";
         const audience = record.resource ?? RESOURCE();
         const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
         const refreshToken = `rt_${(0, import_node_crypto28.randomBytes)(40).toString("base64url")}`;
-        await putRefresh({
+        await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
           refresh_token: refreshToken,
           client_id: clientId,
           identity: record.identity,
@@ -52486,15 +52540,15 @@ var init_oauth_routes = __esm({
           resource: audience,
           expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
           revoked: 0
-        });
+        }));
         return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: refreshToken, scope: record.scope });
       }
       if (grantType === "refresh_token") {
         const refreshToken = get("refresh_token") ?? "";
         const clientId = get("client_id") ?? "";
-        const record = await getRefresh(refreshToken);
+        const record = await withOAuthDatabaseDeadline("get-refresh-token", getRefresh(refreshToken));
         if (record && record.revoked && record.replaced_by && record.rotated_at && Date.now() - new Date(record.rotated_at).getTime() < ROTATION_GRACE_SECONDS * 1e3 && (!clientId || record.client_id === clientId)) {
-          const graceUser = await getUserByEmail(record.identity);
+          const graceUser = await withOAuthDatabaseDeadline("refresh-grace-user", getUserByEmail(record.identity));
           const gracePlan = graceUser ? resolvePlan(graceUser) : "free";
           const graceAudience = record.resource ?? RESOURCE();
           const graceAccess = await mintAccessToken(record.identity, record.scope, gracePlan, graceAudience);
@@ -52503,12 +52557,12 @@ var init_oauth_routes = __esm({
         if (!record || record.revoked) return tokenErrorResponse(c, "invalid_grant", "Refresh token is invalid or revoked", 400);
         if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Refresh token has expired", 400);
         if (clientId && record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
-        const user = await getUserByEmail(record.identity);
+        const user = await withOAuthDatabaseDeadline("refresh-user", getUserByEmail(record.identity));
         const plan = user ? resolvePlan(user) : "free";
         const audience = record.resource ?? RESOURCE();
         const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
         const nextRefresh = `rt_${(0, import_node_crypto28.randomBytes)(40).toString("base64url")}`;
-        await rotateRefresh(refreshToken, {
+        await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
           refresh_token: nextRefresh,
           client_id: record.client_id,
           identity: record.identity,
@@ -52516,7 +52570,7 @@ var init_oauth_routes = __esm({
           resource: audience,
           expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
           revoked: 0
-        });
+        }));
         return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: nextRefresh, scope: record.scope });
       }
       return tokenErrorResponse(c, "unsupported_grant_type", "grant_type must be authorization_code or refresh_token", 400);