mcp-scraper 0.3.40 → 0.3.42

package/dist/bin/api-server.cjs

@@ -5489,7 +5489,7 @@ async function fetchWithKernel(url) {
     });
     await client2.browsers.deleteByID(kb.session_id).catch(() => {
     });
-    void recordKernelSession({ kernelSessionId: kb.session_id, source: "kernel_fetch", stealth: typeof kb.stealth === "boolean" ? kb.stealth : null, headlessSent: useHeadless ? true : null, proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null, fallback: true, openedAtMs, closedAtMs: Date.now() });
+    void recordKernelSession({ kernelSessionId: kb.session_id, source: "kernel_fetch", stealth: typeof kb.stealth === "boolean" ? kb.stealth : null, headlessSent: useHeadless, proxyUsed: typeof kb.proxy_id === "string" ? Boolean(kb.proxy_id) : null, fallback: true, openedAtMs, closedAtMs: Date.now() });
   }
 }
 var import_sdk, import_playwright;
@@ -10265,7 +10265,7 @@ var init_rates = __esm({
     CONCURRENCY_SLOT_PRICE_LABEL = "$5/month";
     CONCURRENCY_SLOT_TERMINAL_COMMAND = "npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout";
     FREE_SIGNUP_MC = 1e5;
-    MEMORY_FREE_QUOTA_BYTES = 1e8 * 1;
+    MEMORY_FREE_QUOTA_BYTES = 1e7 * 1;
     MEMORY_PLANS = {
       "price_1TnMSTS8aAcsk3TGBgiwuvqL": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGBgiwuvqL", interval: "month", monthly_usd: 19, quota_bytes: 5e9 },
       "price_1TnMSTS8aAcsk3TGWBVU2agY": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGWBVU2agY", interval: "year", monthly_usd: 19, quota_bytes: 5e9 }
@@ -25557,7 +25557,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.3.40";
+    PACKAGE_VERSION = "0.3.42";
   }
 });
 
@@ -31600,8 +31600,9 @@ function isAllowedRedirect(uri) {
   } catch {
     return false;
   }
-  if (u.protocol !== "https:") return false;
   const h = u.hostname.toLowerCase();
+  if (u.protocol === "http:" && (h === "localhost" || h === "127.0.0.1" || h === "::1")) return true;
+  if (u.protocol !== "https:") return false;
   if (["claude.ai", "www.claude.ai", "claude.com", "www.claude.com"].includes(h)) return true;
   if (h.endsWith(".claude.ai") || h.endsWith(".claude.com")) return true;
   if (["chatgpt.com", "www.chatgpt.com", "chat.openai.com", "www.chat.openai.com"].includes(h)) return true;
@@ -31719,36 +31720,27 @@ function redirectWithError(redirectUri, state, error) {
   if (state) u.searchParams.set("state", state);
   return new Response(null, { status: 302, headers: { Location: u.toString() } });
 }
-function authServerMetadata(scopes = SUPPORTED_SCOPES) {
+function authServerMetadata() {
   return {
     issuer: ISSUER,
     authorization_endpoint: `${ISSUER}/oauth/authorize`,
     token_endpoint: `${ISSUER}/oauth/token`,
     registration_endpoint: `${ISSUER}/oauth/register`,
     jwks_uri: `${ISSUER}/.well-known/jwks.json`,
-    scopes_supported: [...scopes],
+    scopes_supported: [...SUPPORTED_SCOPES],
     response_types_supported: ["code"],
     grant_types_supported: ["authorization_code", "refresh_token"],
     code_challenge_methods_supported: ["S256"],
     token_endpoint_auth_methods_supported: ["none"]
   };
 }
-function openidConfiguration(scopes = SUPPORTED_SCOPES) {
+function openidConfiguration() {
   return {
-    ...authServerMetadata(scopes),
+    ...authServerMetadata(),
     subject_types_supported: ["public"],
     id_token_signing_alg_values_supported: ["RS256"]
   };
 }
-function scopesForRequest(c) {
-  const host = (c.req.header("host") ?? "").toLowerCase();
-  if (!host) return SUPPORTED_SCOPES;
-  const suffix2 = c.req.path.replace(/^\/\.well-known\/(oauth-authorization-server|openid-configuration)/, "");
-  const candidate = `https://${host}${suffix2}`.replace(/\/$/, "");
-  if (candidate === SCRAPER_RESOURCE()) return SCRAPER_SCOPES;
-  if (candidate === RESOURCE().replace(/\/$/, "")) return MEMORY_SCOPES;
-  return SUPPORTED_SCOPES;
-}
 function parseAuthParams(get) {
   return {
     response_type: get("response_type") ?? "",
@@ -31823,19 +31815,19 @@ var init_oauth_routes = __esm({
     });
     oauthApp.get("/.well-known/oauth-authorization-server", (c) => {
       console.log("[oauth-dcr] GET /.well-known/oauth-authorization-server ua=%s", c.req.header("user-agent") ?? "");
-      return c.json(authServerMetadata(scopesForRequest(c)));
+      return c.json(authServerMetadata());
     });
     oauthApp.get("/.well-known/oauth-authorization-server/*", (c) => {
       console.log("[oauth-dcr] GET %s (path-aware AS) ua=%s", c.req.path, c.req.header("user-agent") ?? "");
-      return c.json(authServerMetadata(scopesForRequest(c)));
+      return c.json(authServerMetadata());
     });
     oauthApp.get("/.well-known/openid-configuration", (c) => {
       console.log("[oauth-dcr] GET /.well-known/openid-configuration ua=%s", c.req.header("user-agent") ?? "");
-      return c.json(openidConfiguration(scopesForRequest(c)));
+      return c.json(openidConfiguration());
     });
     oauthApp.get("/.well-known/openid-configuration/*", (c) => {
       console.log("[oauth-dcr] GET %s (path-aware OIDC) ua=%s", c.req.path, c.req.header("user-agent") ?? "");
-      return c.json(openidConfiguration(scopesForRequest(c)));
+      return c.json(openidConfiguration());
     });
     oauthApp.get("/.well-known/jwks.json", async (c) => {
       const { publicJwk } = await getKeys();
@@ -33299,6 +33291,20 @@ async function entitledVaultRows(identity) {
   }
   return [...seen.values()];
 }
+async function embedBytesByPhysical(physicals) {
+  const m = /* @__PURE__ */ new Map();
+  try {
+    const rows = await sql().query(
+      `SELECT metadata->>'vault' AS vault, COUNT(*)::bigint AS chunks
+       FROM "memory" WHERE metadata->>'vault' = ANY($1) GROUP BY 1`,
+      [physicals]
+    );
+    for (const r of rows) if (r.vault) m.set(r.vault, Number(r.chunks) * EMBED_BYTES_PER_CHUNK);
+  } catch {
+    return m;
+  }
+  return m;
+}
 async function storageByPhysical(physicals) {
   if (physicals.length === 0) return /* @__PURE__ */ new Map();
   const rows = await sql().query(
@@ -33306,8 +33312,9 @@ async function storageByPhysical(physicals) {
      FROM mem_notes WHERE vault = ANY($1) GROUP BY vault`,
     [physicals]
   );
+  const embed = await embedBytesByPhysical(physicals);
   const m = /* @__PURE__ */ new Map();
-  for (const r of rows) m.set(r.vault, { notes: Number(r.notes), bytes: Number(r.bytes) });
+  for (const r of rows) m.set(r.vault, { notes: Number(r.notes), bytes: Number(r.bytes) + (embed.get(r.vault) ?? 0) });
   return m;
 }
 async function dbListVaults(identity) {
@@ -33420,13 +33427,14 @@ async function dbUsage(identity, plan) {
     unlimited: isUnlimited(plan)
   };
 }
-var import_serverless, _sql, FREE_COST_CAP_USD;
+var import_serverless, _sql, EMBED_BYTES_PER_CHUNK, FREE_COST_CAP_USD;
 var init_memory_db = __esm({
   "src/api/memory-db.ts"() {
     "use strict";
     import_serverless = require("@neondatabase/serverless");
     init_rates();
     _sql = null;
+    EMBED_BYTES_PER_CHUNK = Number(process.env.JINA_EMBED_DIM ?? 1024) * 4;
     FREE_COST_CAP_USD = 1;
   }
 });