mcp-scraper 0.64.0 → 0.65.0

package/dist/{server-TMAWQYZE.js → server-AFEA235I.js}

@@ -1,12 +1,16 @@
 import {
   classifyExtractionProblem,
+  commonsSemanticSearchConfigured,
   discoverStaticMedia,
+  embedQueuedCommonsDocuments,
   extractionProblemResponse,
   extractionWireStatus,
   harvestPageMedia,
   publicizeExtractionFailure,
+  queuedCommonsDocumentCount,
+  semanticCommonsEntityScores,
   typeFromMime
-} from "./chunk-5QYSY5KI.js";
+} from "./chunk-AMLFKPLL.js";
 import {
   REQUIRED_LOCATION_STATE_CODES,
   importHostedCensusPlacesCsv,
@@ -199,7 +203,7 @@ import {
   resolveDeploymentProfile,
   resolveLocalSourcebookSchemaType,
   transcribeMediaUrl
-} from "./chunk-EEW3DMI5.js";
+} from "./chunk-PSKRQDGN.js";
 import {
   auditImageUrls,
   auditImages,
@@ -261,7 +265,7 @@ import {
 } from "./chunk-UVFB24RZ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-DYSXI6QU.js";
+} from "./chunk-5BODYBIP.js";
 import {
   SITE_EXTRACT_ARTIFACT_PREFIX,
   abandonExtractSettlement,
@@ -292,7 +296,7 @@ import {
   setExtractJobTotal,
   settleExtractJob,
   terminalExtractJobStatus
-} from "./chunk-BRKCPONM.js";
+} from "./chunk-5PIO7QBG.js";
 import {
   createPrivateArtifact,
   privateArtifactOwnerId,
@@ -352,7 +356,7 @@ import {
   tierChangeLeavesPrivateOffer,
   tierCreditsPerInterval,
   vendorCostUsdToMc
-} from "./chunk-TRSZEK2D.js";
+} from "./chunk-2XUXVKT4.js";
 import {
   CaptchaError,
   MapsSearchIncompletePageError,
@@ -5291,8 +5295,9 @@ async function fetchText(url, auditedHost, timeoutMs = 1e4) {
 }
 function extractSitemapLocs(xml) {
   const locs = [];
-  for (const m of xml.matchAll(/<loc>\s*(https?:\/\/[^<\s]+)\s*<\/loc>/gi)) {
-    const candidate = m[1].trim();
+  for (const m of xml.matchAll(/<loc\b[^>]*>\s*(?:<!\[CDATA\[([\s\S]*?)\]\]>|([^<]+))\s*<\/loc>/gi)) {
+    const candidate = (m[1] ?? m[2] ?? "").replace(/&amp;/gi, "&").replace(/&lt;/gi, "<").replace(/&gt;/gi, ">").replace(/&quot;/gi, '"').replace(/&#39;|&apos;/gi, "'").trim();
+    if (!/^https?:\/\//i.test(candidate)) continue;
     if (boundedDiscoveredUrl(candidate)) locs.push(candidate);
   }
   return locs;
@@ -5519,6 +5524,7 @@ import { chromium as chromium3 } from "playwright";
 var DEFAULT_CONCURRENCY = 1;
 var DEFAULT_URLS_PER_BROWSER = 10;
 var DEFAULT_HEADLESS = true;
+var MAX_RENDERED_DOM_BYTES = 256 * 1024;
 async function discoverSitemapUrls2(startUrl, maxUrls) {
   const domain = new URL(startUrl).hostname.replace(/^www\./, "");
   return discoverSitemapUrls(startUrl, domain, maxUrls);
@@ -5538,7 +5544,52 @@ async function createBrowserWithRetry(client2, headless) {
   throw new Error("unreachable");
 }
 var PAGE_HARD_TIMEOUT_MS = 35e3;
-async function crawlPage(context, url, headless, telemetry) {
+async function settleRenderedPage(page) {
+  await page.waitForLoadState("load", { timeout: 8e3 }).catch(() => void 0);
+  await page.waitForLoadState("networkidle", { timeout: 5e3 }).catch(() => void 0);
+  await page.evaluate(async () => {
+    const wait = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms));
+    const height = Math.max(document.documentElement?.scrollHeight ?? 0, document.body?.scrollHeight ?? 0);
+    const viewport = Math.max(1, window.innerHeight);
+    const steps = Math.min(12, Math.max(1, Math.ceil(height / viewport)));
+    for (let index = 1; index <= steps; index++) {
+      window.scrollTo(0, Math.min(height, index * viewport));
+      await wait(120);
+    }
+    window.scrollTo(0, 0);
+    await wait(250);
+  }).catch(() => void 0);
+}
+async function captureInertRenderedDom(page) {
+  return page.evaluate((maxBytes) => {
+    const root = document.documentElement?.cloneNode(true);
+    if (!root) return { renderedDom: "", renderedDomTruncated: false, renderedDomSanitized: true };
+    root.querySelectorAll('script,noscript,template,iframe,object,embed,meta[http-equiv="refresh"]').forEach((node) => node.remove());
+    root.querySelectorAll("*").forEach((node) => {
+      for (const attribute of [...node.attributes]) {
+        const name = attribute.name.toLowerCase();
+        const value = attribute.value.trim().toLowerCase();
+        if (name.startsWith("on") || (name === "href" || name === "src" || name === "action") && value.startsWith("javascript:")) {
+          node.removeAttribute(attribute.name);
+        }
+      }
+    });
+    const html = "<!doctype html>\n" + root.outerHTML;
+    const encoder = new TextEncoder();
+    if (encoder.encode(html).byteLength <= maxBytes) {
+      return { renderedDom: html, renderedDomTruncated: false, renderedDomSanitized: true };
+    }
+    let low = 0;
+    let high = html.length;
+    while (low < high) {
+      const middle = Math.ceil((low + high) / 2);
+      if (encoder.encode(html.slice(0, middle)).byteLength <= maxBytes) low = middle;
+      else high = middle - 1;
+    }
+    return { renderedDom: html.slice(0, low), renderedDomTruncated: true, renderedDomSanitized: true };
+  }, MAX_RENDERED_DOM_BYTES);
+}
+async function crawlPage(context, url, headless, telemetry, options) {
   const startedAt = Date.now();
   let page = null;
   let guardState = null;
@@ -5547,6 +5598,7 @@ async function crawlPage(context, url, headless, telemetry) {
     guardState = await installBrowserRequestGuard(page, url);
     const work = (async (p) => {
       const resp = await p.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 });
+      if (options.renderJavaScript || options.captureRenderedDom) await settleRenderedPage(p);
       let html = await p.evaluate((maxBytes) => {
         const value = document.documentElement?.outerHTML ?? "";
         if (new TextEncoder().encode(value).byteLength > maxBytes) throw new Error(`Browser HTML exceeds ${maxBytes} byte limit`);
@@ -5582,6 +5634,7 @@ async function crawlPage(context, url, headless, telemetry) {
         };
       }
       const isHttpError = resp != null && (resp.status() < 200 || resp.status() >= 300);
+      const rendered = options.captureRenderedDom ? await captureInertRenderedDom(p) : null;
       return {
         url,
         html,
@@ -5593,10 +5646,12 @@ async function crawlPage(context, url, headless, telemetry) {
         ...isHttpError ? { errorCode: `browser_http_${resp.status()}`, error: `Browser fetch returned HTTP ${resp.status()}` } : {},
         browserMode: headless ? "stealth_headless" : "stealth_headful",
         challengeDetected: false,
-        stopReason: isHttpError ? "http_error" : "usable_page"
+        stopReason: isHttpError ? "http_error" : "usable_page",
+        ...rendered ?? {}
       };
     })(page);
-    const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("page-hard-timeout")), PAGE_HARD_TIMEOUT_MS));
+    const hardTimeoutMs = options.renderJavaScript || options.captureRenderedDom ? 55e3 : PAGE_HARD_TIMEOUT_MS;
+    const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("page-hard-timeout")), hardTimeoutMs));
     return await Promise.race([work, timeout]);
   } catch (error) {
     const message = guardState?.blockedReason ?? (error instanceof Error ? error.message : String(error));
@@ -5668,7 +5723,7 @@ async function crawlWithRotation(urls, opts) {
             }
             const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
             try {
-              r = await crawlPage(context, url, passHeadless, { client: client2, sessionId: kb.session_id });
+              r = await crawlPage(context, url, passHeadless, { client: client2, sessionId: kb.session_id }, opts);
             } finally {
               await context.close().catch(() => {
               });
@@ -6826,17 +6881,29 @@ async function mapWithConcurrency(items, concurrency, fn) {
 }
 async function extractPagesRotating(urls, opts) {
   const uniqueUrls = [...new Set(urls)];
-  const plainPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY, async (url) => {
+  const initialPages = await mapWithConcurrency(uniqueUrls, opts.concurrency ?? EXTRACT_CONCURRENCY, async (url) => {
     const checked = await validatePublicHttpUrl(url, { field: "page URL" });
-    if (checked.error || !checked.parsed) return fetchPagePlain(url);
-    if (normalizeUrl(checked.parsed.href, checked.parsed.href)) return fetchPagePlain(url);
-    const attempt = failedAttempt("fetch", null, 0, "non_page_url", "The URL points to an asset, feed, API, account route, or other non-page path excluded from text-page extraction.");
-    const page = emptyPageData(url, null, "fetch", { code: attempt.errorCode, message: attempt.error, attempts: [attempt] });
+    if (checked.error || !checked.parsed) {
+      if (!opts.forceBrowserRender) return fetchPagePlain(url);
+      const attempt2 = failedAttempt("browser", null, 0, "unsafe_url", checked.error ?? "Page URL was rejected.");
+      return emptyPageData(url, null, "browser", { code: attempt2.errorCode, message: attempt2.error, attempts: [attempt2] });
+    }
+    if (normalizeUrl(checked.parsed.href, checked.parsed.href)) {
+      if (!opts.forceBrowserRender) return fetchPagePlain(url);
+      return emptyPageData(url, null, "browser", {
+        code: "browser_pending",
+        message: "Awaiting forced JavaScript rendering.",
+        attempts: []
+      });
+    }
+    const method = opts.forceBrowserRender ? "browser" : "fetch";
+    const attempt = failedAttempt(method, null, 0, "non_page_url", "The URL points to an asset, feed, API, account route, or other non-page path excluded from text-page extraction.");
+    const page = emptyPageData(url, null, method, { code: attempt.errorCode, message: attempt.error, attempts: [attempt] });
     page.contentKind = "excluded_url";
     return page;
   });
-  const byUrl = new Map(plainPages.map((page) => [page.url, page]));
-  const browserTargets = plainPages.filter((page) => !isPageExtractionSuccessful(page) && ![
+  const byUrl = new Map(initialPages.map((page) => [page.url, page]));
+  const browserTargets = opts.forceBrowserRender ? initialPages.filter((page) => page.failureCode === "browser_pending").map((page) => page.url) : initialPages.filter((page) => !isPageExtractionSuccessful(page) && ![
     "unsafe_url",
     "unsafe_redirect",
     "cross_site_redirect",
@@ -6848,7 +6915,9 @@ async function extractPagesRotating(urls, opts) {
     const fetched = await crawlWithRotation(browserTargets, {
       apiKey: opts.kernelApiKey,
       concurrency: opts.concurrency,
-      urlsPerBrowser: opts.urlsPerBrowser
+      urlsPerBrowser: opts.urlsPerBrowser,
+      renderJavaScript: opts.forceBrowserRender,
+      captureRenderedDom: opts.captureRenderedDom
     });
     for (const result of fetched) {
       const plainPage = byUrl.get(result.url);
@@ -6865,12 +6934,17 @@ async function extractPagesRotating(urls, opts) {
       const browserOk = !redirectRejected && Boolean(result.html) && result.status >= 200 && result.status < 300;
       if (browserOk) {
         const replay = parseWaybackReplayUrl(result.url);
-        byUrl.set(result.url, parsePageData(result.url, result.html, result.status, "browser", {
+        const page = parsePageData(result.url, result.renderedDom ?? result.html, result.status, "browser", {
           headers: result.headers,
           responseTimeMs,
           redirectUrl: result.redirectUrl,
           documentUrl: replay?.originalUrl ?? result.redirectUrl ?? result.url
-        }, [...priorAttempts, ...browserAttemptEntries(result, true)]));
+        }, [...priorAttempts, ...browserAttemptEntries(result, true)]);
+        if (result.renderedDom != null) {
+          page.renderedDomTruncated = result.renderedDomTruncated === true;
+          page.renderedDomSanitized = result.renderedDomSanitized === true;
+        }
+        byUrl.set(result.url, page);
         continue;
       }
       const code = redirectRejected ? "unsafe_browser_redirect" : result.errorCode ?? (result.status > 0 ? `browser_http_${result.status}` : result.html ? "browser_invalid_status" : "browser_empty_response");
@@ -6884,7 +6958,16 @@ async function extractPagesRotating(urls, opts) {
       }));
     }
   }
-  return urls.map((url) => byUrl.get(url) ?? emptyPageData(url, null, "fetch", {
+  if (opts.forceBrowserRender && !opts.kernelApiKey) {
+    for (const url of browserTargets) {
+      byUrl.set(url, emptyPageData(url, null, "browser", {
+        code: "browser_service_unconfigured",
+        message: "Forced JavaScript rendering requires the hosted browser service.",
+        attempts: [failedAttempt("browser", null, 0, "browser_service_unconfigured", "Forced JavaScript rendering requires the hosted browser service.")]
+      }));
+    }
+  }
+  return urls.map((url) => byUrl.get(url) ?? emptyPageData(url, null, opts.forceBrowserRender ? "browser" : "fetch", {
     code: "result_missing",
     message: "No extraction result was recorded."
   }));
@@ -9326,6 +9409,8 @@ var siteExtractFn = inngest.createFunction(
     const maxPages = Number(job.options.maxPages ?? 1e4);
     const concurrency = Number(job.options.concurrency ?? 1);
     const urlsPerBrowser = Number(job.options.urlsPerBrowser ?? job.options.rotateProxyEvery ?? 10);
+    const captureRenderedDom = job.options.captureRenderedDom === true;
+    const forceBrowserRender = job.options.renderJavaScript === true || job.options.semanticSimilarity === true || captureRenderedDom;
     const batchSize = siteExtractBatchSize(concurrency, urlsPerBrowser);
     const waybackReplay = parseWaybackReplayUrl(job.startUrl);
     const waybackTimeline = job.options.waybackTimeline;
@@ -9408,7 +9493,13 @@ var siteExtractFn = inngest.createFunction(
       if (!batch.length) break;
       const frontierCapacity = Math.max(0, maxPages - seen.size);
       const newLinks = await step.run(`crawl-batch-${i}`, async () => {
-        const pages = await runWithCostContext(costContext, () => extractPagesRotating(batch, { kernelApiKey: key, concurrency, urlsPerBrowser }));
+        const pages = await runWithCostContext(costContext, () => extractPagesRotating(batch, {
+          kernelApiKey: key,
+          concurrency,
+          urlsPerBrowser,
+          forceBrowserRender,
+          captureRenderedDom
+        }));
         const storedPages = pages.map((p) => withWaybackProvenance({
           ...p,
           // parsePageData deliberately keeps large bodies non-enumerable so
@@ -9462,7 +9553,7 @@ var siteExtractFn = inngest.createFunction(
       });
     }) : null;
     const finalized = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-FW23CEMG.js");
+      const { assembleExtractArtifacts } = await import("./extract-bundle-UWKJT4MU.js");
       const current = await getExtractJob(jobId2);
       if (!current) throw new Error("extract job disappeared before finalization");
       const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -16187,6 +16278,11 @@ async function prepareSiteExtractStart(input) {
       urlsPerBrowser: input.urlsPerBrowser,
       formats: input.formats,
       downloadImages: input.downloadImages,
+      renderJavaScript: input.renderJavaScript === true,
+      captureRenderedDom: input.captureRenderedDom === true,
+      semanticSimilarity: input.semanticSimilarity === true,
+      similarityThreshold: input.similarityThreshold,
+      similarityMaxPairs: input.similarityMaxPairs,
       ...input.waybackReplay ? {
         waybackReplay: input.waybackReplay,
         disableLinkDiscovery: true
@@ -19383,7 +19479,27 @@ var ExtractSiteBodySchema = z17.object({
   downloadImages: z17.boolean().optional(),
   preserveMedia: z17.boolean().optional(),
   delivery: z17.enum(["auto", "artifact"]).optional(),
-  formats: z17.array(z17.enum(["markdown", "html", "links", "json", "images", "branding", "issues"])).optional()
+  formats: z17.array(z17.enum(["markdown", "html", "links", "json", "images", "branding", "issues"])).optional(),
+  renderJavaScript: z17.boolean().optional(),
+  captureRenderedDom: z17.boolean().optional(),
+  semanticSimilarity: z17.boolean().optional(),
+  similarityThreshold: z17.number().min(0).max(1).optional(),
+  similarityMaxPairs: z17.number().int().min(1).max(5e4).optional()
+}).superRefine((value, ctx) => {
+  if (value.semanticSimilarity && (value.maxPages ?? 100) > 500) {
+    ctx.addIssue({
+      code: z17.ZodIssueCode.custom,
+      path: ["maxPages"],
+      message: "semanticSimilarity supports at most 500 pages per analysis."
+    });
+  }
+  if (value.semanticSimilarity && value.wayback) {
+    ctx.addIssue({
+      code: z17.ZodIssueCode.custom,
+      path: ["wayback"],
+      message: "semanticSimilarity analyzes one live rendered corpus and cannot be combined with a Wayback timeline."
+    });
+  }
 });
 var SiteExportReadBodySchema = z17.object({
   jobId: z17.string().trim().min(1),
@@ -36327,134 +36443,6 @@ async function hostEntityImages(input) {
   return rewrite;
 }
 
-// src/api/commons-embeddings.ts
-import { neon } from "@neondatabase/serverless";
-var COMMONS_EMBED_PROVIDER = "jina";
-var _vectorSql = null;
-var vectorSchemaReady = false;
-function commonsEmbedModel() {
-  return (process.env.JINA_EMBED_MODEL ?? "jina-embeddings-v5-omni-small").trim();
-}
-function commonsEmbedDim() {
-  return Number((process.env.JINA_EMBED_DIM ?? "1024").trim());
-}
-function commonsSemanticSearchConfigured() {
-  return Boolean(process.env.JINA_API_KEY?.trim() && process.env.MEMORY_DATABASE_URL?.trim());
-}
-function vectorSql() {
-  if (_vectorSql) return _vectorSql;
-  const url = process.env.MEMORY_DATABASE_URL?.trim();
-  if (!url) throw new Error("MEMORY_DATABASE_URL is not set; Commons semantic search needs the shared Postgres.");
-  _vectorSql = neon(url);
-  return _vectorSql;
-}
-async function ensureCommonsVectorSchema() {
-  if (vectorSchemaReady) return;
-  const dimension = commonsEmbedDim();
-  await vectorSql().query("CREATE EXTENSION IF NOT EXISTS vector");
-  await vectorSql().query(`
-    CREATE TABLE IF NOT EXISTS commons_index_vectors (
-      document_id   TEXT PRIMARY KEY,
-      entity_id     TEXT NOT NULL,
-      document_type TEXT NOT NULL,
-      title         TEXT NOT NULL,
-      embedding     vector(${dimension}) NOT NULL,
-      model         TEXT NOT NULL,
-      updated_at    TIMESTAMPTZ NOT NULL DEFAULT now()
-    )
-  `);
-  await vectorSql().query("CREATE INDEX IF NOT EXISTS commons_index_vectors_entity ON commons_index_vectors(entity_id)");
-  vectorSchemaReady = true;
-}
-async function embedCommonsTexts(texts) {
-  const apiKey = process.env.JINA_API_KEY?.trim();
-  if (!apiKey) throw new Error("JINA_API_KEY is not set; Commons semantic search cannot embed.");
-  if (!texts.length) return [];
-  const response = await fetch("https://api.jina.ai/v1/embeddings", {
-    method: "POST",
-    headers: { authorization: `Bearer ${apiKey}`, "content-type": "application/json" },
-    body: JSON.stringify({
-      model: commonsEmbedModel(),
-      dimensions: commonsEmbedDim(),
-      input: texts.map((text2) => ({ text: text2.slice(0, 8e3) }))
-    }),
-    signal: AbortSignal.timeout(6e4)
-  });
-  if (!response.ok) {
-    throw new Error(`Jina embedding request failed with HTTP ${response.status}: ${(await response.text()).slice(0, 200)}`);
-  }
-  const payload = await response.json();
-  const vectors = (payload.data ?? []).map((item) => item.embedding);
-  if (vectors.length !== texts.length) {
-    throw new Error(`Jina returned ${vectors.length} embeddings for ${texts.length} inputs.`);
-  }
-  return vectors;
-}
-async function embedQueuedCommonsDocuments(limit = 50) {
-  if (!commonsSemanticSearchConfigured()) return { claimed: 0, embedded: 0, failed: 0, remaining: 0 };
-  await ensureCommonsVectorSchema();
-  const bounded2 = Math.max(1, Math.min(200, Math.floor(limit)));
-  const queued = await getDb().execute({
-    sql: `SELECT id, entity_id, document_type, title, text FROM commons_index_documents
-          WHERE embedding_status IN ('queued', 'failed') ORDER BY updated_at ASC LIMIT ?`,
-    args: [bounded2]
-  });
-  const rows = queued.rows;
-  if (!rows.length) return { claimed: 0, embedded: 0, failed: 0, remaining: await queuedCommonsDocumentCount() };
-  let embedded = 0;
-  let failed = 0;
-  const model = commonsEmbedModel();
-  try {
-    const vectors = await embedCommonsTexts(rows.map((row) => `${row.title}
-
-${row.text}`));
-    for (const [index, row] of rows.entries()) {
-      const literal = `[${vectors[index].join(",")}]`;
-      await vectorSql().query(
-        `INSERT INTO commons_index_vectors (document_id, entity_id, document_type, title, embedding, model, updated_at)
-         VALUES ($1, $2, $3, $4, $5::vector, $6, now())
-         ON CONFLICT (document_id) DO UPDATE SET entity_id = EXCLUDED.entity_id, document_type = EXCLUDED.document_type,
-           title = EXCLUDED.title, embedding = EXCLUDED.embedding, model = EXCLUDED.model, updated_at = now()`,
-        [row.id, row.entity_id, row.document_type, row.title, literal, model]
-      );
-      await getDb().execute({
-        sql: `UPDATE commons_index_documents SET embedding_status = 'indexed', embedding_provider = ?, embedding_model = ?,
-              vector_ref = ?, indexed_at = ?, error = NULL WHERE id = ?`,
-        args: [COMMONS_EMBED_PROVIDER, model, row.id, (/* @__PURE__ */ new Date()).toISOString(), row.id]
-      });
-      embedded += 1;
-    }
-  } catch (error) {
-    failed = rows.length - embedded;
-    const message = (error instanceof Error ? error.message : String(error)).slice(0, 500);
-    for (const row of rows.slice(embedded)) {
-      await getDb().execute({
-        sql: `UPDATE commons_index_documents SET embedding_status = 'failed', error = ? WHERE id = ?`,
-        args: [message, row.id]
-      }).catch(() => void 0);
-    }
-  }
-  return { claimed: rows.length, embedded, failed, remaining: await queuedCommonsDocumentCount() };
-}
-async function queuedCommonsDocumentCount() {
-  const result = await getDb().execute(`SELECT COUNT(*) AS n FROM commons_index_documents WHERE embedding_status IN ('queued', 'failed')`);
-  return Number(result.rows[0]?.n ?? 0);
-}
-async function semanticCommonsEntityScores(query, limit = 40) {
-  const scores = /* @__PURE__ */ new Map();
-  if (!commonsSemanticSearchConfigured() || !query.trim()) return scores;
-  await ensureCommonsVectorSchema();
-  const [vector] = await embedCommonsTexts([query]);
-  if (!vector) return scores;
-  const rows = await vectorSql().query(
-    `SELECT entity_id, MAX(1 - (embedding <=> $1::vector)) AS score
-     FROM commons_index_vectors GROUP BY entity_id ORDER BY score DESC LIMIT $2`,
-    [`[${vector.join(",")}]`, Math.max(1, Math.min(100, limit))]
-  );
-  for (const row of rows) scores.set(String(row.entity_id), Number(row.score));
-  return scores;
-}
-
 // src/api/schema-presence.ts
 var cache = null;
 async function loadSchemaObjects() {
@@ -43194,7 +43182,7 @@ function requestedSiteMaxPages(maxPages) {
   return Math.min(ABSOLUTE_SITE_MAX_PAGES, Math.max(1, maxPages ?? DEFAULT_SITE_MAX_PAGES));
 }
 function shouldRunSiteExtractInBackground(input) {
-  return input.background === true || input.downloadImages === true || input.preserveMedia === true || requestedSiteMaxPages(input.maxPages) > MAX_SYNCHRONOUS_SITE_PAGES;
+  return input.background === true || input.downloadImages === true || input.preserveMedia === true || input.renderJavaScript === true || input.captureRenderedDom === true || input.semanticSimilarity === true || requestedSiteMaxPages(input.maxPages) > MAX_SYNCHRONOUS_SITE_PAGES;
 }
 
 // src/api/page-media-artifacts.ts
@@ -45013,7 +45001,7 @@ async function getFreeCreditBreakdown(userId) {
 }
 
 // src/api/memory-db.ts
-import { neon as neon2 } from "@neondatabase/serverless";
+import { neon } from "@neondatabase/serverless";
 
 // src/api/memory-universe.ts
 var SEP = "\n";
@@ -45110,7 +45098,7 @@ function sql() {
   if (_sql) return _sql;
   const url = process.env.MEMORY_DATABASE_URL?.trim();
   if (!url) throw new Error("MEMORY_DATABASE_URL is not set (paa-crawler needs mcp-memory Neon URL for reads)");
-  _sql = neon2(url);
+  _sql = neon(url);
   return _sql;
 }
 function physicalVault(owner, vault) {
@@ -50323,6 +50311,13 @@ app.post("/extract-site", auth4, async (c) => {
   const bodyResult = ExtractSiteBodySchema.safeParse(raw);
   if (!bodyResult.success) return c.json({ error: bodyResult.error.issues[0]?.message ?? "Invalid request" }, 400);
   const body = bodyResult.data;
+  if (body.semanticSimilarity && !process.env.JINA_API_KEY?.trim()) {
+    return c.json({
+      error: "Semantic site similarity is not configured on this deployment.",
+      errorCode: "semantic_similarity_unconfigured",
+      retryable: false
+    }, 503);
+  }
   if (body.preserveMedia !== void 0 && body.downloadImages !== void 0 && body.preserveMedia !== body.downloadImages) {
     return c.json({ error: "preserveMedia conflicts with deprecated downloadImages." }, 400);
   }
@@ -50376,7 +50371,12 @@ app.post("/extract-site", auth4, async (c) => {
       maxPages: requestedMaxPages,
       rotateProxyEvery: body.rotateProxyEvery ?? 10,
       formats: [...body.formats ?? []].sort(),
-      downloadImages
+      downloadImages,
+      renderJavaScript: body.renderJavaScript === true,
+      captureRenderedDom: body.captureRenderedDom === true,
+      semanticSimilarity: body.semanticSimilarity === true,
+      similarityThreshold: body.similarityThreshold ?? null,
+      similarityMaxPairs: body.similarityMaxPairs ?? null
     }));
     const prepared = await prepareSiteExtractStart({
       jobId: jobId2,
@@ -50390,6 +50390,11 @@ app.post("/extract-site", auth4, async (c) => {
       urlsPerBrowser: body.rotateProxyEvery ?? 10,
       formats: body.formats,
       downloadImages,
+      renderJavaScript: body.renderJavaScript === true,
+      captureRenderedDom: body.captureRenderedDom === true,
+      semanticSimilarity: body.semanticSimilarity === true,
+      similarityThreshold: body.similarityThreshold,
+      similarityMaxPairs: body.similarityMaxPairs,
       debitKey: `site-extract:${user.id}:${jobId2}:hold`,
       waybackReplay: waybackReplay && !body.wayback ? {
         timestamp: waybackReplay.timestamp,
@@ -50483,8 +50488,10 @@ app.post("/extract-site", auth4, async (c) => {
       startUrl: crawlStartUrl,
       maxPages: siteMaxPages,
       seedUrls: waybackCaptures?.map((capture) => capture.rawReplayUrl),
-      kernelApiKey: rotateProxies || wantsBranding || body.browserFallback || body.kernelFallback ? browserServiceApiKey() : void 0,
+      kernelApiKey: rotateProxies || wantsBranding || body.browserFallback || body.kernelFallback || body.renderJavaScript || body.captureRenderedDom || body.semanticSimilarity ? browserServiceApiKey() : void 0,
       formats: body.formats,
+      forceBrowserRender: body.renderJavaScript || body.captureRenderedDom || body.semanticSimilarity,
+      captureRenderedDom: body.captureRenderedDom,
       ...rotateProxies ? {
         rotateProxyEvery: body.rotateProxyEvery ?? 30,
         parallelism: concurrencyLimitForUser(user)
@@ -51006,7 +51013,7 @@ app.get("/cron/tick", async (c) => {
     return c.json({ error: "Unauthorized" }, 401);
   }
   const startedAt = Date.now();
-  const { drainQueue } = await import("./worker-DD243CON.js");
+  const { drainQueue } = await import("./worker-YKRIK5LS.js");
   const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
   const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
   const unfinished = [];
@@ -51086,8 +51093,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
     finishExtractJob: finishExtractJob2,
     terminalExtractJobStatus: terminalExtractJobStatus2,
     extractJobLimitInfo: extractJobLimitInfo2
-  } = await import("./site-extract-repository-UXSFD2QM.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-FW23CEMG.js");
+  } = await import("./site-extract-repository-J3RH3MOS.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-UWKJT4MU.js");
   const existing = await getExtractJob2(jobId2);
   if (!existing) return c.json({ error: "job not found" }, 404);
   if (existing.status !== "failed") {
@@ -51114,7 +51121,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
   if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
   let settlement = "already_settled_or_refunded";
   if (job.billedMc == null && job.userId != null) {
-    const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-UXSFD2QM.js");
+    const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-J3RH3MOS.js");
     const heldMc = Number(job.options.heldMc ?? 0);
     const successful = await countSuccessfulPages2(jobId2);
     const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -51301,4 +51308,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-TMAWQYZE.js.map
+//# sourceMappingURL=server-AFEA235I.js.map