mcp-scraper 0.88.2 → 0.89.0

package/dist/chunk-F5GQJWZU.js

@@ -1,732 +0,0 @@
-import {
-  isVendorUnavailableError
-} from "./chunk-OZJMVCDK.js";
-import {
-  loadHtmlDocument,
-  validatePublicHttpUrl
-} from "./chunk-GGZEC22A.js";
-import {
-  buildPublicErrorEnvelope,
-  getDb
-} from "./chunk-YXNDOQXN.js";
-
-// src/lib/media-extractor.ts
-import { createWriteStream, mkdirSync, rmSync } from "fs";
-import { homedir } from "os";
-import { join, extname, basename } from "path";
-import { pipeline } from "stream/promises";
-import { Readable, Transform } from "stream";
-var AD_PATTERNS = [
-  "doubleclick.net",
-  "googlesyndication.com",
-  "googletagmanager.com",
-  "google-analytics.com",
-  "googletagservices.com",
-  "adservice.google",
-  "googletag.",
-  "pagead2.googlesyndication",
-  "facebook.net/tr",
-  "connect.facebook.net",
-  "fbcdn.net/rsrc",
-  "analytics.twitter.com",
-  "static.ads-twitter.com",
-  "ads.twitter.com",
-  "t.co/i/adsct",
-  "pixel.advertising.com",
-  "hotjar.com",
-  "clarity.ms",
-  "quantserve.com",
-  "scorecardresearch.com",
-  "newrelic.com",
-  "nr-data.net",
-  "segment.io",
-  "segment.com",
-  "amplitude.com",
-  "mixpanel.com",
-  "heap.io",
-  "fullstory.com",
-  "moatads.com",
-  "criteo.com",
-  "adsrvr.org",
-  "rubiconproject.com",
-  "pubmatic.com",
-  "openx.net",
-  "appnexus.com",
-  "amazon-adsystem.com",
-  "media.net",
-  "yieldmo.com",
-  "triplelift.com",
-  "sharethrough.com",
-  "prebid.",
-  "smaato.net",
-  "indexworm.com",
-  "casalemedia.com",
-  "outbrain.com",
-  "taboola.com",
-  "revcontent.com",
-  "mgid.com",
-  "tawk.to",
-  "intercom.io",
-  "drift.com",
-  "hs-scripts.com",
-  "zopim.com",
-  "livechatinc.com",
-  "userlike.com",
-  "onetrust.com",
-  "cookielaw.org",
-  "cookieinformation.com",
-  "trustarc.com",
-  "/ads/",
-  "/ad/",
-  "/banner/",
-  "/banners/",
-  "/pixel/",
-  "/beacon/",
-  "/tracking/",
-  "/tracker/",
-  "/remarketing/",
-  "/conversion/",
-  "1x1.gif",
-  "spacer.gif",
-  "blank.gif",
-  "transparent.gif"
-];
-var IMAGE_EXTS = /* @__PURE__ */ new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".svg", ".tiff"]);
-var VIDEO_EXTS = /* @__PURE__ */ new Set([".mp4", ".webm", ".mov", ".avi", ".m4v", ".ogv", ".mkv"]);
-var AUDIO_EXTS = /* @__PURE__ */ new Set([".mp3", ".wav", ".ogg", ".aac", ".m4a", ".flac", ".opus"]);
-function isAdUrl(url) {
-  const lower = url.toLowerCase();
-  return AD_PATTERNS.some((p) => lower.includes(p));
-}
-function isDataUri(url) {
-  return url.startsWith("data:");
-}
-function typeFromUrl(url) {
-  try {
-    const ext = extname(new URL(url).pathname).toLowerCase();
-    if (IMAGE_EXTS.has(ext)) return "image";
-    if (VIDEO_EXTS.has(ext)) return "video";
-    if (AUDIO_EXTS.has(ext)) return "audio";
-  } catch {
-  }
-  return null;
-}
-function typeFromMime(mime) {
-  const lower = mime.toLowerCase();
-  if (lower.startsWith("image/")) return "image";
-  if (lower.startsWith("video/")) return "video";
-  if (lower.startsWith("audio/")) return "audio";
-  return null;
-}
-function resolveUrl(raw, base) {
-  if (!raw || isDataUri(raw)) return null;
-  try {
-    return new URL(raw, base).href;
-  } catch {
-    return null;
-  }
-}
-function safeFilename(url, index) {
-  try {
-    const u = new URL(url);
-    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
-    return base || `asset-${index}`;
-  } catch {
-    return `asset-${index}`;
-  }
-}
-function boundedText(value, max) {
-  const normalized = value?.replace(/&(?:amp|#0?38|#x26);/gi, "&").replace(/\s+/g, " ").trim();
-  return normalized ? normalized.slice(0, max) : null;
-}
-function dimensionsFromUrl(rawUrl) {
-  const match = rawUrl.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);
-  return match ? { width: Number(match[1]), height: Number(match[2]) } : null;
-}
-function mediaFamilyKey(rawUrl) {
-  try {
-    const url = new URL(rawUrl);
-    url.hash = "";
-    url.pathname = url.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i, "");
-    for (const key of ["w", "width", "h", "height", "resize"]) url.searchParams.delete(key);
-    url.searchParams.sort();
-    return url.href;
-  } catch {
-    return rawUrl;
-  }
-}
-function candidateScore(candidate) {
-  const area = (candidate.width ?? 0) * (candidate.height ?? 0);
-  const methodBoost = candidate.discoveryMethods.some((method) => method.includes("lightbox")) ? 1e12 : 0;
-  const originalBoost = /-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(candidate.url) ? 0 : 1e11;
-  return methodBoost + originalBoost + area;
-}
-function mergeCandidate(target, incoming) {
-  const existing = target.get(incoming.url);
-  if (!existing) {
-    target.set(incoming.url, incoming);
-    return;
-  }
-  existing.discoveryMethods = [.../* @__PURE__ */ new Set([...existing.discoveryMethods, ...incoming.discoveryMethods])];
-  existing.altTexts = [.../* @__PURE__ */ new Set([...existing.altTexts, ...incoming.altTexts])];
-  existing.contexts = [.../* @__PURE__ */ new Set([...existing.contexts, ...incoming.contexts])];
-  if (!existing.type && incoming.type) existing.type = incoming.type;
-  if ((incoming.width ?? 0) * (incoming.height ?? 0) > (existing.width ?? 0) * (existing.height ?? 0)) {
-    existing.width = incoming.width;
-    existing.height = incoming.height;
-  }
-}
-function discoverStaticMedia(html, baseUrl) {
-  const found = /* @__PURE__ */ new Map();
-  const add = (raw, method, explicitType = null, alt, context, dimensions) => {
-    const resolved = raw ? resolveUrl(raw.trim().replace(/&/g, "&"), baseUrl) : null;
-    if (!resolved || isAdUrl(resolved)) return;
-    const inferredDimensions = dimensions ?? dimensionsFromUrl(resolved);
-    mergeCandidate(found, {
-      url: resolved,
-      type: explicitType ?? typeFromUrl(resolved),
-      discoveryMethods: [method],
-      altTexts: alt ? [alt] : [],
-      contexts: context ? [context] : [],
-      width: inferredDimensions?.width ?? null,
-      height: inferredDimensions?.height ?? null
-    });
-  };
-  const scanSrcset = (value, method, type, alt) => {
-    for (const part of (value ?? "").split(",")) {
-      const [url, descriptor] = part.trim().split(/\s+/);
-      if (!url) continue;
-      const width = descriptor?.match(/^(\d+)w$/)?.[1];
-      add(url, method, type, alt, null, width ? { width: Number(width), height: 0 } : null);
-    }
-  };
-  const scan = (document2, suffix = "") => {
-    const $ = document2.$;
-    for (const image of document2.images) {
-      const attrs = image.attributes;
-      const alt = boundedText(image.alt, 500);
-      const dimensions = image.width && image.height ? { width: image.width, height: image.height } : null;
-      for (const attr of ["src", "data-src", "data-lazy-src", "data-original", "data-bg", "data-background", "data-bg-url", "data-lazy-bg", "data-echo"]) {
-        const value = attrs[attr];
-        add(value, `static-${attr}${suffix}`, "image", alt, null, dimensions);
-      }
-      scanSrcset(attrs.srcset, `static-srcset${suffix}`, "image", alt);
-      scanSrcset(attrs["data-srcset"], `static-data-srcset${suffix}`, "image", alt);
-    }
-    $("source").each((_index, element) => {
-      const node = $(element);
-      const declared = node.attr("type") ?? "";
-      const type = declared.startsWith("video/") ? "video" : declared.startsWith("audio/") ? "audio" : "image";
-      add(node.attr("src"), `static-source${suffix}`, type);
-      scanSrcset(node.attr("srcset"), `static-source-srcset${suffix}`, type);
-    });
-    $("video,audio").each((_index, element) => {
-      const node = $(element);
-      const type = element.tagName.toLowerCase();
-      add(node.attr("src"), `static-${type}${suffix}`, type);
-      if (type === "video") add(node.attr("poster"), `static-video-poster${suffix}`, "image");
-    });
-    $("meta").each((_index, element) => {
-      const node = $(element);
-      const prop = (node.attr("property") ?? node.attr("name") ?? "").toLowerCase();
-      if (prop === "og:image" || prop === "og:image:url" || prop === "twitter:image" || prop === "twitter:image:src") {
-        add(node.attr("content"), `static-${prop}${suffix}`, "image");
-      }
-    });
-    $("link[href]").each((_index, element) => {
-      const node = $(element);
-      const rel = (node.attr("rel") ?? "").toLowerCase().split(/\s+/);
-      if (rel.some((value) => value === "icon" || value === "apple-touch-icon")) {
-        add(node.attr("href"), `static-icon${suffix}`, "image");
-      }
-    });
-    $("svg image").each((_index, element) => {
-      const node = $(element);
-      add(node.attr("href") ?? node.attr("xlink:href"), `static-svg-image${suffix}`, "image");
-    });
-    $("a[href]").has("img").each((_index, element) => {
-      const node = $(element);
-      const href = node.attr("href");
-      if (!href) return;
-      if (/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${node.attr("class") ?? ""} ${node.attr("data-elementor-open-lightbox") ?? ""}`) || /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(href)) {
-        const alt = boundedText(node.find("img").first().attr("alt"), 500);
-        add(href, `static-lightbox-target${suffix}`, "image", alt);
-      }
-    });
-    const cssSources = $("style").toArray().map((element) => $(element).text());
-    $("[style]").each((_index, element) => {
-      cssSources.push($(element).attr("style") ?? "");
-    });
-    for (const match of cssSources.join("\n").matchAll(/(?:background(?:-image)?\s*:\s*)?url\(\s*(?:["']|"|&#0?34;|'|&#0?39;)?([^"')&]+)(?:["']|"|&#0?34;|'|&#0?39;)?\s*\)/gi)) {
-      add(match[1], `static-css-url${suffix}`, "image");
-    }
-    for (const match of document2.sourceHtml.matchAll(/["'](https?:\/\/[^"'\s<>]+\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\?[^"'\s<>]*)?)["']/gi)) {
-      add(match[1], `static-bare-url${suffix}`);
-    }
-  };
-  const document = typeof html === "string" ? loadHtmlDocument(html) : html;
-  scan(document);
-  document.$("script").each((_index, element) => {
-    const source = document.$(element).text();
-    const unescaped = source.replace(/\\u003[cC]/g, "<").replace(/\\u003[eE]/g, ">").replace(/\\u0026/gi, "&").replace(/\\["']/g, (match) => match.slice(1)).replace(/\\\//g, "/");
-    if (unescaped !== source && /<(?:img|source|video|audio)\b/i.test(unescaped)) {
-      scan(loadHtmlDocument(unescaped), "-json-unescaped");
-    }
-  });
-  return [...found.values()];
-}
-function mergeMediaDiscovery(staticAssets, rendered, maxAssets = 100) {
-  const exact = /* @__PURE__ */ new Map();
-  for (const asset of staticAssets) mergeCandidate(exact, asset);
-  for (const asset of rendered?.assets ?? []) mergeCandidate(exact, { ...asset, type: asset.type });
-  const families = /* @__PURE__ */ new Map();
-  for (const asset of exact.values()) {
-    const key = mediaFamilyKey(asset.url);
-    const family = families.get(key) ?? [];
-    family.push(asset);
-    families.set(key, family);
-  }
-  return [...families.values()].map((family) => {
-    const ordered = [...family].sort((a, b) => candidateScore(b) - candidateScore(a) || a.url.localeCompare(b.url));
-    const preferred = { ...ordered[0] };
-    preferred.discoveryMethods = [...new Set(family.flatMap((asset) => asset.discoveryMethods))];
-    preferred.altTexts = [...new Set(family.flatMap((asset) => asset.altTexts))];
-    preferred.contexts = [...new Set(family.flatMap((asset) => asset.contexts))];
-    return { ...preferred, variants: ordered.slice(1).map((asset) => asset.url) };
-  }).sort((a, b) => candidateScore(b) - candidateScore(a) || a.url.localeCompare(b.url)).slice(0, Math.max(1, maxAssets));
-}
-async function downloadAsset(url, destDir, filename, options = {}) {
-  const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024));
-  let target = url;
-  let res = null;
-  for (let redirects = 0; redirects <= 5; redirects++) {
-    const checked = await validatePublicHttpUrl(target, { field: "media URL" });
-    if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Media URL was rejected");
-    res = await fetch(checked.parsed.href, {
-      signal: AbortSignal.timeout(15e3),
-      redirect: "manual"
-    });
-    if (res.status >= 300 && res.status < 400) {
-      const location = res.headers.get("location");
-      if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`);
-      target = new URL(location, checked.parsed.href).href;
-      res = null;
-      continue;
-    }
-    break;
-  }
-  if (!res) throw new Error("Media download exceeded five redirects");
-  if (!res.ok) throw new Error(`HTTP ${res.status}`);
-  if (!res.body) throw new Error("Empty response body");
-  const mimeType = res.headers.get("content-type")?.split(";")[0].trim() ?? null;
-  if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {
-    throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? "no content-type"}`);
-  }
-  const declaredBytes = Number(res.headers.get("content-length") ?? 0);
-  if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {
-    throw new Error(`Media asset exceeds ${maxBytes} byte limit`);
-  }
-  let dest = join(destDir, filename);
-  if (mimeType && !extname(filename)) {
-    const mimeExt = {
-      "image/jpeg": ".jpg",
-      "image/png": ".png",
-      "image/webp": ".webp",
-      "image/gif": ".gif",
-      "image/svg+xml": ".svg",
-      "image/avif": ".avif",
-      "video/mp4": ".mp4",
-      "video/webm": ".webm",
-      "audio/mpeg": ".mp3",
-      "audio/ogg": ".ogg",
-      "audio/wav": ".wav"
-    };
-    const ext = mimeExt[mimeType];
-    if (ext) dest = dest + ext;
-  }
-  const writer = createWriteStream(dest);
-  await new Promise((resolve, reject) => {
-    writer.once("open", () => resolve());
-    writer.once("error", reject);
-  });
-  let streamedBytes = 0;
-  const limiter = new Transform({
-    transform(chunk, _encoding, callback) {
-      const bytes = chunk.length;
-      if (streamedBytes + bytes > maxBytes) {
-        callback(new Error(`Media asset exceeds ${maxBytes} byte limit`));
-        return;
-      }
-      if (options.consumeBytes && !options.consumeBytes(bytes)) {
-        callback(new Error("Media export total byte limit exceeded"));
-        return;
-      }
-      streamedBytes += bytes;
-      callback(null, chunk);
-    }
-  });
-  try {
-    await pipeline(Readable.fromWeb(res.body), limiter, writer);
-  } catch (error) {
-    writer.destroy();
-    rmSync(dest, { force: true });
-    throw error;
-  }
-  const { statSync } = await import("fs");
-  const sizeBytes = statSync(dest).size;
-  return { savedPath: dest, sizeBytes, mimeType };
-}
-async function harvestPageMedia(html, pageUrl, options = {}) {
-  const types = options.types ?? ["image", "video", "audio"];
-  const typesSet = new Set(types);
-  const staticAssets = discoverStaticMedia(html, pageUrl);
-  const maxAssets = Math.max(1, Math.min(options.maxAssets ?? 100, 250));
-  const allDiscovered = mergeMediaDiscovery(staticAssets, options.rendered, 2e3);
-  const discovered = allDiscovered.slice(0, maxAssets);
-  const totalFound = (/* @__PURE__ */ new Set([
-    ...staticAssets.map((asset) => asset.url),
-    ...(options.rendered?.assets ?? []).map((asset) => asset.url)
-  ])).size;
-  const kept = discovered.flatMap((asset) => {
-    const type = asset.type ?? typeFromUrl(asset.url);
-    return type && typesSet.has(type) ? [{ ...asset, type }] : [];
-  });
-  const filteredCount = totalFound - kept.length;
-  const retentionLimitReached = allDiscovered.length > maxAssets;
-  const exhausted = options.rendered?.exhausted === true && !retentionLimitReached;
-  const stopReason = retentionLimitReached ? "asset_limit" : options.rendered?.stopReason ?? "render_unavailable";
-  const warnings = retentionLimitReached ? [`Media discovery found ${allDiscovered.length} responsive families; only the requested ${maxAssets} records were retained.`] : options.rendered ? options.rendered.exhausted ? [] : [`Rendered media discovery stopped with ${options.rendered.stopReason}; the inventory may be incomplete.`] : ["Rendered media discovery was unavailable; this manifest contains static-source evidence only."];
-  const baseManifest = {
-    pageUrl,
-    staticFound: staticAssets.length,
-    renderedFound: options.rendered?.assets.length ?? 0,
-    filteredCount,
-    totalFound,
-    completeness: exhausted ? "complete" : "partial",
-    exhausted,
-    stopReason,
-    scrollRounds: options.rendered?.scrollRounds ?? 0,
-    warnings,
-    artifact: null
-  };
-  if (options.outputDir === null) {
-    return {
-      ...baseManifest,
-      outputDir: null,
-      assets: kept.map((asset, i) => ({
-        ...asset,
-        mimeType: null,
-        filename: safeFilename(asset.url, i),
-        savedPath: null,
-        sizeBytes: null,
-        sha256: null,
-        downloadStatus: "not_attempted",
-        downloadError: null,
-        inlinePreview: null
-      }))
-    };
-  }
-  const domain = (() => {
-    try {
-      return new URL(pageUrl).hostname.replace(/^www\./, "");
-    } catch {
-      return "unknown";
-    }
-  })();
-  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
-  const outDir = options.outputDir ?? join(homedir(), "Downloads", "mcp-scraper", "media", `${stamp}-${domain}`);
-  mkdirSync(outDir, { recursive: true });
-  const assets = [];
-  await Promise.allSettled(
-    kept.map(async (asset, i) => {
-      const filename = safeFilename(asset.url, i);
-      try {
-        const { savedPath, sizeBytes, mimeType } = await downloadAsset(asset.url, outDir, filename);
-        const resolvedType = mimeType ? typeFromMime(mimeType) ?? asset.type : asset.type;
-        assets.push({
-          ...asset,
-          type: resolvedType,
-          mimeType,
-          filename: basename(savedPath),
-          savedPath,
-          sizeBytes,
-          sha256: null,
-          downloadStatus: "downloaded",
-          downloadError: null,
-          inlinePreview: null
-        });
-      } catch (error) {
-        assets.push({
-          ...asset,
-          mimeType: null,
-          filename,
-          savedPath: null,
-          sizeBytes: null,
-          sha256: null,
-          downloadStatus: "failed",
-          downloadError: error instanceof Error ? error.message : "media_download_failed",
-          inlinePreview: null
-        });
-      }
-    })
-  );
-  assets.sort((a, b) => {
-    if (a.savedPath && !b.savedPath) return -1;
-    if (!a.savedPath && b.savedPath) return 1;
-    return a.url.localeCompare(b.url);
-  });
-  return { ...baseManifest, outputDir: outDir, assets };
-}
-
-// src/api/extraction-problems.ts
-var UNREACHABLE_CODES = /* @__PURE__ */ new Set([
-  "browser_navigation_blocked",
-  "browser_navigation_failed",
-  "browser_timeout",
-  "browser_session_failed",
-  "browser_result_missing"
-]);
-function errorCode(err) {
-  return err instanceof Error ? err.code : void 0;
-}
-function errorMessage(err) {
-  return err instanceof Error ? err.message : String(err);
-}
-function httpStatusFromBrowserHttpCode(value) {
-  const match = value?.match(/^browser_http_(\d+)$/);
-  return match ? Number(match[1]) : null;
-}
-function bucketHttpStatus(status) {
-  if (status === 404) return "page_not_found";
-  if (status === 403) return "page_forbidden";
-  if (status === 429) return "page_rate_limited";
-  if (status >= 500 && status <= 599) return "page_server_error";
-  return "page_http_error";
-}
-function classifyExtractionProblem(err) {
-  const message = errorMessage(err);
-  const code = errorCode(err);
-  if (isVendorUnavailableError(err, message)) {
-    return { errorCode: "vendor_unavailable", httpStatus: null, retryable: true };
-  }
-  const httpStatus = httpStatusFromBrowserHttpCode(code) ?? httpStatusFromBrowserHttpCode(message.match(/^(browser_http_\d+)/)?.[1]);
-  if (httpStatus != null) {
-    return { errorCode: bucketHttpStatus(httpStatus), httpStatus, retryable: httpStatus === 429 || httpStatus >= 500 };
-  }
-  if (code === "browser_challenge_unresolved" || /^browser_challenge_unresolved/.test(message)) {
-    return { errorCode: "bot_check_unresolved", httpStatus: null, retryable: true };
-  }
-  if (code === "browser_response_too_large" || /exceeds \d+ byte limit/i.test(message)) {
-    return { errorCode: "page_too_large", httpStatus: null, retryable: false };
-  }
-  if (/browser (?:has been )?closed|context closed|target page, context or browser has been closed|session closed/i.test(message)) {
-    return { errorCode: "browser_session_interrupted", httpStatus: null, retryable: true };
-  }
-  if (UNREACHABLE_CODES.has(code ?? "") || /timeout|navigation_failed|ERR_/i.test(message)) {
-    return { errorCode: "page_unreachable", httpStatus: null, retryable: true };
-  }
-  return { errorCode: "extraction_failed", httpStatus: null, retryable: false };
-}
-function publicExtractionErrorMessage(code, httpStatus) {
-  switch (code) {
-    case "page_not_found":
-      return `The page could not be found (HTTP ${httpStatus ?? 404}). It may have been moved or deleted.`;
-    case "page_forbidden":
-      return `The site refused the request (HTTP ${httpStatus ?? 403}).`;
-    case "page_rate_limited":
-      return "The site is rate-limiting requests right now (HTTP 429). Retrying after a short wait usually works.";
-    case "page_server_error":
-      return `The page's own server returned an error (HTTP ${httpStatus}). This is on the target site's side, not something a retry here can fix.`;
-    case "page_http_error":
-      return `The page returned an unexpected HTTP ${httpStatus} response.`;
-    case "bot_check_unresolved":
-      return "This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.";
-    case "page_too_large":
-      return "The page is larger than we can safely process.";
-    case "page_unreachable":
-      return "The page did not respond in time, or the connection could not be completed.";
-    case "browser_session_interrupted":
-      return "The browser session closed before extraction completed. A retry uses a fresh browser session.";
-    case "vendor_unavailable":
-      return "MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed \u2014 this is not caused by your request. Please retry in a few minutes.";
-    case "extraction_failed":
-      return "The page could not be extracted. Please retry, or contact support if this persists for the same URL.";
-  }
-}
-function extractionProblemResponse(problem, context = {}) {
-  const message = publicExtractionErrorMessage(problem.errorCode, problem.httpStatus);
-  const envelope = buildPublicErrorEnvelope({
-    errorCode: problem.errorCode,
-    retryable: problem.retryable,
-    chargeStatus: context.chargeStatus,
-    details: {
-      ...context.details ?? {},
-      ...problem.httpStatus != null ? { http_status: problem.httpStatus } : {}
-    }
-  });
-  return { ...envelope, message, error: message };
-}
-function extractionWireStatus(problem) {
-  if (problem.httpStatus === 404) return 404;
-  if (problem.httpStatus === 403) return 403;
-  if (problem.httpStatus === 429) return 429;
-  if (problem.httpStatus != null && problem.httpStatus >= 500 && problem.httpStatus <= 599) return 500;
-  if (problem.httpStatus != null) return 502;
-  if (problem.errorCode === "vendor_unavailable") return 503;
-  return 502;
-}
-var WAYBACK_CAPTURE_MISSING = "wayback_capture_missing";
-function publicizeExtractionFailure(failureCode, failureReason) {
-  if (failureCode == null) return { failureCode: null, failureReason: null };
-  if (failureCode === WAYBACK_CAPTURE_MISSING) return { failureCode, failureReason: failureReason ?? null };
-  const err = new Error(failureReason ?? failureCode);
-  err.code = failureCode;
-  const problem = classifyExtractionProblem(err);
-  return { failureCode: problem.errorCode, failureReason: publicExtractionErrorMessage(problem.errorCode, problem.httpStatus) };
-}
-
-// src/api/commons-embeddings.ts
-import { neon } from "@neondatabase/serverless";
-var COMMONS_EMBED_PROVIDER = "jina";
-var _vectorSql = null;
-var vectorSchemaReady = false;
-function commonsEmbedModel() {
-  return (process.env.JINA_EMBED_MODEL ?? "jina-embeddings-v5-omni-small").trim();
-}
-function commonsEmbedDim() {
-  return Number((process.env.JINA_EMBED_DIM ?? "1024").trim());
-}
-function commonsSemanticSearchConfigured() {
-  return Boolean(process.env.JINA_API_KEY?.trim() && process.env.MEMORY_DATABASE_URL?.trim());
-}
-function vectorSql() {
-  if (_vectorSql) return _vectorSql;
-  const url = process.env.MEMORY_DATABASE_URL?.trim();
-  if (!url) throw new Error("MEMORY_DATABASE_URL is not set; Commons semantic search needs the shared Postgres.");
-  _vectorSql = neon(url);
-  return _vectorSql;
-}
-async function ensureCommonsVectorSchema() {
-  if (vectorSchemaReady) return;
-  const dimension = commonsEmbedDim();
-  await vectorSql().query("CREATE EXTENSION IF NOT EXISTS vector");
-  await vectorSql().query(`
-    CREATE TABLE IF NOT EXISTS commons_index_vectors (
-      document_id   TEXT PRIMARY KEY,
-      entity_id     TEXT NOT NULL,
-      document_type TEXT NOT NULL,
-      title         TEXT NOT NULL,
-      embedding     vector(${dimension}) NOT NULL,
-      model         TEXT NOT NULL,
-      updated_at    TIMESTAMPTZ NOT NULL DEFAULT now()
-    )
-  `);
-  await vectorSql().query("CREATE INDEX IF NOT EXISTS commons_index_vectors_entity ON commons_index_vectors(entity_id)");
-  vectorSchemaReady = true;
-}
-async function embedCommonsTexts(texts) {
-  const apiKey = process.env.JINA_API_KEY?.trim();
-  if (!apiKey) throw new Error("JINA_API_KEY is not set; Commons semantic search cannot embed.");
-  if (!texts.length) return [];
-  const response = await fetch("https://api.jina.ai/v1/embeddings", {
-    method: "POST",
-    headers: { authorization: `Bearer ${apiKey}`, "content-type": "application/json" },
-    body: JSON.stringify({
-      model: commonsEmbedModel(),
-      dimensions: commonsEmbedDim(),
-      input: texts.map((text) => ({ text: text.slice(0, 8e3) }))
-    }),
-    signal: AbortSignal.timeout(6e4)
-  });
-  if (!response.ok) {
-    throw new Error(`Jina embedding request failed with HTTP ${response.status}: ${(await response.text()).slice(0, 200)}`);
-  }
-  const payload = await response.json();
-  const vectors = (payload.data ?? []).map((item) => item.embedding);
-  if (vectors.length !== texts.length) {
-    throw new Error(`Jina returned ${vectors.length} embeddings for ${texts.length} inputs.`);
-  }
-  return vectors;
-}
-async function embedQueuedCommonsDocuments(limit = 50) {
-  if (!commonsSemanticSearchConfigured()) return { claimed: 0, embedded: 0, failed: 0, remaining: 0 };
-  await ensureCommonsVectorSchema();
-  const bounded = Math.max(1, Math.min(200, Math.floor(limit)));
-  const queued = await getDb().execute({
-    sql: `SELECT id, entity_id, document_type, title, text FROM commons_index_documents
-          WHERE embedding_status IN ('queued', 'failed') ORDER BY updated_at ASC LIMIT ?`,
-    args: [bounded]
-  });
-  const rows = queued.rows;
-  if (!rows.length) return { claimed: 0, embedded: 0, failed: 0, remaining: await queuedCommonsDocumentCount() };
-  let embedded = 0;
-  let failed = 0;
-  const model = commonsEmbedModel();
-  try {
-    const vectors = await embedCommonsTexts(rows.map((row) => `${row.title}
-
-${row.text}`));
-    for (const [index, row] of rows.entries()) {
-      const literal = `[${vectors[index].join(",")}]`;
-      await vectorSql().query(
-        `INSERT INTO commons_index_vectors (document_id, entity_id, document_type, title, embedding, model, updated_at)
-         VALUES ($1, $2, $3, $4, $5::vector, $6, now())
-         ON CONFLICT (document_id) DO UPDATE SET entity_id = EXCLUDED.entity_id, document_type = EXCLUDED.document_type,
-           title = EXCLUDED.title, embedding = EXCLUDED.embedding, model = EXCLUDED.model, updated_at = now()`,
-        [row.id, row.entity_id, row.document_type, row.title, literal, model]
-      );
-      await getDb().execute({
-        sql: `UPDATE commons_index_documents SET embedding_status = 'indexed', embedding_provider = ?, embedding_model = ?,
-              vector_ref = ?, indexed_at = ?, error = NULL WHERE id = ?`,
-        args: [COMMONS_EMBED_PROVIDER, model, row.id, (/* @__PURE__ */ new Date()).toISOString(), row.id]
-      });
-      embedded += 1;
-    }
-  } catch (error) {
-    failed = rows.length - embedded;
-    const message = (error instanceof Error ? error.message : String(error)).slice(0, 500);
-    for (const row of rows.slice(embedded)) {
-      await getDb().execute({
-        sql: `UPDATE commons_index_documents SET embedding_status = 'failed', error = ? WHERE id = ?`,
-        args: [message, row.id]
-      }).catch(() => void 0);
-    }
-  }
-  return { claimed: rows.length, embedded, failed, remaining: await queuedCommonsDocumentCount() };
-}
-async function queuedCommonsDocumentCount() {
-  const result = await getDb().execute(`SELECT COUNT(*) AS n FROM commons_index_documents WHERE embedding_status IN ('queued', 'failed')`);
-  return Number(result.rows[0]?.n ?? 0);
-}
-async function semanticCommonsEntityScores(query, limit = 40) {
-  const scores = /* @__PURE__ */ new Map();
-  if (!commonsSemanticSearchConfigured() || !query.trim()) return scores;
-  await ensureCommonsVectorSchema();
-  const [vector] = await embedCommonsTexts([query]);
-  if (!vector) return scores;
-  const rows = await vectorSql().query(
-    `SELECT entity_id, MAX(1 - (embedding <=> $1::vector)) AS score
-     FROM commons_index_vectors GROUP BY entity_id ORDER BY score DESC LIMIT $2`,
-    [`[${vector.join(",")}]`, Math.max(1, Math.min(100, limit))]
-  );
-  for (const row of rows) scores.set(String(row.entity_id), Number(row.score));
-  return scores;
-}
-
-export {
-  typeFromMime,
-  discoverStaticMedia,
-  downloadAsset,
-  harvestPageMedia,
-  classifyExtractionProblem,
-  extractionProblemResponse,
-  extractionWireStatus,
-  publicizeExtractionFailure,
-  commonsEmbedModel,
-  commonsEmbedDim,
-  commonsSemanticSearchConfigured,
-  embedCommonsTexts,
-  embedQueuedCommonsDocuments,
-  queuedCommonsDocumentCount,
-  semanticCommonsEntityScores
-};