mcp-scraper 0.35.1 → 0.37.0

package/dist/chunk-YV2FUEBX.js

@@ -1,851 +0,0 @@
-// src/lib/media-extractor.ts
-import { createWriteStream, mkdirSync } from "fs";
-import { homedir } from "os";
-import { join, extname, basename } from "path";
-import { pipeline } from "stream/promises";
-import { Readable } from "stream";
-var AD_PATTERNS = [
-  "doubleclick.net",
-  "googlesyndication.com",
-  "googletagmanager.com",
-  "google-analytics.com",
-  "googletagservices.com",
-  "adservice.google",
-  "googletag.",
-  "pagead2.googlesyndication",
-  "facebook.net/tr",
-  "connect.facebook.net",
-  "fbcdn.net/rsrc",
-  "analytics.twitter.com",
-  "static.ads-twitter.com",
-  "ads.twitter.com",
-  "t.co/i/adsct",
-  "pixel.advertising.com",
-  "hotjar.com",
-  "clarity.ms",
-  "quantserve.com",
-  "scorecardresearch.com",
-  "newrelic.com",
-  "nr-data.net",
-  "segment.io",
-  "segment.com",
-  "amplitude.com",
-  "mixpanel.com",
-  "heap.io",
-  "fullstory.com",
-  "moatads.com",
-  "criteo.com",
-  "adsrvr.org",
-  "rubiconproject.com",
-  "pubmatic.com",
-  "openx.net",
-  "appnexus.com",
-  "amazon-adsystem.com",
-  "media.net",
-  "yieldmo.com",
-  "triplelift.com",
-  "sharethrough.com",
-  "prebid.",
-  "smaato.net",
-  "indexworm.com",
-  "casalemedia.com",
-  "outbrain.com",
-  "taboola.com",
-  "revcontent.com",
-  "mgid.com",
-  "tawk.to",
-  "intercom.io",
-  "drift.com",
-  "hs-scripts.com",
-  "zopim.com",
-  "livechatinc.com",
-  "userlike.com",
-  "onetrust.com",
-  "cookielaw.org",
-  "cookieinformation.com",
-  "trustarc.com",
-  "/ads/",
-  "/ad/",
-  "/banner/",
-  "/banners/",
-  "/pixel/",
-  "/beacon/",
-  "/tracking/",
-  "/tracker/",
-  "/remarketing/",
-  "/conversion/",
-  "1x1.gif",
-  "spacer.gif",
-  "blank.gif",
-  "transparent.gif"
-];
-var IMAGE_EXTS = /* @__PURE__ */ new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".svg", ".tiff"]);
-var VIDEO_EXTS = /* @__PURE__ */ new Set([".mp4", ".webm", ".mov", ".avi", ".m4v", ".ogv", ".mkv"]);
-var AUDIO_EXTS = /* @__PURE__ */ new Set([".mp3", ".wav", ".ogg", ".aac", ".m4a", ".flac", ".opus"]);
-function isAdUrl(url) {
-  const lower = url.toLowerCase();
-  return AD_PATTERNS.some((p) => lower.includes(p));
-}
-function isDataUri(url) {
-  return url.startsWith("data:");
-}
-function typeFromUrl(url) {
-  try {
-    const ext = extname(new URL(url).pathname).toLowerCase();
-    if (IMAGE_EXTS.has(ext)) return "image";
-    if (VIDEO_EXTS.has(ext)) return "video";
-    if (AUDIO_EXTS.has(ext)) return "audio";
-  } catch {
-  }
-  return null;
-}
-function typeFromMime(mime) {
-  const lower = mime.toLowerCase();
-  if (lower.startsWith("image/")) return "image";
-  if (lower.startsWith("video/")) return "video";
-  if (lower.startsWith("audio/")) return "audio";
-  return null;
-}
-function resolveUrl(raw, base) {
-  if (!raw || isDataUri(raw)) return null;
-  try {
-    return new URL(raw, base).href;
-  } catch {
-    return null;
-  }
-}
-function safeFilename(url, index) {
-  try {
-    const u = new URL(url);
-    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
-    return base || `asset-${index}`;
-  } catch {
-    return `asset-${index}`;
-  }
-}
-function extractMediaUrls(html, baseUrl) {
-  const seen = /* @__PURE__ */ new Set();
-  const urls = [];
-  const add = (raw) => {
-    const resolved = resolveUrl(raw.trim(), baseUrl);
-    if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return;
-    seen.add(resolved);
-    urls.push(resolved);
-  };
-  for (const m of html.matchAll(/<img\s[^>]*>/gi)) {
-    const tag = m[0];
-    for (const attr of ["src", "data-src", "data-lazy-src", "data-original"]) {
-      const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
-      if (v) add(v);
-    }
-    const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
-    if (srcset) {
-      for (const part of srcset.split(",")) {
-        const u = part.trim().split(/\s+/)[0];
-        if (u) add(u);
-      }
-    }
-  }
-  for (const m of html.matchAll(/<source\s[^>]*>/gi)) {
-    const tag = m[0];
-    const src = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
-    if (src) add(src);
-    const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
-    if (srcset) {
-      for (const part of srcset.split(",")) {
-        const u = part.trim().split(/\s+/)[0];
-        if (u) add(u);
-      }
-    }
-  }
-  for (const m of html.matchAll(/<video\s[^>]*>/gi)) {
-    const tag = m[0];
-    for (const attr of ["src", "poster"]) {
-      const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
-      if (v) add(v);
-    }
-  }
-  for (const m of html.matchAll(/<audio\s[^>]*>/gi)) {
-    const tag = m[0];
-    const v = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
-    if (v) add(v);
-  }
-  for (const m of html.matchAll(/<meta\s[^>]*>/gi)) {
-    const tag = m[0];
-    const prop = (tag.match(/\b(?:property|name)\s*=\s*["']([^"']+)["']/i) ?? [])[1]?.toLowerCase();
-    if (prop === "og:image" || prop === "twitter:image") {
-      const content = (tag.match(/\bcontent\s*=\s*["']([^"']+)["']/i) ?? [])[1];
-      if (content) add(content);
-    }
-  }
-  for (const m of html.matchAll(/background(?:-image)?\s*:\s*url\(\s*["']?([^"')]+)["']?\s*\)/gi)) {
-    add(m[1]);
-  }
-  return urls;
-}
-async function downloadAsset(url, destDir, filename) {
-  const res = await fetch(url, {
-    headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
-    signal: AbortSignal.timeout(15e3)
-  });
-  if (!res.ok) throw new Error(`HTTP ${res.status}`);
-  if (!res.body) throw new Error("Empty response body");
-  const mimeType = res.headers.get("content-type")?.split(";")[0].trim() ?? null;
-  let dest = join(destDir, filename);
-  if (mimeType && !extname(filename)) {
-    const mimeExt = {
-      "image/jpeg": ".jpg",
-      "image/png": ".png",
-      "image/webp": ".webp",
-      "image/gif": ".gif",
-      "image/svg+xml": ".svg",
-      "image/avif": ".avif",
-      "video/mp4": ".mp4",
-      "video/webm": ".webm",
-      "audio/mpeg": ".mp3",
-      "audio/ogg": ".ogg",
-      "audio/wav": ".wav"
-    };
-    const ext = mimeExt[mimeType];
-    if (ext) dest = dest + ext;
-  }
-  const writer = createWriteStream(dest);
-  await pipeline(Readable.fromWeb(res.body), writer);
-  const { statSync } = await import("fs");
-  const sizeBytes = statSync(dest).size;
-  return { savedPath: dest, sizeBytes, mimeType };
-}
-async function harvestPageMedia(html, pageUrl, options = {}) {
-  const types = options.types ?? ["image", "video", "audio"];
-  const typesSet = new Set(types);
-  const rawUrls = extractMediaUrls(html, pageUrl);
-  const totalFound = rawUrls.length;
-  const filteredUrls = [];
-  const kept = [];
-  for (const url of rawUrls) {
-    const type = typeFromUrl(url);
-    if (!type || !typesSet.has(type)) {
-      filteredUrls.push(url);
-      continue;
-    }
-    kept.push({ url, type });
-  }
-  if (options.outputDir === null) {
-    return {
-      pageUrl,
-      outputDir: null,
-      assets: kept.map(({ url, type }, i) => ({
-        url,
-        type,
-        mimeType: null,
-        filename: safeFilename(url, i),
-        savedPath: null,
-        sizeBytes: null
-      })),
-      filteredCount: filteredUrls.length,
-      totalFound
-    };
-  }
-  const domain = (() => {
-    try {
-      return new URL(pageUrl).hostname.replace(/^www\./, "");
-    } catch {
-      return "unknown";
-    }
-  })();
-  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
-  const outDir = options.outputDir ?? join(homedir(), "Downloads", "mcp-scraper", "media", `${stamp}-${domain}`);
-  mkdirSync(outDir, { recursive: true });
-  const assets = [];
-  await Promise.allSettled(
-    kept.map(async ({ url, type }, i) => {
-      const filename = safeFilename(url, i);
-      try {
-        const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename);
-        const resolvedType = mimeType ? typeFromMime(mimeType) ?? type : type;
-        assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes });
-      } catch {
-        assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null });
-      }
-    })
-  );
-  assets.sort((a, b) => {
-    if (a.savedPath && !b.savedPath) return -1;
-    if (!a.savedPath && b.savedPath) return 1;
-    return a.url.localeCompare(b.url);
-  });
-  return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound };
-}
-
-// src/api/wayback.ts
-import pLimit from "p-limit";
-var WAYBACK_HOSTS = /* @__PURE__ */ new Set(["web.archive.org"]);
-var MAX_CDX_RESPONSE_BYTES = 5 * 1024 * 1024;
-var MAX_CDX_ROWS = 5e3;
-var MAX_ARCHIVE_URL_CHARS = 4096;
-var MAX_SELECTED_TIMELINE_CELLS = 1e3;
-var TIMELINE_CDX_CONCURRENCY = 4;
-function validTimestamp(value) {
-  return /^\d{1,14}$/.test(value);
-}
-function safeHttpUrl(value, base) {
-  try {
-    const parsed = base ? new URL(value, base) : new URL(value);
-    if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
-    parsed.hash = "";
-    return parsed.href;
-  } catch {
-    return null;
-  }
-}
-function parseWaybackReplayUrl(value) {
-  let parsed;
-  try {
-    parsed = new URL(value);
-  } catch {
-    return null;
-  }
-  if (!WAYBACK_HOSTS.has(parsed.hostname.toLowerCase())) return null;
-  const match = parsed.pathname.match(/^\/web\/(\d{1,14})([a-z_]+)?\/(https?:\/\/.+)$/i);
-  if (!match || !validTimestamp(match[1])) return null;
-  const originalUrl = safeHttpUrl(`${match[3]}${parsed.search}`);
-  if (!originalUrl || originalUrl.length > MAX_ARCHIVE_URL_CHARS) return null;
-  const timestamp = match[1];
-  return {
-    timestamp,
-    modifier: match[2] ?? null,
-    originalUrl,
-    replayUrl: `https://web.archive.org/web/${timestamp}/${originalUrl}`,
-    rawReplayUrl: `https://web.archive.org/web/${timestamp}id_/${originalUrl}`
-  };
-}
-function buildWaybackReplayUrl(timestamp, originalUrl, modifier = "id_") {
-  if (!validTimestamp(timestamp)) throw new Error("Wayback timestamp must contain 1-14 digits.");
-  const safeOriginal = safeHttpUrl(originalUrl);
-  if (!safeOriginal || safeOriginal.length > MAX_ARCHIVE_URL_CHARS) throw new Error("Wayback original URL is invalid.");
-  return `https://web.archive.org/web/${timestamp}${modifier}/${safeOriginal}`;
-}
-function timestampMillis(value) {
-  const padded = value.padEnd(14, "0");
-  const year = Number(padded.slice(0, 4));
-  const month = Number(padded.slice(4, 6) || "1");
-  const day = Number(padded.slice(6, 8) || "1");
-  const hour = Number(padded.slice(8, 10) || "0");
-  const minute = Number(padded.slice(10, 12) || "0");
-  const second = Number(padded.slice(12, 14) || "0");
-  return Date.UTC(year, Math.max(0, month - 1), Math.max(1, day), hour, minute, second);
-}
-function monthAnchor(month) {
-  return `${month.replace("-", "")}15120000`;
-}
-function expandWaybackMonths(input) {
-  if (input.months?.length) return [...new Set(input.months)].sort();
-  if (!input.from || !input.to) return [];
-  const [fromYear, fromMonth] = input.from.split("-").map(Number);
-  const [toYear, toMonth] = input.to.split("-").map(Number);
-  const interval = input.intervalMonths ?? 1;
-  const cursor = new Date(Date.UTC(fromYear, fromMonth - 1, 1));
-  const end = new Date(Date.UTC(toYear, toMonth - 1, 1));
-  const months = [];
-  while (cursor <= end && months.length < 60) {
-    months.push(`${cursor.getUTCFullYear()}-${String(cursor.getUTCMonth() + 1).padStart(2, "0")}`);
-    cursor.setUTCMonth(cursor.getUTCMonth() + interval);
-  }
-  return months;
-}
-async function readBoundedText(response, maxBytes) {
-  const reader = response.body?.getReader();
-  if (!reader) return "";
-  const chunks = [];
-  let bytes = 0;
-  for (; ; ) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    bytes += value.byteLength;
-    if (bytes > maxBytes) {
-      await reader.cancel();
-      throw new Error("Wayback CDX response exceeded the safe response limit.");
-    }
-    chunks.push(value);
-  }
-  return Buffer.concat(chunks).toString("utf8");
-}
-async function fetchCdxRaw(params) {
-  const endpoint = new URL("https://web.archive.org/cdx/search/cdx");
-  endpoint.search = params.toString();
-  let raw = "";
-  let lastError;
-  for (let attempt = 0; attempt < 2; attempt++) {
-    try {
-      const response = await fetch(endpoint, {
-        headers: { "User-Agent": "MCP-Scraper/Wayback (+https://mcpscraper.dev)" },
-        signal: AbortSignal.timeout(45e3)
-      });
-      if (!response.ok) {
-        const error = new Error(`Wayback CDX returned HTTP ${response.status}.`);
-        if (attempt === 0 && (response.status === 429 || response.status >= 500)) {
-          lastError = error;
-          continue;
-        }
-        throw error;
-      }
-      raw = await readBoundedText(response, MAX_CDX_RESPONSE_BYTES);
-      lastError = null;
-      break;
-    } catch (error) {
-      lastError = error;
-      if (attempt === 0 && (error instanceof DOMException || error instanceof TypeError)) continue;
-      throw error;
-    }
-  }
-  if (lastError) throw lastError;
-  return raw;
-}
-function parseCdxPage(raw) {
-  let rows;
-  try {
-    rows = JSON.parse(raw);
-  } catch {
-    throw new Error("Wayback CDX returned invalid JSON.");
-  }
-  if (!Array.isArray(rows) || rows.length < 1) return { captures: [], resumeKey: null };
-  const header = Array.isArray(rows[0]) ? rows[0].map(String) : [];
-  const timestampIndex = header.indexOf("timestamp");
-  const originalIndex = header.indexOf("original");
-  const digestIndex = header.indexOf("digest");
-  const statusIndex = header.indexOf("statuscode");
-  const mimeIndex = header.indexOf("mimetype");
-  const lengthIndex = header.indexOf("length");
-  if (timestampIndex < 0 || originalIndex < 0) throw new Error("Wayback CDX response is missing required fields.");
-  const captures = [];
-  let resumeKey = null;
-  let sawResumeSeparator = false;
-  for (const row of rows.slice(1)) {
-    if (!Array.isArray(row)) continue;
-    if (row.length === 0) {
-      sawResumeSeparator = true;
-      continue;
-    }
-    if (sawResumeSeparator && row.length === 1 && typeof row[0] === "string") {
-      resumeKey = row[0];
-      continue;
-    }
-    const timestamp = String(row[timestampIndex] ?? "");
-    const originalUrl = safeHttpUrl(String(row[originalIndex] ?? ""));
-    if (!validTimestamp(timestamp) || !originalUrl || originalUrl.length > MAX_ARCHIVE_URL_CHARS) continue;
-    const statusValue = statusIndex >= 0 ? Number(row[statusIndex]) : NaN;
-    const lengthValue = lengthIndex >= 0 ? Number(row[lengthIndex]) : NaN;
-    captures.push({
-      timestamp,
-      originalUrl,
-      digest: digestIndex >= 0 && row[digestIndex] != null ? String(row[digestIndex]) : null,
-      rawReplayUrl: buildWaybackReplayUrl(timestamp, originalUrl),
-      replayUrl: buildWaybackReplayUrl(timestamp, originalUrl, ""),
-      statusCode: Number.isInteger(statusValue) ? statusValue : null,
-      mimeType: mimeIndex >= 0 && row[mimeIndex] != null ? String(row[mimeIndex]) : null,
-      length: Number.isFinite(lengthValue) && lengthValue >= 0 ? lengthValue : null
-    });
-  }
-  return { captures, resumeKey };
-}
-async function queryCdxPage(baseParams, limit, resumeKey) {
-  const params = new URLSearchParams(baseParams);
-  params.set("limit", String(Math.max(1, Math.min(MAX_CDX_ROWS, Math.floor(limit)))));
-  params.set("showResumeKey", "true");
-  if (resumeKey) params.set("resumeKey", resumeKey);
-  else params.delete("resumeKey");
-  return parseCdxPage(await fetchCdxRaw(params));
-}
-async function queryCdx(params) {
-  return (await queryCdxPage(params, Number(params.get("limit") ?? MAX_CDX_ROWS))).captures;
-}
-function siteHost(value) {
-  return new URL(value).hostname.replace(/^www\./, "").toLowerCase();
-}
-function pageIdentity(value) {
-  const safe = safeHttpUrl(value);
-  if (!safe) return null;
-  const parsed = new URL(safe);
-  parsed.hash = "";
-  parsed.hostname = parsed.hostname.replace(/^www\./, "").toLowerCase();
-  for (const key of [...parsed.searchParams.keys()]) {
-    if (/^(utm_|fbclid$|gclid$|mc_)/i.test(key)) parsed.searchParams.delete(key);
-  }
-  return parsed.href;
-}
-function isLikelyContentPage(value) {
-  const parsed = new URL(value);
-  const path = parsed.pathname.toLowerCase();
-  if (/(?:^|\/)(?:wp-admin|wp-login\.php|wp-json|xmlrpc\.php)(?:\/|$)/.test(path) || /(?:^|\/)(?:author|tag|category|feed|comments|search)(?:\/|$)/.test(path) || /\.(?:css|js|json|xml|txt|ico|svg|png|jpe?g|gif|webp|avif|mp4|mp3|pdf|zip|woff2?|ttf|eot)$/i.test(path)) return false;
-  if ([...parsed.searchParams.keys()].some((key) => /^(?:redirect_to|reauth|action|rest_route|s|replytocom)$/i.test(key))) return false;
-  return true;
-}
-function selectClosestCaptures(captures, replay, maxPages) {
-  const expectedHost = siteHost(replay.originalUrl);
-  const targetTime = timestampMillis(replay.timestamp);
-  const byPage = /* @__PURE__ */ new Map();
-  for (const capture of captures) {
-    if (siteHost(capture.originalUrl) !== expectedHost) continue;
-    if (!isLikelyContentPage(capture.originalUrl)) continue;
-    const identity = pageIdentity(capture.originalUrl);
-    if (!identity) continue;
-    const previous = byPage.get(identity);
-    if (!previous) {
-      byPage.set(identity, capture);
-      continue;
-    }
-    const candidateDistance = Math.abs(timestampMillis(capture.timestamp) - targetTime);
-    const previousDistance = Math.abs(timestampMillis(previous.timestamp) - targetTime);
-    if (candidateDistance < previousDistance) byPage.set(identity, capture);
-  }
-  const root = new URL(replay.originalUrl).origin + "/";
-  return [...byPage.values()].sort((a, b) => {
-    if (a.originalUrl === root) return -1;
-    if (b.originalUrl === root) return 1;
-    const timeDelta = Math.abs(timestampMillis(a.timestamp) - targetTime) - Math.abs(timestampMillis(b.timestamp) - targetTime);
-    return timeDelta || a.originalUrl.localeCompare(b.originalUrl);
-  }).slice(0, maxPages);
-}
-function cdxParams(replay, maxRows, from, to) {
-  const original = new URL(replay.originalUrl);
-  const params = new URLSearchParams({
-    url: `${original.hostname}/*`,
-    output: "json",
-    fl: "timestamp,original,statuscode,mimetype,digest",
-    filter: "statuscode:200",
-    limit: String(Math.min(MAX_CDX_ROWS, Math.max(maxRows, 100)))
-  });
-  params.append("filter", "mimetype:text/html");
-  if (from) params.set("from", from);
-  if (to) params.set("to", to);
-  return params;
-}
-function exactCdxParams(originalUrl, month) {
-  const params = new URLSearchParams({
-    url: originalUrl,
-    output: "json",
-    fl: "timestamp,original,statuscode,mimetype,digest",
-    filter: "statuscode:200",
-    from: month.replace("-", ""),
-    to: month.replace("-", ""),
-    limit: "100"
-  });
-  params.append("filter", "mimetype:text/html");
-  return params;
-}
-function closestCapture(captures, timestamp) {
-  const target = timestampMillis(timestamp);
-  return captures.reduce((best, capture) => {
-    if (!best) return capture;
-    return Math.abs(timestampMillis(capture.timestamp) - target) < Math.abs(timestampMillis(best.timestamp) - target) ? capture : best;
-  }, null);
-}
-async function resolveWaybackSiteCaptures(replay, maxPages) {
-  const safeMaxPages = Math.max(1, Math.min(500, Math.floor(maxPages)));
-  const maxRows = Math.min(MAX_CDX_ROWS, safeMaxPages * 25);
-  const month = replay.timestamp.slice(0, 6);
-  const year = replay.timestamp.slice(0, 4);
-  let captures = await queryCdx(cdxParams(replay, maxRows, month, month));
-  let selected = selectClosestCaptures(captures, replay, safeMaxPages);
-  if (selected.length < Math.min(5, safeMaxPages)) {
-    captures = captures.concat(await queryCdx(cdxParams(replay, maxRows, year, year)));
-    selected = selectClosestCaptures(captures, replay, safeMaxPages);
-  }
-  if (selected.length === 0) {
-    captures = await queryCdx(cdxParams(replay, maxRows, void 0, replay.timestamp));
-    selected = selectClosestCaptures(captures, replay, safeMaxPages);
-  }
-  const exactPage = {
-    timestamp: replay.timestamp,
-    originalUrl: replay.originalUrl,
-    digest: null,
-    rawReplayUrl: replay.rawReplayUrl
-  };
-  const exactIdentity = pageIdentity(exactPage.originalUrl);
-  const withoutDuplicate = selected.filter((capture) => pageIdentity(capture.originalUrl) !== exactIdentity);
-  return [exactPage, ...withoutDuplicate].slice(0, safeMaxPages);
-}
-async function resolveWaybackTimelineCaptures(input) {
-  const rootUrl = safeHttpUrl(input.rootUrl);
-  if (!rootUrl) throw new Error("Wayback timeline root URL is invalid.");
-  const rootHost = siteHost(rootUrl);
-  const months = expandWaybackMonths(input.timeline);
-  if (months.length === 0) throw new Error("At least one Wayback month is required.");
-  const maxPagesPerSnapshot = Math.max(1, Math.min(500, Math.floor(input.maxPagesPerSnapshot)));
-  const maxCaptures = Math.max(1, Math.min(1e4, Math.floor(input.maxCaptures ?? 1e4)));
-  const requestedUrls = input.timeline.urls ? [...new Set(input.timeline.urls.map((url) => safeHttpUrl(url)).filter((url) => Boolean(url)))].slice(0, maxPagesPerSnapshot) : null;
-  if (requestedUrls?.some((url) => siteHost(url) !== rootHost)) {
-    throw new Error("Every selected Wayback URL must belong to the same site as the root URL.");
-  }
-  if (requestedUrls && requestedUrls.length * months.length > MAX_SELECTED_TIMELINE_CELLS) {
-    throw new Error(`Selected-page Wayback timelines are limited to ${MAX_SELECTED_TIMELINE_CELLS} page-month cells per job.`);
-  }
-  const captures = [];
-  const missing = [];
-  if (requestedUrls) {
-    const limit = pLimit(TIMELINE_CDX_CONCURRENCY);
-    const cells = months.flatMap((requestedMonth) => requestedUrls.map((originalUrl) => ({ requestedMonth, originalUrl }))).slice(0, maxCaptures);
-    const resolved = await Promise.all(cells.map((cell) => limit(async () => {
-      const rows = await queryCdx(exactCdxParams(cell.originalUrl, cell.requestedMonth));
-      const capture = closestCapture(
-        rows.filter((row) => pageIdentity(row.originalUrl) === pageIdentity(cell.originalUrl)),
-        monthAnchor(cell.requestedMonth)
-      );
-      return { ...cell, capture };
-    })));
-    for (const cell of resolved) {
-      if (!cell.capture) {
-        missing.push({ requestedMonth: cell.requestedMonth, originalUrl: cell.originalUrl });
-        continue;
-      }
-      captures.push({ ...cell.capture, requestedMonth: cell.requestedMonth });
-    }
-  } else {
-    for (const requestedMonth of months) {
-      const replay = {
-        timestamp: monthAnchor(requestedMonth),
-        modifier: null,
-        originalUrl: rootUrl,
-        replayUrl: buildWaybackReplayUrl(monthAnchor(requestedMonth), rootUrl, ""),
-        rawReplayUrl: buildWaybackReplayUrl(monthAnchor(requestedMonth), rootUrl)
-      };
-      const month = requestedMonth.replace("-", "");
-      const rows = await queryCdx(cdxParams(replay, Math.min(MAX_CDX_ROWS, maxPagesPerSnapshot * 25), month, month));
-      const selected = selectClosestCaptures(rows, replay, maxPagesPerSnapshot);
-      if (selected.length === 0) {
-        missing.push({ requestedMonth, originalUrl: rootUrl });
-        continue;
-      }
-      captures.push(...selected.map((capture) => ({ ...capture, requestedMonth })));
-      if (captures.length >= maxCaptures) break;
-    }
-  }
-  return {
-    months,
-    captures: captures.slice(0, maxCaptures),
-    requestedUrls,
-    missing
-  };
-}
-function normalizeCdxRangeValue(value) {
-  if (!value) return null;
-  const normalized = value.replace(/-/g, "");
-  if (!/^(?:\d{4}|\d{6}|\d{8}|\d{14})$/.test(normalized)) {
-    throw new Error("Wayback date ranges must use YYYY, YYYY-MM, YYYY-MM-DD, or a 14-digit timestamp.");
-  }
-  return normalized;
-}
-function monthRange(from, to) {
-  if (!from || !to || from.length < 6 || to.length < 6) return [];
-  const start = new Date(Date.UTC(Number(from.slice(0, 4)), Number(from.slice(4, 6)) - 1, 1));
-  const end = new Date(Date.UTC(Number(to.slice(0, 4)), Number(to.slice(4, 6)) - 1, 1));
-  const months = [];
-  while (start <= end && months.length < 600) {
-    months.push(`${start.getUTCFullYear()}-${String(start.getUTCMonth() + 1).padStart(2, "0")}`);
-    start.setUTCMonth(start.getUTCMonth() + 1);
-  }
-  return months;
-}
-function inventoryCdxParams(input) {
-  const parsed = new URL(input.url);
-  const target = input.scope === "host" || input.scope === "domain" ? parsed.hostname : parsed.href;
-  const params = new URLSearchParams({
-    url: target,
-    output: "json",
-    fl: "timestamp,original,statuscode,mimetype,digest,length"
-  });
-  if (input.scope !== "exact") params.set("matchType", input.scope);
-  if (input.from) params.set("from", input.from);
-  if (input.to) params.set("to", input.to);
-  if (input.successfulHtmlOnly) {
-    params.append("filter", "statuscode:200");
-    params.append("filter", "mimetype:text/html");
-  }
-  return params;
-}
-async function inventoryWaybackSnapshots(input) {
-  const startedAt = Date.now();
-  const replay = parseWaybackReplayUrl(input.url);
-  const rootUrl = safeHttpUrl(replay?.originalUrl ?? input.url);
-  if (!rootUrl) throw new Error("Wayback inventory URL is invalid.");
-  const scope = input.scope ?? "exact";
-  const from = normalizeCdxRangeValue(input.from);
-  const to = normalizeCdxRangeValue(input.to);
-  if (from && to && from > to) throw new Error("Wayback inventory from must not be after to.");
-  const successfulHtmlOnly = input.successfulHtmlOnly !== false;
-  const maxCaptures = Math.max(1, Math.min(1e5, Math.floor(input.maxCaptures ?? 1e4)));
-  const maxCaptureRows = Math.max(0, Math.min(1e3, Math.floor(input.maxCaptureRows ?? 500)));
-  const selectedUrls = input.urls?.length ? [...new Set(input.urls.map((url) => safeHttpUrl(url)).filter((url) => Boolean(url)))] : null;
-  const rootHost = siteHost(rootUrl);
-  if (selectedUrls?.some((url) => siteHost(url) !== rootHost)) {
-    throw new Error("Every selected Wayback inventory URL must belong to the same site as url.");
-  }
-  const targets = selectedUrls ?? [rootUrl];
-  const effectiveScope = selectedUrls ? "exact" : scope;
-  const monthly = /* @__PURE__ */ new Map();
-  const yearly = /* @__PURE__ */ new Map();
-  const digests = /* @__PURE__ */ new Set();
-  const perUrlState = /* @__PURE__ */ new Map();
-  const captureRows = [];
-  let totalCaptures = 0;
-  let queryPages = 0;
-  let firstCapture = null;
-  let lastCapture = null;
-  let truncated = false;
-  targetLoop:
-    for (let targetIndex = 0; targetIndex < targets.length; targetIndex++) {
-      const target = targets[targetIndex];
-      const params = inventoryCdxParams({
-        url: target,
-        scope: effectiveScope,
-        from,
-        to,
-        successfulHtmlOnly
-      });
-      let resumeKey;
-      do {
-        const remaining = maxCaptures - totalCaptures;
-        if (remaining <= 0) {
-          truncated = true;
-          break targetLoop;
-        }
-        const page = await queryCdxPage(params, Math.min(MAX_CDX_ROWS, remaining), resumeKey);
-        queryPages++;
-        for (const capture of page.captures) {
-          totalCaptures++;
-          const month = `${capture.timestamp.slice(0, 4)}-${capture.timestamp.slice(4, 6)}`;
-          const year = capture.timestamp.slice(0, 4);
-          monthly.set(month, (monthly.get(month) ?? 0) + 1);
-          yearly.set(year, (yearly.get(year) ?? 0) + 1);
-          if (capture.digest) digests.add(capture.digest);
-          const state = perUrlState.get(capture.originalUrl) ?? {
-            captures: 0,
-            digests: /* @__PURE__ */ new Set(),
-            firstTimestamp: capture.timestamp,
-            lastTimestamp: capture.timestamp
-          };
-          state.captures++;
-          if (capture.digest) state.digests.add(capture.digest);
-          if (capture.timestamp < state.firstTimestamp) state.firstTimestamp = capture.timestamp;
-          if (capture.timestamp > state.lastTimestamp) state.lastTimestamp = capture.timestamp;
-          perUrlState.set(capture.originalUrl, state);
-          if (!firstCapture || capture.timestamp < firstCapture.timestamp) firstCapture = capture;
-          if (!lastCapture || capture.timestamp > lastCapture.timestamp) lastCapture = capture;
-          if (input.includeCaptures === true && captureRows.length < maxCaptureRows) captureRows.push(capture);
-        }
-        resumeKey = page.resumeKey ?? void 0;
-        if (resumeKey && totalCaptures >= maxCaptures) {
-          truncated = true;
-          break targetLoop;
-        }
-      } while (resumeKey);
-      if (targetIndex < targets.length - 1 && totalCaptures >= maxCaptures) {
-        truncated = true;
-        break;
-      }
-    }
-  const monthlyCounts = [...monthly.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([month, captures]) => ({ month, captures }));
-  const yearlyCounts = [...yearly.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([year, captures]) => ({ year, captures }));
-  const allPerUrl = [...perUrlState.entries()].map(([url, state]) => ({
-    url,
-    captures: state.captures,
-    uniqueDigests: state.digests.size,
-    firstTimestamp: state.firstTimestamp,
-    lastTimestamp: state.lastTimestamp
-  })).sort((a, b) => b.captures - a.captures || a.url.localeCompare(b.url));
-  const availableMonths = new Set(monthly.keys());
-  return {
-    url: rootUrl,
-    scope: effectiveScope,
-    selectedUrls,
-    from,
-    to,
-    successfulHtmlOnly,
-    totalCaptures,
-    countType: truncated ? "lower_bound" : "exact",
-    complete: !truncated,
-    truncated,
-    maxCaptures,
-    queryPages,
-    uniqueUrls: perUrlState.size,
-    uniqueDigests: digests.size,
-    firstCapture,
-    lastCapture,
-    monthlyCounts,
-    yearlyCounts,
-    missingMonths: monthRange(from, to).filter((month) => !availableMonths.has(month)),
-    perUrl: allPerUrl.slice(0, 500),
-    perUrlTruncatedCount: Math.max(0, allPerUrl.length - 500),
-    captures: captureRows,
-    captureRowsTruncatedCount: input.includeCaptures === true ? Math.max(0, totalCaptures - captureRows.length) : totalCaptures,
-    durationMs: Date.now() - startedAt
-  };
-}
-function firstJsonLdImage(value) {
-  if (typeof value === "string") return value;
-  if (Array.isArray(value)) {
-    for (const item of value) {
-      const found = firstJsonLdImage(item);
-      if (found) return found;
-    }
-    return null;
-  }
-  if (!value || typeof value !== "object") return null;
-  const record = value;
-  for (const key of ["url", "contentUrl"]) {
-    if (typeof record[key] === "string") return record[key];
-  }
-  for (const key of ["image", "thumbnailUrl", "@graph"]) {
-    const found = firstJsonLdImage(record[key]);
-    if (found) return found;
-  }
-  return null;
-}
-function firstContentImage(html) {
-  for (const match of html.matchAll(/<img\b[^>]*>/gi)) {
-    const tag = match[0];
-    const attr = (name) => tag.match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]+)"|'([^']+)'|([^\\s>]+))`, "i"))?.slice(1).find(Boolean) ?? null;
-    const candidate = attr("data-src") ?? attr("src") ?? attr("data-lazy-src");
-    if (!candidate || /^(?:data:|javascript:)/i.test(candidate)) continue;
-    const descriptor = `${candidate} ${attr("alt") ?? ""} ${attr("class") ?? ""}`.toLowerCase();
-    if (/(?:logo|icon|avatar|sprite|pixel|tracking|spacer)/.test(descriptor)) continue;
-    const width = Number(attr("width") ?? 0);
-    const height = Number(attr("height") ?? 0);
-    if (width > 0 && width < 200 || height > 0 && height < 120) continue;
-    return candidate;
-  }
-  return null;
-}
-function resolveFeaturedImage(input) {
-  const replay = parseWaybackReplayUrl(input.pageUrl);
-  const baseUrl = replay?.originalUrl ?? input.pageUrl;
-  const candidates = [
-    { value: input.meta["og:image"] ?? input.meta["og:image:url"] ?? input.meta["og:image:secure_url"], source: "og:image" },
-    { value: input.meta["twitter:image"] ?? input.meta["twitter:image:src"], source: "twitter:image" },
-    { value: firstJsonLdImage(input.schema), source: "json-ld" },
-    { value: firstContentImage(input.html), source: "content-image" }
-  ];
-  for (const candidate of candidates) {
-    if (!candidate.value) continue;
-    const url = safeHttpUrl(candidate.value, baseUrl);
-    if (!url) continue;
-    return {
-      url,
-      archivedUrl: replay ? buildWaybackReplayUrl(replay.timestamp, url, "im_") : null,
-      source: candidate.source
-    };
-  }
-  return null;
-}
-
-export {
-  downloadAsset,
-  harvestPageMedia,
-  parseWaybackReplayUrl,
-  expandWaybackMonths,
-  resolveWaybackSiteCaptures,
-  resolveWaybackTimelineCaptures,
-  inventoryWaybackSnapshots,
-  resolveFeaturedImage
-};
-//# sourceMappingURL=chunk-YV2FUEBX.js.map