mcp-scraper 0.51.2 → 0.52.0

package/dist/chunk-SG2PEHR3.js

@@ -0,0 +1,562 @@
+import {
+  createPrivateArtifactFromStream,
+  privateArtifactOwnerId,
+  readPrivateArtifactBuffer,
+  renewPrivateArtifactDownload,
+  validatePublicHttpUrl
+} from "./chunk-2K74LVV7.js";
+
+// src/lib/media-extractor.ts
+import { createWriteStream, mkdirSync, rmSync } from "fs";
+import { homedir } from "os";
+import { join, extname, basename } from "path";
+import { pipeline } from "stream/promises";
+import { Readable, Transform } from "stream";
+var AD_PATTERNS = [
+  "doubleclick.net",
+  "googlesyndication.com",
+  "googletagmanager.com",
+  "google-analytics.com",
+  "googletagservices.com",
+  "adservice.google",
+  "googletag.",
+  "pagead2.googlesyndication",
+  "facebook.net/tr",
+  "connect.facebook.net",
+  "fbcdn.net/rsrc",
+  "analytics.twitter.com",
+  "static.ads-twitter.com",
+  "ads.twitter.com",
+  "t.co/i/adsct",
+  "pixel.advertising.com",
+  "hotjar.com",
+  "clarity.ms",
+  "quantserve.com",
+  "scorecardresearch.com",
+  "newrelic.com",
+  "nr-data.net",
+  "segment.io",
+  "segment.com",
+  "amplitude.com",
+  "mixpanel.com",
+  "heap.io",
+  "fullstory.com",
+  "moatads.com",
+  "criteo.com",
+  "adsrvr.org",
+  "rubiconproject.com",
+  "pubmatic.com",
+  "openx.net",
+  "appnexus.com",
+  "amazon-adsystem.com",
+  "media.net",
+  "yieldmo.com",
+  "triplelift.com",
+  "sharethrough.com",
+  "prebid.",
+  "smaato.net",
+  "indexworm.com",
+  "casalemedia.com",
+  "outbrain.com",
+  "taboola.com",
+  "revcontent.com",
+  "mgid.com",
+  "tawk.to",
+  "intercom.io",
+  "drift.com",
+  "hs-scripts.com",
+  "zopim.com",
+  "livechatinc.com",
+  "userlike.com",
+  "onetrust.com",
+  "cookielaw.org",
+  "cookieinformation.com",
+  "trustarc.com",
+  "/ads/",
+  "/ad/",
+  "/banner/",
+  "/banners/",
+  "/pixel/",
+  "/beacon/",
+  "/tracking/",
+  "/tracker/",
+  "/remarketing/",
+  "/conversion/",
+  "1x1.gif",
+  "spacer.gif",
+  "blank.gif",
+  "transparent.gif"
+];
+var IMAGE_EXTS = /* @__PURE__ */ new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".svg", ".tiff"]);
+var VIDEO_EXTS = /* @__PURE__ */ new Set([".mp4", ".webm", ".mov", ".avi", ".m4v", ".ogv", ".mkv"]);
+var AUDIO_EXTS = /* @__PURE__ */ new Set([".mp3", ".wav", ".ogg", ".aac", ".m4a", ".flac", ".opus"]);
+function isAdUrl(url) {
+  const lower = url.toLowerCase();
+  return AD_PATTERNS.some((p) => lower.includes(p));
+}
+function isDataUri(url) {
+  return url.startsWith("data:");
+}
+function typeFromUrl(url) {
+  try {
+    const ext = extname(new URL(url).pathname).toLowerCase();
+    if (IMAGE_EXTS.has(ext)) return "image";
+    if (VIDEO_EXTS.has(ext)) return "video";
+    if (AUDIO_EXTS.has(ext)) return "audio";
+  } catch {
+  }
+  return null;
+}
+function typeFromMime(mime) {
+  const lower = mime.toLowerCase();
+  if (lower.startsWith("image/")) return "image";
+  if (lower.startsWith("video/")) return "video";
+  if (lower.startsWith("audio/")) return "audio";
+  return null;
+}
+function resolveUrl(raw, base) {
+  if (!raw || isDataUri(raw)) return null;
+  try {
+    return new URL(raw, base).href;
+  } catch {
+    return null;
+  }
+}
+function safeFilename(url, index) {
+  try {
+    const u = new URL(url);
+    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
+    return base || `asset-${index}`;
+  } catch {
+    return `asset-${index}`;
+  }
+}
+function boundedText(value, max) {
+  const normalized = value?.replace(/&(?:amp|#0?38|#x26);/gi, "&").replace(/\s+/g, " ").trim();
+  return normalized ? normalized.slice(0, max) : null;
+}
+function dimensionsFromUrl(rawUrl) {
+  const match = rawUrl.match(/[-_/](\d{2,5})x(\d{2,5})(?=[._/?#-]|$)/i);
+  return match ? { width: Number(match[1]), height: Number(match[2]) } : null;
+}
+function mediaFamilyKey(rawUrl) {
+  try {
+    const url = new URL(rawUrl);
+    url.hash = "";
+    url.pathname = url.pathname.replace(/-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}$)/i, "");
+    for (const key of ["w", "width", "h", "height", "resize"]) url.searchParams.delete(key);
+    url.searchParams.sort();
+    return url.href;
+  } catch {
+    return rawUrl;
+  }
+}
+function candidateScore(candidate) {
+  const area = (candidate.width ?? 0) * (candidate.height ?? 0);
+  const methodBoost = candidate.discoveryMethods.some((method) => method.includes("lightbox")) ? 1e12 : 0;
+  const originalBoost = /-(?:\d{2,5}x\d{2,5}|scaled|e\d{6,})(?=\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(candidate.url) ? 0 : 1e11;
+  return methodBoost + originalBoost + area;
+}
+function mergeCandidate(target, incoming) {
+  const existing = target.get(incoming.url);
+  if (!existing) {
+    target.set(incoming.url, incoming);
+    return;
+  }
+  existing.discoveryMethods = [.../* @__PURE__ */ new Set([...existing.discoveryMethods, ...incoming.discoveryMethods])];
+  existing.altTexts = [.../* @__PURE__ */ new Set([...existing.altTexts, ...incoming.altTexts])];
+  existing.contexts = [.../* @__PURE__ */ new Set([...existing.contexts, ...incoming.contexts])];
+  if (!existing.type && incoming.type) existing.type = incoming.type;
+  if ((incoming.width ?? 0) * (incoming.height ?? 0) > (existing.width ?? 0) * (existing.height ?? 0)) {
+    existing.width = incoming.width;
+    existing.height = incoming.height;
+  }
+}
+function discoverStaticMedia(html, baseUrl) {
+  const found = /* @__PURE__ */ new Map();
+  const add = (raw, method, explicitType = null, alt, context, dimensions) => {
+    const resolved = raw ? resolveUrl(raw.trim().replace(/&/g, "&"), baseUrl) : null;
+    if (!resolved || isAdUrl(resolved)) return;
+    const inferredDimensions = dimensions ?? dimensionsFromUrl(resolved);
+    mergeCandidate(found, {
+      url: resolved,
+      type: explicitType ?? typeFromUrl(resolved),
+      discoveryMethods: [method],
+      altTexts: alt ? [alt] : [],
+      contexts: context ? [context] : [],
+      width: inferredDimensions?.width ?? null,
+      height: inferredDimensions?.height ?? null
+    });
+  };
+  const scanSrcset = (value, method, type, alt) => {
+    for (const part of (value ?? "").split(",")) {
+      const [url, descriptor] = part.trim().split(/\s+/);
+      if (!url) continue;
+      const width = descriptor?.match(/^(\d+)w$/)?.[1];
+      add(url, method, type, alt, null, width ? { width: Number(width), height: 0 } : null);
+    }
+  };
+  const scan = (source, suffix = "") => {
+    for (const match of source.matchAll(/<img\b([^>]*)>/gi)) {
+      const attrs = match[1];
+      const alt = boundedText(attrs.match(/\balt\s*=\s*["']([^"']*)["']/i)?.[1], 500);
+      const width = Number(attrs.match(/\bwidth\s*=\s*["']?(\d+)/i)?.[1] ?? 0);
+      const height = Number(attrs.match(/\bheight\s*=\s*["']?(\d+)/i)?.[1] ?? 0);
+      const dimensions = width > 0 && height > 0 ? { width, height } : null;
+      for (const attr of ["src", "data-src", "data-lazy-src", "data-original", "data-bg", "data-background", "data-bg-url", "data-lazy-bg", "data-echo"]) {
+        const value = attrs.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i"))?.[1];
+        add(value, `static-${attr}${suffix}`, "image", alt, null, dimensions);
+      }
+      scanSrcset(attrs.match(/\bsrcset\s*=\s*["']([^"']+)["']/i)?.[1], `static-srcset${suffix}`, "image", alt);
+      scanSrcset(attrs.match(/\bdata-srcset\s*=\s*["']([^"']+)["']/i)?.[1], `static-data-srcset${suffix}`, "image", alt);
+    }
+    for (const match of source.matchAll(/<source\b([^>]*)>/gi)) {
+      const attrs = match[1];
+      const declared = attrs.match(/\btype\s*=\s*["']([^"']+)["']/i)?.[1] ?? "";
+      const type = declared.startsWith("video/") ? "video" : declared.startsWith("audio/") ? "audio" : "image";
+      add(attrs.match(/\bsrc\s*=\s*["']([^"']+)["']/i)?.[1], `static-source${suffix}`, type);
+      scanSrcset(attrs.match(/\bsrcset\s*=\s*["']([^"']+)["']/i)?.[1], `static-source-srcset${suffix}`, type);
+    }
+    for (const match of source.matchAll(/<(video|audio)\b([^>]*)>/gi)) {
+      const type = match[1].toLowerCase();
+      const attrs = match[2];
+      add(attrs.match(/\bsrc\s*=\s*["']([^"']+)["']/i)?.[1], `static-${type}${suffix}`, type);
+      if (type === "video") add(attrs.match(/\bposter\s*=\s*["']([^"']+)["']/i)?.[1], `static-video-poster${suffix}`, "image");
+    }
+    for (const match of source.matchAll(/<meta\b([^>]*)>/gi)) {
+      const attrs = match[1];
+      const prop = attrs.match(/\b(?:property|name)\s*=\s*["']([^"']+)["']/i)?.[1]?.toLowerCase();
+      if (prop === "og:image" || prop === "og:image:url" || prop === "twitter:image" || prop === "twitter:image:src") {
+        add(attrs.match(/\bcontent\s*=\s*["']([^"']+)["']/i)?.[1], `static-${prop}${suffix}`, "image");
+      }
+    }
+    for (const match of source.matchAll(/<link\b([^>]*)>/gi)) {
+      const attrs = match[1];
+      if (/\brel\s*=\s*["'][^"']*(?:icon|apple-touch-icon)[^"']*["']/i.test(attrs)) {
+        add(attrs.match(/\bhref\s*=\s*["']([^"']+)["']/i)?.[1], `static-icon${suffix}`, "image");
+      }
+    }
+    for (const match of source.matchAll(/<image\b([^>]*)>/gi)) {
+      const attrs = match[1];
+      add(attrs.match(/\b(?:href|xlink:href)\s*=\s*["']([^"']+)["']/i)?.[1], `static-svg-image${suffix}`, "image");
+    }
+    for (const match of source.matchAll(/<a\b([^>]*)>[\s\S]*?<img\b([^>]*)>[\s\S]*?<\/a>/gi)) {
+      const anchorAttrs = match[1];
+      const href = anchorAttrs.match(/\bhref\s*=\s*["']([^"']+)["']/i)?.[1];
+      if (!href) continue;
+      if (/lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(anchorAttrs) || /\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(href)) {
+        const alt = boundedText(match[2].match(/\balt\s*=\s*["']([^"']*)["']/i)?.[1], 500);
+        add(href, `static-lightbox-target${suffix}`, "image", alt);
+      }
+    }
+    for (const match of source.matchAll(/(?:background(?:-image)?\s*:\s*)?url\(\s*(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?([^"')&]+)(?:["']|&quot;|&#0?34;|&apos;|&#0?39;)?\s*\)/gi)) {
+      add(match[1], `static-css-url${suffix}`, "image");
+    }
+    for (const match of source.matchAll(/["'](https?:\/\/[^"'\s<>]+\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\?[^"'\s<>]*)?)["']/gi)) {
+      add(match[1], `static-bare-url${suffix}`);
+    }
+  };
+  scan(html);
+  const unescaped = html.replace(/\\u003[cC]/g, "<").replace(/\\u003[eE]/g, ">").replace(/\\u0026/gi, "&").replace(/\\["']/g, (match) => match.slice(1)).replace(/\\\//g, "/");
+  if (unescaped !== html) scan(unescaped, "-json-unescaped");
+  return [...found.values()];
+}
+function mergeMediaDiscovery(staticAssets, rendered, maxAssets = 100) {
+  const exact = /* @__PURE__ */ new Map();
+  for (const asset of staticAssets) mergeCandidate(exact, asset);
+  for (const asset of rendered?.assets ?? []) mergeCandidate(exact, { ...asset, type: asset.type });
+  const families = /* @__PURE__ */ new Map();
+  for (const asset of exact.values()) {
+    const key = mediaFamilyKey(asset.url);
+    const family = families.get(key) ?? [];
+    family.push(asset);
+    families.set(key, family);
+  }
+  return [...families.values()].map((family) => {
+    const ordered = [...family].sort((a, b) => candidateScore(b) - candidateScore(a) || a.url.localeCompare(b.url));
+    const preferred = { ...ordered[0] };
+    preferred.discoveryMethods = [...new Set(family.flatMap((asset) => asset.discoveryMethods))];
+    preferred.altTexts = [...new Set(family.flatMap((asset) => asset.altTexts))];
+    preferred.contexts = [...new Set(family.flatMap((asset) => asset.contexts))];
+    return { ...preferred, variants: ordered.slice(1).map((asset) => asset.url) };
+  }).sort((a, b) => candidateScore(b) - candidateScore(a) || a.url.localeCompare(b.url)).slice(0, Math.max(1, maxAssets));
+}
+async function downloadAsset(url, destDir, filename, options = {}) {
+  const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024));
+  let target = url;
+  let res = null;
+  for (let redirects = 0; redirects <= 5; redirects++) {
+    const checked = await validatePublicHttpUrl(target, { field: "media URL" });
+    if (checked.error || !checked.parsed) throw new Error(checked.error ?? "Media URL was rejected");
+    res = await fetch(checked.parsed.href, {
+      signal: AbortSignal.timeout(15e3),
+      redirect: "manual"
+    });
+    if (res.status >= 300 && res.status < 400) {
+      const location = res.headers.get("location");
+      if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`);
+      target = new URL(location, checked.parsed.href).href;
+      res = null;
+      continue;
+    }
+    break;
+  }
+  if (!res) throw new Error("Media download exceeded five redirects");
+  if (!res.ok) throw new Error(`HTTP ${res.status}`);
+  if (!res.body) throw new Error("Empty response body");
+  const mimeType = res.headers.get("content-type")?.split(";")[0].trim() ?? null;
+  if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {
+    throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? "no content-type"}`);
+  }
+  const declaredBytes = Number(res.headers.get("content-length") ?? 0);
+  if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {
+    throw new Error(`Media asset exceeds ${maxBytes} byte limit`);
+  }
+  let dest = join(destDir, filename);
+  if (mimeType && !extname(filename)) {
+    const mimeExt = {
+      "image/jpeg": ".jpg",
+      "image/png": ".png",
+      "image/webp": ".webp",
+      "image/gif": ".gif",
+      "image/svg+xml": ".svg",
+      "image/avif": ".avif",
+      "video/mp4": ".mp4",
+      "video/webm": ".webm",
+      "audio/mpeg": ".mp3",
+      "audio/ogg": ".ogg",
+      "audio/wav": ".wav"
+    };
+    const ext = mimeExt[mimeType];
+    if (ext) dest = dest + ext;
+  }
+  const writer = createWriteStream(dest);
+  await new Promise((resolve, reject) => {
+    writer.once("open", () => resolve());
+    writer.once("error", reject);
+  });
+  let streamedBytes = 0;
+  const limiter = new Transform({
+    transform(chunk, _encoding, callback) {
+      const bytes = chunk.length;
+      if (streamedBytes + bytes > maxBytes) {
+        callback(new Error(`Media asset exceeds ${maxBytes} byte limit`));
+        return;
+      }
+      if (options.consumeBytes && !options.consumeBytes(bytes)) {
+        callback(new Error("Media export total byte limit exceeded"));
+        return;
+      }
+      streamedBytes += bytes;
+      callback(null, chunk);
+    }
+  });
+  try {
+    await pipeline(Readable.fromWeb(res.body), limiter, writer);
+  } catch (error) {
+    writer.destroy();
+    rmSync(dest, { force: true });
+    throw error;
+  }
+  const { statSync } = await import("fs");
+  const sizeBytes = statSync(dest).size;
+  return { savedPath: dest, sizeBytes, mimeType };
+}
+async function harvestPageMedia(html, pageUrl, options = {}) {
+  const types = options.types ?? ["image", "video", "audio"];
+  const typesSet = new Set(types);
+  const staticAssets = discoverStaticMedia(html, pageUrl);
+  const maxAssets = Math.max(1, Math.min(options.maxAssets ?? 100, 250));
+  const allDiscovered = mergeMediaDiscovery(staticAssets, options.rendered, 2e3);
+  const discovered = allDiscovered.slice(0, maxAssets);
+  const totalFound = (/* @__PURE__ */ new Set([
+    ...staticAssets.map((asset) => asset.url),
+    ...(options.rendered?.assets ?? []).map((asset) => asset.url)
+  ])).size;
+  const kept = discovered.flatMap((asset) => {
+    const type = asset.type ?? typeFromUrl(asset.url);
+    return type && typesSet.has(type) ? [{ ...asset, type }] : [];
+  });
+  const filteredCount = totalFound - kept.length;
+  const retentionLimitReached = allDiscovered.length > maxAssets;
+  const exhausted = options.rendered?.exhausted === true && !retentionLimitReached;
+  const stopReason = retentionLimitReached ? "asset_limit" : options.rendered?.stopReason ?? "render_unavailable";
+  const warnings = retentionLimitReached ? [`Media discovery found ${allDiscovered.length} responsive families; only the requested ${maxAssets} records were retained.`] : options.rendered ? options.rendered.exhausted ? [] : [`Rendered media discovery stopped with ${options.rendered.stopReason}; the inventory may be incomplete.`] : ["Rendered media discovery was unavailable; this manifest contains static-source evidence only."];
+  const baseManifest = {
+    pageUrl,
+    staticFound: staticAssets.length,
+    renderedFound: options.rendered?.assets.length ?? 0,
+    filteredCount,
+    totalFound,
+    completeness: exhausted ? "complete" : "partial",
+    exhausted,
+    stopReason,
+    scrollRounds: options.rendered?.scrollRounds ?? 0,
+    warnings,
+    artifact: null
+  };
+  if (options.outputDir === null) {
+    return {
+      ...baseManifest,
+      outputDir: null,
+      assets: kept.map((asset, i) => ({
+        ...asset,
+        mimeType: null,
+        filename: safeFilename(asset.url, i),
+        savedPath: null,
+        sizeBytes: null,
+        sha256: null,
+        downloadStatus: "not_attempted",
+        downloadError: null,
+        inlinePreview: null
+      }))
+    };
+  }
+  const domain = (() => {
+    try {
+      return new URL(pageUrl).hostname.replace(/^www\./, "");
+    } catch {
+      return "unknown";
+    }
+  })();
+  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
+  const outDir = options.outputDir ?? join(homedir(), "Downloads", "mcp-scraper", "media", `${stamp}-${domain}`);
+  mkdirSync(outDir, { recursive: true });
+  const assets = [];
+  await Promise.allSettled(
+    kept.map(async (asset, i) => {
+      const filename = safeFilename(asset.url, i);
+      try {
+        const { savedPath, sizeBytes, mimeType } = await downloadAsset(asset.url, outDir, filename);
+        const resolvedType = mimeType ? typeFromMime(mimeType) ?? asset.type : asset.type;
+        assets.push({
+          ...asset,
+          type: resolvedType,
+          mimeType,
+          filename: basename(savedPath),
+          savedPath,
+          sizeBytes,
+          sha256: null,
+          downloadStatus: "downloaded",
+          downloadError: null,
+          inlinePreview: null
+        });
+      } catch (error) {
+        assets.push({
+          ...asset,
+          mimeType: null,
+          filename,
+          savedPath: null,
+          sizeBytes: null,
+          sha256: null,
+          downloadStatus: "failed",
+          downloadError: error instanceof Error ? error.message : "media_download_failed",
+          inlinePreview: null
+        });
+      }
+    })
+  );
+  assets.sort((a, b) => {
+    if (a.savedPath && !b.savedPath) return -1;
+    if (!a.savedPath && b.savedPath) return 1;
+    return a.url.localeCompare(b.url);
+  });
+  return { ...baseManifest, outputDir: outDir, assets };
+}
+
+// src/api/site-extract-artifacts.ts
+var SITE_EXTRACT_ARTIFACT_PREFIX = "site-extracts/";
+var SITE_EXTRACT_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
+var SITE_EXTRACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
+function siteExtractArtifactToken() {
+  return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
+}
+function hostedByEnvironment() {
+  return process.env.VERCEL === "1" || process.env.NODE_ENV === "production";
+}
+function policy() {
+  return {
+    prefix: SITE_EXTRACT_ARTIFACT_PREFIX,
+    artifactTtlMs: SITE_EXTRACT_ARTIFACT_TTL_MS,
+    downloadTtlMs: SITE_EXTRACT_DOWNLOAD_TTL_MS,
+    token: siteExtractArtifactToken()
+  };
+}
+function siteExtractArtifactOwnerId(artifactId) {
+  return privateArtifactOwnerId(artifactId, SITE_EXTRACT_ARTIFACT_PREFIX);
+}
+async function createSiteExtractBundleArtifactStream(args) {
+  const pointer = await createPrivateArtifactFromStream({
+    policy: policy(),
+    ownerId: args.ownerId,
+    artifactKey: `${args.jobId}.zip`,
+    createdAt: args.createdAt,
+    filename: `${args.jobId}-site-export.zip`,
+    contentType: "application/zip",
+    content: args.content
+  });
+  return {
+    key: pointer.artifactId,
+    url: pointer.downloadUrl ?? "",
+    bytes: pointer.bytes,
+    contentType: pointer.contentType,
+    filename: pointer.filename,
+    sha256: pointer.sha256,
+    expiresAt: pointer.expiresAt,
+    downloadUrlExpiresAt: pointer.downloadUrlExpiresAt
+  };
+}
+async function renewSiteExtractArtifactDownload(args) {
+  return renewPrivateArtifactDownload({
+    policy: policy(),
+    artifactId: args.artifactId,
+    ownerId: args.ownerId
+  });
+}
+async function readSiteExtractArtifactBuffer(artifactId) {
+  return readPrivateArtifactBuffer({
+    policy: policy(),
+    artifactId,
+    maxBytes: 50 * 1024 * 1024
+  });
+}
+async function readOwnedSiteExtractArtifactBuffer(args) {
+  if (siteExtractArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
+  return readSiteExtractArtifactBuffer(args.artifactId);
+}
+async function cleanupExpiredSiteExtractArtifacts(args = {}) {
+  const now = args.now ?? /* @__PURE__ */ new Date();
+  const token = siteExtractArtifactToken();
+  if (!token) return { deleted: 0, store: hostedByEnvironment() ? "none" : "local" };
+  if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 21)) {
+    return { deleted: 0, store: "private-vercel-blob", skipped: true };
+  }
+  const cutoff = now.getTime() - SITE_EXTRACT_ARTIFACT_TTL_MS;
+  const { list, del } = await import("@vercel/blob");
+  let cursor;
+  let deleted = 0;
+  for (let page = 0; page < 20; page += 1) {
+    const result = await list({ prefix: SITE_EXTRACT_ARTIFACT_PREFIX, token, limit: 1e3, cursor });
+    const expired = result.blobs.filter((blob) => new Date(blob.uploadedAt).getTime() <= cutoff);
+    if (expired.length > 0) {
+      await del(expired.map((blob) => blob.pathname), { token });
+      deleted += expired.length;
+    }
+    if (!result.hasMore || !result.cursor) break;
+    cursor = result.cursor;
+  }
+  return { deleted, store: "private-vercel-blob" };
+}
+
+export {
+  typeFromMime,
+  discoverStaticMedia,
+  downloadAsset,
+  harvestPageMedia,
+  SITE_EXTRACT_ARTIFACT_PREFIX,
+  createSiteExtractBundleArtifactStream,
+  renewSiteExtractArtifactDownload,
+  readOwnedSiteExtractArtifactBuffer,
+  cleanupExpiredSiteExtractArtifacts
+};
+//# sourceMappingURL=chunk-SG2PEHR3.js.map

package/dist/chunk-SG2PEHR3.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/lib/media-extractor.ts","../src/api/site-extract-artifacts.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, rmSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join, extname, basename } from 'node:path'\nimport { pipeline } from 'node:stream/promises'\nimport { Readable, Transform } from 'node:stream'\nimport { validatePublicHttpUrl } from '../api/url-utils.js'\nimport type { RenderedMediaDiscovery } from './rendered-media.js'\n\nconst AD_PATTERNS: string[] = [\n  'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',\n  'google-analytics.com', 'googletagservices.com', 'adservice.google',\n  'googletag.', 'pagead2.googlesyndication',\n  'facebook.net/tr', 'connect.facebook.net', 'fbcdn.net/rsrc',\n  'analytics.twitter.com', 'static.ads-twitter.com', 'ads.twitter.com',\n  't.co/i/adsct', 'pixel.advertising.com',\n  'hotjar.com', 'clarity.ms', 'quantserve.com', 'scorecardresearch.com',\n  'newrelic.com', 'nr-data.net', 'segment.io', 'segment.com',\n  'amplitude.com', 'mixpanel.com', 'heap.io', 'fullstory.com',\n  'moatads.com', 'criteo.com', 'adsrvr.org', 'rubiconproject.com',\n  'pubmatic.com', 'openx.net', 'appnexus.com', 'amazon-adsystem.com',\n  'media.net', 'yieldmo.com', 'triplelift.com', 'sharethrough.com',\n  'prebid.', 'smaato.net', 'indexworm.com', 'casalemedia.com',\n  'outbrain.com', 'taboola.com', 'revcontent.com', 'mgid.com',\n  'tawk.to', 'intercom.io', 'drift.com', 'hs-scripts.com',\n  'zopim.com', 'livechatinc.com', 'userlike.com',\n  'onetrust.com', 'cookielaw.org', 'cookieinformation.com', 'trustarc.com',\n  '/ads/', '/ad/', '/banner/', '/banners/', '/pixel/', '/beacon/',\n  '/tracking/', '/tracker/', '/remarketing/', '/conversion/',\n  '1x1.gif', 'spacer.gif', 'blank.gif', 'transparent.gif',\n]\n\nconst IMAGE_EXTS  = new Set(['.jpg', '.jpeg', '.png', '.webp', '.gif', '.avif', '.svg', '.tiff'])\nconst VIDEO_EXTS  = new Set(['.mp4', '.webm', '.mov', '.avi', '.m4v', '.ogv', '.mkv'])\nconst AUDIO_EXTS  = new Set(['.mp3', '.wav', '.ogg', '.aac', '.m4a', '.flac', '.opus'])\n\nexport type MediaType = 'image' | 'video' | 'audio'\n\nexport interface MediaAsset {\n  url:        string\n  type:       MediaType\n  mimeType:   string | null\n  filename:   string\n  savedPath:  string | null\n  sizeBytes:  number | null\n  discoveryMethods: string[]\n  altTexts: string[]\n  contexts: string[]\n  width: number | null\n  height: number | null\n  variants: string[]\n  finalUrl?: string | null\n  duplicateOf?: string | null\n  sha256?: string | null\n  downloadStatus?: 'downloaded' | 'failed' | 'not_attempted'\n  downloadError?: string | null\n  inlinePreview?: { data: string; mimeType: string } | null\n}\n\nexport interface MediaManifest {\n  pageUrl:       string\n  outputDir:     string | null\n  assets:        MediaAsset[]\n  filteredCount: number\n  totalFound:    number\n  staticFound: number\n  renderedFound: number\n  completeness: 'complete' | 'partial'\n  exhausted: boolean\n  stopReason: 'page_exhausted' | 'asset_limit' | 'scroll_round_limit' | 'render_unavailable'\n  scrollRounds: number\n  warnings: string[]\n  artifact: {\n    artifactId: string\n    filename: string\n    contentType: string\n    bytes: number\n    sha256: string\n    expiresAt: string\n    downloadUrl: string | null\n    downloadUrlExpiresAt: string | null\n    localPath: string | null\n  } | null\n}\n\nexport interface MediaExtractOptions {\n  types?:      MediaType[]\n  outputDir?:  string | null\n  rendered?: RenderedMediaDiscovery | null\n  maxAssets?: number\n}\n\nexport interface DiscoveredMediaCandidate {\n  url: string\n  type: MediaType | null\n  discoveryMethods: string[]\n  altTexts: string[]\n  contexts: string[]\n  width: number | null\n  height: number | null\n}\n\nfunction isAdUrl(url: string): boolean {\n  const lower = url.toLowerCase()\n  return AD_PATTERNS.some(p => lower.includes(p))\n}\n\nfunction isDataUri(url: string): boolean {\n  return url.startsWith('data:')\n}\n\nexport function typeFromUrl(url: string): MediaType | null {\n  try {\n    const ext = extname(new URL(url).pathname).toLowerCase()\n    if (IMAGE_EXTS.has(ext)) return 'image'\n    if (VIDEO_EXTS.has(ext)) return 'video'\n    if (AUDIO_EXTS.has(ext)) return 'audio'\n  } catch { /* non-parseable URL */ }\n  return null\n}\n\nexport function typeFromMime(mime: string): MediaType | null {\n  const lower = mime.toLowerCase()\n  if (lower.startsWith('image/')) return 'image'\n  if (lower.startsWith('video/')) return 'video'\n  if (lower.startsWith('audio/')) return 'audio'\n  return null\n}\n\nfunction resolveUrl(raw: string, base: string): string | null {\n  if (!raw || isDataUri(raw)) return null\n  try { return new URL(raw, base).href } catch { return null }\n}\n\nfunction safeFilename(url: string, index: number): string {\n  try {\n    const u = new URL(url)\n    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n    return base || `asset-${index}`\n  } catch {\n    return `asset-${index}`\n  }\n}\n\nfunction boundedText(value: string | undefined | null, max: number): string | null {\n  const normalized = value?.replace(/&(?:amp|#0?38|#x26);/gi, '&').replace(/\\s+/g, ' ').trim()\n  return normalized ? normalized.slice(0, max) : null\n}\n\nfunction dimensionsFromUrl(rawUrl: string): { width: number; height: number } | null {\n  const match = rawUrl.match(/[-_/](\\d{2,5})x(\\d{2,5})(?=[._/?#-]|$)/i)\n  return match ? { width: Number(match[1]), height: Number(match[2]) } : null\n}\n\nexport function mediaFamilyKey(rawUrl: string): string {\n  try {\n    const url = new URL(rawUrl)\n    url.hash = ''\n    url.pathname = url.pathname.replace(/-(?:\\d{2,5}x\\d{2,5}|scaled|e\\d{6,})(?=\\.[a-z0-9]{2,6}$)/i, '')\n    for (const key of ['w', 'width', 'h', 'height', 'resize']) url.searchParams.delete(key)\n    url.searchParams.sort()\n    return url.href\n  } catch {\n    return rawUrl\n  }\n}\n\nfunction candidateScore(candidate: DiscoveredMediaCandidate): number {\n  const area = (candidate.width ?? 0) * (candidate.height ?? 0)\n  const methodBoost = candidate.discoveryMethods.some(method => method.includes('lightbox')) ? 1_000_000_000_000 : 0\n  const originalBoost = /-(?:\\d{2,5}x\\d{2,5}|scaled|e\\d{6,})(?=\\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(candidate.url) ? 0 : 100_000_000_000\n  return methodBoost + originalBoost + area\n}\n\nfunction mergeCandidate(target: Map<string, DiscoveredMediaCandidate>, incoming: DiscoveredMediaCandidate): void {\n  const existing = target.get(incoming.url)\n  if (!existing) {\n    target.set(incoming.url, incoming)\n    return\n  }\n  existing.discoveryMethods = [...new Set([...existing.discoveryMethods, ...incoming.discoveryMethods])]\n  existing.altTexts = [...new Set([...existing.altTexts, ...incoming.altTexts])]\n  existing.contexts = [...new Set([...existing.contexts, ...incoming.contexts])]\n  if (!existing.type && incoming.type) existing.type = incoming.type\n  if ((incoming.width ?? 0) * (incoming.height ?? 0) > (existing.width ?? 0) * (existing.height ?? 0)) {\n    existing.width = incoming.width\n    existing.height = incoming.height\n  }\n}\n\nexport function discoverStaticMedia(html: string, baseUrl: string): DiscoveredMediaCandidate[] {\n  const found = new Map<string, DiscoveredMediaCandidate>()\n  const add = (\n    raw: string | undefined,\n    method: string,\n    explicitType: MediaType | null = null,\n    alt?: string | null,\n    context?: string | null,\n    dimensions?: { width: number; height: number } | null,\n  ) => {\n    const resolved = raw ? resolveUrl(raw.trim().replace(/&amp;/g, '&'), baseUrl) : null\n    if (!resolved || isAdUrl(resolved)) return\n    const inferredDimensions = dimensions ?? dimensionsFromUrl(resolved)\n    mergeCandidate(found, {\n      url: resolved,\n      type: explicitType ?? typeFromUrl(resolved),\n      discoveryMethods: [method],\n      altTexts: alt ? [alt] : [],\n      contexts: context ? [context] : [],\n      width: inferredDimensions?.width ?? null,\n      height: inferredDimensions?.height ?? null,\n    })\n  }\n  const scanSrcset = (value: string | undefined, method: string, type: MediaType, alt?: string | null) => {\n    for (const part of (value ?? '').split(',')) {\n      const [url, descriptor] = part.trim().split(/\\s+/)\n      if (!url) continue\n      const width = descriptor?.match(/^(\\d+)w$/)?.[1]\n      add(url, method, type, alt, null, width ? { width: Number(width), height: 0 } : null)\n    }\n  }\n  const scan = (source: string, suffix = '') => {\n    for (const match of source.matchAll(/<img\\b([^>]*)>/gi)) {\n      const attrs = match[1]\n      const alt = boundedText(attrs.match(/\\balt\\s*=\\s*[\"']([^\"']*)[\"']/i)?.[1], 500)\n      const width = Number(attrs.match(/\\bwidth\\s*=\\s*[\"']?(\\d+)/i)?.[1] ?? 0)\n      const height = Number(attrs.match(/\\bheight\\s*=\\s*[\"']?(\\d+)/i)?.[1] ?? 0)\n      const dimensions = width > 0 && height > 0 ? { width, height } : null\n      for (const attr of ['src', 'data-src', 'data-lazy-src', 'data-original', 'data-bg', 'data-background', 'data-bg-url', 'data-lazy-bg', 'data-echo']) {\n        const value = attrs.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i'))?.[1]\n        add(value, `static-${attr}${suffix}`, 'image', alt, null, dimensions)\n      }\n      scanSrcset(attrs.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1], `static-srcset${suffix}`, 'image', alt)\n      scanSrcset(attrs.match(/\\bdata-srcset\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1], `static-data-srcset${suffix}`, 'image', alt)\n    }\n    for (const match of source.matchAll(/<source\\b([^>]*)>/gi)) {\n      const attrs = match[1]\n      const declared = attrs.match(/\\btype\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1] ?? ''\n      const type: MediaType = declared.startsWith('video/') ? 'video' : declared.startsWith('audio/') ? 'audio' : 'image'\n      add(attrs.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1], `static-source${suffix}`, type)\n      scanSrcset(attrs.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1], `static-source-srcset${suffix}`, type)\n    }\n    for (const match of source.matchAll(/<(video|audio)\\b([^>]*)>/gi)) {\n      const type = match[1].toLowerCase() as 'video' | 'audio'\n      const attrs = match[2]\n      add(attrs.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1], `static-${type}${suffix}`, type)\n      if (type === 'video') add(attrs.match(/\\bposter\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1], `static-video-poster${suffix}`, 'image')\n    }\n    for (const match of source.matchAll(/<meta\\b([^>]*)>/gi)) {\n      const attrs = match[1]\n      const prop = attrs.match(/\\b(?:property|name)\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1]?.toLowerCase()\n      if (prop === 'og:image' || prop === 'og:image:url' || prop === 'twitter:image' || prop === 'twitter:image:src') {\n        add(attrs.match(/\\bcontent\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1], `static-${prop}${suffix}`, 'image')\n      }\n    }\n    for (const match of source.matchAll(/<link\\b([^>]*)>/gi)) {\n      const attrs = match[1]\n      if (/\\brel\\s*=\\s*[\"'][^\"']*(?:icon|apple-touch-icon)[^\"']*[\"']/i.test(attrs)) {\n        add(attrs.match(/\\bhref\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1], `static-icon${suffix}`, 'image')\n      }\n    }\n    for (const match of source.matchAll(/<image\\b([^>]*)>/gi)) {\n      const attrs = match[1]\n      add(attrs.match(/\\b(?:href|xlink:href)\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1], `static-svg-image${suffix}`, 'image')\n    }\n    for (const match of source.matchAll(/<a\\b([^>]*)>[\\s\\S]*?<img\\b([^>]*)>[\\s\\S]*?<\\/a>/gi)) {\n      const anchorAttrs = match[1]\n      const href = anchorAttrs.match(/\\bhref\\s*=\\s*[\"']([^\"']+)[\"']/i)?.[1]\n      if (!href) continue\n      if (\n        /lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(anchorAttrs)\n        || /\\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(href)\n      ) {\n        const alt = boundedText(match[2].match(/\\balt\\s*=\\s*[\"']([^\"']*)[\"']/i)?.[1], 500)\n        add(href, `static-lightbox-target${suffix}`, 'image', alt)\n      }\n    }\n    for (const match of source.matchAll(/(?:background(?:-image)?\\s*:\\s*)?url\\(\\s*(?:[\"']|&quot;|&#0?34;|&apos;|&#0?39;)?([^\"')&]+)(?:[\"']|&quot;|&#0?34;|&apos;|&#0?39;)?\\s*\\)/gi)) {\n      add(match[1], `static-css-url${suffix}`, 'image')\n    }\n    for (const match of source.matchAll(/[\"'](https?:\\/\\/[^\"'\\s<>]+\\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\\?[^\"'\\s<>]*)?)[\"']/gi)) {\n      add(match[1], `static-bare-url${suffix}`)\n    }\n  }\n\n  scan(html)\n  const unescaped = html\n    .replace(/\\\\u003[cC]/g, '<').replace(/\\\\u003[eE]/g, '>').replace(/\\\\u0026/gi, '&')\n    .replace(/\\\\[\"']/g, match => match.slice(1)).replace(/\\\\\\//g, '/')\n  if (unescaped !== html) scan(unescaped, '-json-unescaped')\n  return [...found.values()]\n}\n\nexport function mergeMediaDiscovery(\n  staticAssets: DiscoveredMediaCandidate[],\n  rendered: RenderedMediaDiscovery | null | undefined,\n  maxAssets = 100,\n): Array<DiscoveredMediaCandidate & { variants: string[] }> {\n  const exact = new Map<string, DiscoveredMediaCandidate>()\n  for (const asset of staticAssets) mergeCandidate(exact, asset)\n  for (const asset of rendered?.assets ?? []) mergeCandidate(exact, { ...asset, type: asset.type })\n\n  const families = new Map<string, DiscoveredMediaCandidate[]>()\n  for (const asset of exact.values()) {\n    const key = mediaFamilyKey(asset.url)\n    const family = families.get(key) ?? []\n    family.push(asset)\n    families.set(key, family)\n  }\n  return [...families.values()]\n    .map(family => {\n      const ordered = [...family].sort((a, b) => candidateScore(b) - candidateScore(a) || a.url.localeCompare(b.url))\n      const preferred = { ...ordered[0] }\n      preferred.discoveryMethods = [...new Set(family.flatMap(asset => asset.discoveryMethods))]\n      preferred.altTexts = [...new Set(family.flatMap(asset => asset.altTexts))]\n      preferred.contexts = [...new Set(family.flatMap(asset => asset.contexts))]\n      return { ...preferred, variants: ordered.slice(1).map(asset => asset.url) }\n    })\n    .sort((a, b) => candidateScore(b) - candidateScore(a) || a.url.localeCompare(b.url))\n    .slice(0, Math.max(1, maxAssets))\n}\n\nexport function extractMediaUrls(html: string, baseUrl: string): string[] {\n  return mergeMediaDiscovery(discoverStaticMedia(html, baseUrl), null, 2_000).map(asset => asset.url)\n}\n\nexport async function downloadAsset(\n  url: string,\n  destDir: string,\n  filename: string,\n  options: {\n    expectedType?: MediaType\n    maxBytes?: number\n    consumeBytes?: (bytes: number) => boolean\n  } = {},\n): Promise<{ savedPath: string; sizeBytes: number; mimeType: string | null }> {\n  const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024))\n  let target = url\n  let res: Response | null = null\n  for (let redirects = 0; redirects <= 5; redirects++) {\n    const checked = await validatePublicHttpUrl(target, { field: 'media URL' })\n    if (checked.error || !checked.parsed) throw new Error(checked.error ?? 'Media URL was rejected')\n    res = await fetch(checked.parsed.href, {\n      signal: AbortSignal.timeout(15_000),\n      redirect: 'manual',\n    })\n    if (res.status >= 300 && res.status < 400) {\n      const location = res.headers.get('location')\n      if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`)\n      target = new URL(location, checked.parsed.href).href\n      res = null\n      continue\n    }\n    break\n  }\n  if (!res) throw new Error('Media download exceeded five redirects')\n  if (!res.ok) throw new Error(`HTTP ${res.status}`)\n  if (!res.body) throw new Error('Empty response body')\n\n  const mimeType = res.headers.get('content-type')?.split(';')[0].trim() ?? null\n  if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {\n    throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? 'no content-type'}`)\n  }\n  const declaredBytes = Number(res.headers.get('content-length') ?? 0)\n  if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {\n    throw new Error(`Media asset exceeds ${maxBytes} byte limit`)\n  }\n\n  let dest = join(destDir, filename)\n  if (mimeType && !extname(filename)) {\n    const mimeExt: Record<string, string> = {\n      'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp',\n      'image/gif': '.gif', 'image/svg+xml': '.svg', 'image/avif': '.avif',\n      'video/mp4': '.mp4', 'video/webm': '.webm',\n      'audio/mpeg': '.mp3', 'audio/ogg': '.ogg', 'audio/wav': '.wav',\n    }\n    const ext = mimeExt[mimeType]\n    if (ext) dest = dest + ext\n  }\n\n  const writer = createWriteStream(dest)\n  await new Promise<void>((resolve, reject) => {\n    writer.once('open', () => resolve())\n    writer.once('error', reject)\n  })\n  let streamedBytes = 0\n  const limiter = new Transform({\n    transform(chunk: Buffer, _encoding, callback) {\n      const bytes = chunk.length\n      if (streamedBytes + bytes > maxBytes) {\n        callback(new Error(`Media asset exceeds ${maxBytes} byte limit`))\n        return\n      }\n      if (options.consumeBytes && !options.consumeBytes(bytes)) {\n        callback(new Error('Media export total byte limit exceeded'))\n        return\n      }\n      streamedBytes += bytes\n      callback(null, chunk)\n    },\n  })\n  try {\n    await pipeline(Readable.fromWeb(res.body as import('stream/web').ReadableStream), limiter, writer)\n  } catch (error) {\n    writer.destroy()\n    rmSync(dest, { force: true })\n    throw error\n  }\n\n  const { statSync } = await import('node:fs')\n  const sizeBytes = statSync(dest).size\n\n  return { savedPath: dest, sizeBytes, mimeType }\n}\n\nexport async function harvestPageMedia(\n  html: string,\n  pageUrl: string,\n  options: MediaExtractOptions = {},\n): Promise<MediaManifest> {\n  const types    = options.types ?? ['image', 'video', 'audio']\n  const typesSet = new Set(types)\n  const staticAssets = discoverStaticMedia(html, pageUrl)\n  const maxAssets = Math.max(1, Math.min(options.maxAssets ?? 100, 250))\n  const allDiscovered = mergeMediaDiscovery(staticAssets, options.rendered, 2_000)\n  const discovered = allDiscovered.slice(0, maxAssets)\n  const totalFound = new Set([\n    ...staticAssets.map(asset => asset.url),\n    ...(options.rendered?.assets ?? []).map(asset => asset.url),\n  ]).size\n  const kept = discovered.flatMap(asset => {\n    const type = asset.type ?? typeFromUrl(asset.url)\n    return type && typesSet.has(type) ? [{ ...asset, type }] : []\n  })\n  const filteredCount = totalFound - kept.length\n  const retentionLimitReached = allDiscovered.length > maxAssets\n  const exhausted = options.rendered?.exhausted === true && !retentionLimitReached\n  const stopReason = retentionLimitReached ? 'asset_limit' : options.rendered?.stopReason ?? 'render_unavailable'\n  const warnings = retentionLimitReached\n    ? [`Media discovery found ${allDiscovered.length} responsive families; only the requested ${maxAssets} records were retained.`]\n    : options.rendered\n    ? options.rendered.exhausted ? [] : [`Rendered media discovery stopped with ${options.rendered.stopReason}; the inventory may be incomplete.`]\n    : ['Rendered media discovery was unavailable; this manifest contains static-source evidence only.']\n\n  const baseManifest: Omit<MediaManifest, 'outputDir' | 'assets'> = {\n    pageUrl,\n    staticFound: staticAssets.length,\n    renderedFound: options.rendered?.assets.length ?? 0,\n    filteredCount,\n    totalFound,\n    completeness: exhausted ? 'complete' as const : 'partial' as const,\n    exhausted,\n    stopReason,\n    scrollRounds: options.rendered?.scrollRounds ?? 0,\n    warnings,\n    artifact: null,\n  }\n\n  if (options.outputDir === null) {\n    return {\n      ...baseManifest,\n      outputDir: null,\n      assets: kept.map((asset, i) => ({\n        ...asset,\n        mimeType: null, filename: safeFilename(asset.url, i), savedPath: null, sizeBytes: null,\n        sha256: null, downloadStatus: 'not_attempted' as const, downloadError: null, inlinePreview: null,\n      })),\n    }\n  }\n\n  const domain  = (() => { try { return new URL(pageUrl).hostname.replace(/^www\\./, '') } catch { return 'unknown' } })()\n  const stamp   = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)\n  const outDir  = options.outputDir ?? join(homedir(), 'Downloads', 'mcp-scraper', 'media', `${stamp}-${domain}`)\n  mkdirSync(outDir, { recursive: true })\n\n  const assets: MediaAsset[] = []\n\n  await Promise.allSettled(\n    kept.map(async (asset, i) => {\n      const filename = safeFilename(asset.url, i)\n      try {\n        const { savedPath, sizeBytes, mimeType } = await downloadAsset(asset.url, outDir, filename)\n        const resolvedType = mimeType ? (typeFromMime(mimeType) ?? asset.type) : asset.type\n        assets.push({\n          ...asset, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes,\n          sha256: null, downloadStatus: 'downloaded', downloadError: null, inlinePreview: null,\n        })\n      } catch (error) {\n        assets.push({\n          ...asset, mimeType: null, filename, savedPath: null, sizeBytes: null, sha256: null,\n          downloadStatus: 'failed', downloadError: error instanceof Error ? error.message : 'media_download_failed', inlinePreview: null,\n        })\n      }\n    })\n  )\n\n  assets.sort((a, b) => {\n    if (a.savedPath && !b.savedPath) return -1\n    if (!a.savedPath && b.savedPath) return 1\n    return a.url.localeCompare(b.url)\n  })\n\n  return { ...baseManifest, outputDir: outDir, assets }\n}\n","import {\n  createPrivateArtifact,\n  createPrivateArtifactFromStream,\n  privateArtifactOwnerId,\n  renewPrivateArtifactDownload,\n  readPrivateArtifactBuffer,\n  type PrivateArtifactPolicy,\n} from './private-artifacts.js'\nimport type { ExtractJobArtifact } from './site-extract-repository.js'\nimport type { Readable } from 'node:stream'\n\nexport const SITE_EXTRACT_ARTIFACT_PREFIX = 'site-extracts/'\nexport const SITE_EXTRACT_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1_000\nexport const SITE_EXTRACT_DOWNLOAD_TTL_MS = 15 * 60 * 1_000\n\nfunction siteExtractArtifactToken(): string | null {\n  return process.env.SITE_EXTRACT_ARTIFACT_READ_WRITE_TOKEN?.trim()\n    || process.env.PRIVATE_ARTIFACT_READ_WRITE_TOKEN?.trim()\n    || process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim()\n    || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim()\n    || null\n}\n\nfunction hostedByEnvironment(): boolean {\n  return process.env.VERCEL === '1' || process.env.NODE_ENV === 'production'\n}\n\nfunction policy(): PrivateArtifactPolicy {\n  return {\n    prefix: SITE_EXTRACT_ARTIFACT_PREFIX,\n    artifactTtlMs: SITE_EXTRACT_ARTIFACT_TTL_MS,\n    downloadTtlMs: SITE_EXTRACT_DOWNLOAD_TTL_MS,\n    token: siteExtractArtifactToken(),\n  }\n}\n\nexport function siteExtractArtifactOwnerId(artifactId: string): string | null {\n  return privateArtifactOwnerId(artifactId, SITE_EXTRACT_ARTIFACT_PREFIX)\n}\n\nexport async function createSiteExtractBundleArtifact(args: {\n  ownerId: string\n  jobId: string\n  createdAt: Date | string | number\n  content: Buffer\n}): Promise<ExtractJobArtifact> {\n  const pointer = await createPrivateArtifact({\n    policy: policy(),\n    ownerId: args.ownerId,\n    artifactKey: `${args.jobId}.zip`,\n    createdAt: args.createdAt,\n    filename: `${args.jobId}-site-export.zip`,\n    contentType: 'application/zip',\n    content: args.content,\n  })\n  return {\n    key: pointer.artifactId,\n    url: pointer.downloadUrl ?? '',\n    bytes: pointer.bytes,\n    contentType: pointer.contentType,\n    filename: pointer.filename,\n    sha256: pointer.sha256,\n    expiresAt: pointer.expiresAt,\n    downloadUrlExpiresAt: pointer.downloadUrlExpiresAt,\n  }\n}\n\nexport async function createSiteExtractBundleArtifactStream(args: {\n  ownerId: string\n  jobId: string\n  createdAt: Date | string | number\n  content: Readable\n}): Promise<ExtractJobArtifact> {\n  const pointer = await createPrivateArtifactFromStream({\n    policy: policy(),\n    ownerId: args.ownerId,\n    artifactKey: `${args.jobId}.zip`,\n    createdAt: args.createdAt,\n    filename: `${args.jobId}-site-export.zip`,\n    contentType: 'application/zip',\n    content: args.content,\n  })\n  return {\n    key: pointer.artifactId,\n    url: pointer.downloadUrl ?? '',\n    bytes: pointer.bytes,\n    contentType: pointer.contentType,\n    filename: pointer.filename,\n    sha256: pointer.sha256,\n    expiresAt: pointer.expiresAt,\n    downloadUrlExpiresAt: pointer.downloadUrlExpiresAt,\n  }\n}\n\nexport async function renewSiteExtractArtifactDownload(args: {\n  artifactId: string\n  ownerId: string\n}): Promise<{ downloadUrl: string; downloadUrlExpiresAt: string; expiresAt: string } | null> {\n  return renewPrivateArtifactDownload({\n    policy: policy(),\n    artifactId: args.artifactId,\n    ownerId: args.ownerId,\n  })\n}\n\nexport async function readSiteExtractArtifactBuffer(artifactId: string): Promise<Buffer | null> {\n  return readPrivateArtifactBuffer({\n    policy: policy(),\n    artifactId,\n    maxBytes: 50 * 1024 * 1024,\n  })\n}\n\n/**\n * Owner-authorized read entrypoint for site-export artifacts. Keep the ownership\n * check adjacent to the storage read so callers cannot accidentally validate an\n * ID and then dereference a different one.\n */\nexport async function readOwnedSiteExtractArtifactBuffer(args: {\n  artifactId: string\n  ownerId: string\n}): Promise<Buffer | null> {\n  if (siteExtractArtifactOwnerId(args.artifactId) !== args.ownerId) return null\n  return readSiteExtractArtifactBuffer(args.artifactId)\n}\n\nexport async function cleanupExpiredSiteExtractArtifacts(args: {\n  now?: Date\n  force?: boolean\n} = {}): Promise<{ deleted: number; store: 'private-vercel-blob' | 'local' | 'none'; skipped?: boolean }> {\n  const now = args.now ?? new Date()\n  const token = siteExtractArtifactToken()\n  if (!token) return { deleted: 0, store: hostedByEnvironment() ? 'none' : 'local' }\n  if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 21)) {\n    return { deleted: 0, store: 'private-vercel-blob', skipped: true }\n  }\n  const cutoff = now.getTime() - SITE_EXTRACT_ARTIFACT_TTL_MS\n  const { list, del } = await import('@vercel/blob')\n  let cursor: string | undefined\n  let deleted = 0\n  for (let page = 0; page < 20; page += 1) {\n    const result = await list({ prefix: SITE_EXTRACT_ARTIFACT_PREFIX, token, limit: 1_000, cursor })\n    const expired = result.blobs.filter(blob => new Date(blob.uploadedAt).getTime() <= cutoff)\n    if (expired.length > 0) {\n      await del(expired.map(blob => blob.pathname), { token })\n      deleted += expired.length\n    }\n    if (!result.hasMore || !result.cursor) break\n    cursor = result.cursor\n  }\n  return { deleted, store: 'private-vercel-blob' }\n}\n"],"mappings":";;;;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc;AACrD,SAAS,eAAe;AACxB,SAAS,MAAM,SAAS,gBAAgB;AACxC,SAAS,gBAAgB;AACzB,SAAS,UAAU,iBAAiB;AAIpC,IAAM,cAAwB;AAAA,EAC5B;AAAA,EAAmB;AAAA,EAAyB;AAAA,EAC5C;AAAA,EAAwB;AAAA,EAAyB;AAAA,EACjD;AAAA,EAAc;AAAA,EACd;AAAA,EAAmB;AAAA,EAAwB;AAAA,EAC3C;AAAA,EAAyB;AAAA,EAA0B;AAAA,EACnD;AAAA,EAAgB;AAAA,EAChB;AAAA,EAAc;AAAA,EAAc;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAc;AAAA,EAC7C;AAAA,EAAiB;AAAA,EAAgB;AAAA,EAAW;AAAA,EAC5C;AAAA,EAAe;AAAA,EAAc;AAAA,EAAc;AAAA,EAC3C;AAAA,EAAgB;AAAA,EAAa;AAAA,EAAgB;AAAA,EAC7C;AAAA,EAAa;AAAA,EAAe;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAiB;AAAA,EAC1C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAkB;AAAA,EACjD;AAAA,EAAW;AAAA,EAAe;AAAA,EAAa;AAAA,EACvC;AAAA,EAAa;AAAA,EAAmB;AAAA,EAChC;AAAA,EAAgB;AAAA,EAAiB;AAAA,EAAyB;AAAA,EAC1D;AAAA,EAAS;AAAA,EAAQ;AAAA,EAAY;AAAA,EAAa;AAAA,EAAW;AAAA,EACrD;AAAA,EAAc;AAAA,EAAa;AAAA,EAAiB;AAAA,EAC5C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAa;AACxC;AAEA,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ,OAAO,CAAC;AAChG,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,QAAQ,QAAQ,QAAQ,MAAM,CAAC;AACrF,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,SAAS,OAAO,CAAC;AAoEtF,SAAS,QAAQ,KAAsB;AACrC,QAAM,QAAQ,IAAI,YAAY;AAC9B,SAAO,YAAY,KAAK,OAAK,MAAM,SAAS,CAAC,CAAC;AAChD;AAEA,SAAS,UAAU,KAAsB;AACvC,SAAO,IAAI,WAAW,OAAO;AAC/B;AAEO,SAAS,YAAY,KAA+B;AACzD,MAAI;AACF,UAAM,MAAM,QAAQ,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,YAAY;AACvD,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAAA,EAClC,QAAQ;AAAA,EAA0B;AAClC,SAAO;AACT;AAEO,SAAS,aAAa,MAAgC;AAC3D,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,SAAO;AACT;AAEA,SAAS,WAAW,KAAa,MAA6B;AAC5D,MAAI,CAAC,OAAO,UAAU,GAAG,EAAG,QAAO;AACnC,MAAI;AAAE,WAAO,IAAI,IAAI,KAAK,IAAI,EAAE;AAAA,EAAK,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC7D;AAEA,SAAS,aAAa,KAAa,OAAuB;AACxD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEA,SAAS,YAAY,OAAkC,KAA4B;AACjF,QAAM,aAAa,OAAO,QAAQ,0BAA0B,GAAG,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAC3F,SAAO,aAAa,WAAW,MAAM,GAAG,GAAG,IAAI;AACjD;AAEA,SAAS,kBAAkB,QAA0D;AACnF,QAAM,QAAQ,OAAO,MAAM,yCAAyC;AACpE,SAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,CAAC,CAAC,GAAG,QAAQ,OAAO,MAAM,CAAC,CAAC,EAAE,IAAI;AACzE;AAEO,SAAS,eAAe,QAAwB;AACrD,MAAI;AACF,UAAM,MAAM,IAAI,IAAI,MAAM;AAC1B,QAAI,OAAO;AACX,QAAI,WAAW,IAAI,SAAS,QAAQ,4DAA4D,EAAE;AAClG,eAAW,OAAO,CAAC,KAAK,SAAS,KAAK,UAAU,QAAQ,EAAG,KAAI,aAAa,OAAO,GAAG;AACtF,QAAI,aAAa,KAAK;AACtB,WAAO,IAAI;AAAA,EACb,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,eAAe,WAA6C;AACnE,QAAM,QAAQ,UAAU,SAAS,MAAM,UAAU,UAAU;AAC3D,QAAM,cAAc,UAAU,iBAAiB,KAAK,YAAU,OAAO,SAAS,UAAU,CAAC,IAAI,OAAoB;AACjH,QAAM,gBAAgB,oEAAoE,KAAK,UAAU,GAAG,IAAI,IAAI;AACpH,SAAO,cAAc,gBAAgB;AACvC;AAEA,SAAS,eAAe,QAA+C,UAA0C;AAC/G,QAAM,WAAW,OAAO,IAAI,SAAS,GAAG;AACxC,MAAI,CAAC,UAAU;AACb,WAAO,IAAI,SAAS,KAAK,QAAQ;AACjC;AAAA,EACF;AACA,WAAS,mBAAmB,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,kBAAkB,GAAG,SAAS,gBAAgB,CAAC,CAAC;AACrG,WAAS,WAAW,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,UAAU,GAAG,SAAS,QAAQ,CAAC,CAAC;AAC7E,WAAS,WAAW,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,UAAU,GAAG,SAAS,QAAQ,CAAC,CAAC;AAC7E,MAAI,CAAC,SAAS,QAAQ,SAAS,KAAM,UAAS,OAAO,SAAS;AAC9D,OAAK,SAAS,SAAS,MAAM,SAAS,UAAU,MAAM,SAAS,SAAS,MAAM,SAAS,UAAU,IAAI;AACnG,aAAS,QAAQ,SAAS;AAC1B,aAAS,SAAS,SAAS;AAAA,EAC7B;AACF;AAEO,SAAS,oBAAoB,MAAc,SAA6C;AAC7F,QAAM,QAAQ,oBAAI,IAAsC;AACxD,QAAM,MAAM,CACV,KACA,QACA,eAAiC,MACjC,KACA,SACA,eACG;AACH,UAAM,WAAW,MAAM,WAAW,IAAI,KAAK,EAAE,QAAQ,UAAU,GAAG,GAAG,OAAO,IAAI;AAChF,QAAI,CAAC,YAAY,QAAQ,QAAQ,EAAG;AACpC,UAAM,qBAAqB,cAAc,kBAAkB,QAAQ;AACnE,mBAAe,OAAO;AAAA,MACpB,KAAK;AAAA,MACL,MAAM,gBAAgB,YAAY,QAAQ;AAAA,MAC1C,kBAAkB,CAAC,MAAM;AAAA,MACzB,UAAU,MAAM,CAAC,GAAG,IAAI,CAAC;AAAA,MACzB,UAAU,UAAU,CAAC,OAAO,IAAI,CAAC;AAAA,MACjC,OAAO,oBAAoB,SAAS;AAAA,MACpC,QAAQ,oBAAoB,UAAU;AAAA,IACxC,CAAC;AAAA,EACH;AACA,QAAM,aAAa,CAAC,OAA2B,QAAgB,MAAiB,QAAwB;AACtG,eAAW,SAAS,SAAS,IAAI,MAAM,GAAG,GAAG;AAC3C,YAAM,CAAC,KAAK,UAAU,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK;AACjD,UAAI,CAAC,IAAK;AACV,YAAM,QAAQ,YAAY,MAAM,UAAU,IAAI,CAAC;AAC/C,UAAI,KAAK,QAAQ,MAAM,KAAK,MAAM,QAAQ,EAAE,OAAO,OAAO,KAAK,GAAG,QAAQ,EAAE,IAAI,IAAI;AAAA,IACtF;AAAA,EACF;AACA,QAAM,OAAO,CAAC,QAAgB,SAAS,OAAO;AAC5C,eAAW,SAAS,OAAO,SAAS,kBAAkB,GAAG;AACvD,YAAM,QAAQ,MAAM,CAAC;AACrB,YAAM,MAAM,YAAY,MAAM,MAAM,+BAA+B,IAAI,CAAC,GAAG,GAAG;AAC9E,YAAM,QAAQ,OAAO,MAAM,MAAM,2BAA2B,IAAI,CAAC,KAAK,CAAC;AACvE,YAAM,SAAS,OAAO,MAAM,MAAM,4BAA4B,IAAI,CAAC,KAAK,CAAC;AACzE,YAAM,aAAa,QAAQ,KAAK,SAAS,IAAI,EAAE,OAAO,OAAO,IAAI;AACjE,iBAAW,QAAQ,CAAC,OAAO,YAAY,iBAAiB,iBAAiB,WAAW,mBAAmB,eAAe,gBAAgB,WAAW,GAAG;AAClJ,cAAM,QAAQ,MAAM,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,IAAI,CAAC;AACrF,YAAI,OAAO,UAAU,IAAI,GAAG,MAAM,IAAI,SAAS,KAAK,MAAM,UAAU;AAAA,MACtE;AACA,iBAAW,MAAM,MAAM,kCAAkC,IAAI,CAAC,GAAG,gBAAgB,MAAM,IAAI,SAAS,GAAG;AACvG,iBAAW,MAAM,MAAM,uCAAuC,IAAI,CAAC,GAAG,qBAAqB,MAAM,IAAI,SAAS,GAAG;AAAA,IACnH;AACA,eAAW,SAAS,OAAO,SAAS,qBAAqB,GAAG;AAC1D,YAAM,QAAQ,MAAM,CAAC;AACrB,YAAM,WAAW,MAAM,MAAM,gCAAgC,IAAI,CAAC,KAAK;AACvE,YAAM,OAAkB,SAAS,WAAW,QAAQ,IAAI,UAAU,SAAS,WAAW,QAAQ,IAAI,UAAU;AAC5G,UAAI,MAAM,MAAM,+BAA+B,IAAI,CAAC,GAAG,gBAAgB,MAAM,IAAI,IAAI;AACrF,iBAAW,MAAM,MAAM,kCAAkC,IAAI,CAAC,GAAG,uBAAuB,MAAM,IAAI,IAAI;AAAA,IACxG;AACA,eAAW,SAAS,OAAO,SAAS,4BAA4B,GAAG;AACjE,YAAM,OAAO,MAAM,CAAC,EAAE,YAAY;AAClC,YAAM,QAAQ,MAAM,CAAC;AACrB,UAAI,MAAM,MAAM,+BAA+B,IAAI,CAAC,GAAG,UAAU,IAAI,GAAG,MAAM,IAAI,IAAI;AACtF,UAAI,SAAS,QAAS,KAAI,MAAM,MAAM,kCAAkC,IAAI,CAAC,GAAG,sBAAsB,MAAM,IAAI,OAAO;AAAA,IACzH;AACA,eAAW,SAAS,OAAO,SAAS,mBAAmB,GAAG;AACxD,YAAM,QAAQ,MAAM,CAAC;AACrB,YAAM,OAAO,MAAM,MAAM,6CAA6C,IAAI,CAAC,GAAG,YAAY;AAC1F,UAAI,SAAS,cAAc,SAAS,kBAAkB,SAAS,mBAAmB,SAAS,qBAAqB;AAC9G,YAAI,MAAM,MAAM,mCAAmC,IAAI,CAAC,GAAG,UAAU,IAAI,GAAG,MAAM,IAAI,OAAO;AAAA,MAC/F;AAAA,IACF;AACA,eAAW,SAAS,OAAO,SAAS,mBAAmB,GAAG;AACxD,YAAM,QAAQ,MAAM,CAAC;AACrB,UAAI,6DAA6D,KAAK,KAAK,GAAG;AAC5E,YAAI,MAAM,MAAM,gCAAgC,IAAI,CAAC,GAAG,cAAc,MAAM,IAAI,OAAO;AAAA,MACzF;AAAA,IACF;AACA,eAAW,SAAS,OAAO,SAAS,oBAAoB,GAAG;AACzD,YAAM,QAAQ,MAAM,CAAC;AACrB,UAAI,MAAM,MAAM,+CAA+C,IAAI,CAAC,GAAG,mBAAmB,MAAM,IAAI,OAAO;AAAA,IAC7G;AACA,eAAW,SAAS,OAAO,SAAS,mDAAmD,GAAG;AACxF,YAAM,cAAc,MAAM,CAAC;AAC3B,YAAM,OAAO,YAAY,MAAM,gCAAgC,IAAI,CAAC;AACpE,UAAI,CAAC,KAAM;AACX,UACE,wEAAwE,KAAK,WAAW,KACrF,+CAA+C,KAAK,IAAI,GAC3D;AACA,cAAM,MAAM,YAAY,MAAM,CAAC,EAAE,MAAM,+BAA+B,IAAI,CAAC,GAAG,GAAG;AACjF,YAAI,MAAM,yBAAyB,MAAM,IAAI,SAAS,GAAG;AAAA,MAC3D;AAAA,IACF;AACA,eAAW,SAAS,OAAO,SAAS,0IAA0I,GAAG;AAC/K,UAAI,MAAM,CAAC,GAAG,iBAAiB,MAAM,IAAI,OAAO;AAAA,IAClD;AACA,eAAW,SAAS,OAAO,SAAS,sGAAsG,GAAG;AAC3I,UAAI,MAAM,CAAC,GAAG,kBAAkB,MAAM,EAAE;AAAA,IAC1C;AAAA,EACF;AAEA,OAAK,IAAI;AACT,QAAM,YAAY,KACf,QAAQ,eAAe,GAAG,EAAE,QAAQ,eAAe,GAAG,EAAE,QAAQ,aAAa,GAAG,EAChF,QAAQ,WAAW,WAAS,MAAM,MAAM,CAAC,CAAC,EAAE,QAAQ,SAAS,GAAG;AACnE,MAAI,cAAc,KAAM,MAAK,WAAW,iBAAiB;AACzD,SAAO,CAAC,GAAG,MAAM,OAAO,CAAC;AAC3B;AAEO,SAAS,oBACd,cACA,UACA,YAAY,KAC8C;AAC1D,QAAM,QAAQ,oBAAI,IAAsC;AACxD,aAAW,SAAS,aAAc,gBAAe,OAAO,KAAK;AAC7D,aAAW,SAAS,UAAU,UAAU,CAAC,EAAG,gBAAe,OAAO,EAAE,GAAG,OAAO,MAAM,MAAM,KAAK,CAAC;AAEhG,QAAM,WAAW,oBAAI,IAAwC;AAC7D,aAAW,SAAS,MAAM,OAAO,GAAG;AAClC,UAAM,MAAM,eAAe,MAAM,GAAG;AACpC,UAAM,SAAS,SAAS,IAAI,GAAG,KAAK,CAAC;AACrC,WAAO,KAAK,KAAK;AACjB,aAAS,IAAI,KAAK,MAAM;AAAA,EAC1B;AACA,SAAO,CAAC,GAAG,SAAS,OAAO,CAAC,EACzB,IAAI,YAAU;AACb,UAAM,UAAU,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,eAAe,CAAC,IAAI,eAAe,CAAC,KAAK,EAAE,IAAI,cAAc,EAAE,GAAG,CAAC;AAC9G,UAAM,YAAY,EAAE,GAAG,QAAQ,CAAC,EAAE;AAClC,cAAU,mBAAmB,CAAC,GAAG,IAAI,IAAI,OAAO,QAAQ,WAAS,MAAM,gBAAgB,CAAC,CAAC;AACzF,cAAU,WAAW,CAAC,GAAG,IAAI,IAAI,OAAO,QAAQ,WAAS,MAAM,QAAQ,CAAC,CAAC;AACzE,cAAU,WAAW,CAAC,GAAG,IAAI,IAAI,OAAO,QAAQ,WAAS,MAAM,QAAQ,CAAC,CAAC;AACzE,WAAO,EAAE,GAAG,WAAW,UAAU,QAAQ,MAAM,CAAC,EAAE,IAAI,WAAS,MAAM,GAAG,EAAE;AAAA,EAC5E,CAAC,EACA,KAAK,CAAC,GAAG,MAAM,eAAe,CAAC,IAAI,eAAe,CAAC,KAAK,EAAE,IAAI,cAAc,EAAE,GAAG,CAAC,EAClF,MAAM,GAAG,KAAK,IAAI,GAAG,SAAS,CAAC;AACpC;AAMA,eAAsB,cACpB,KACA,SACA,UACA,UAII,CAAC,GACuE;AAC5E,QAAM,WAAW,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,YAAY,KAAK,OAAO,MAAM,MAAM,OAAO,IAAI,CAAC;AAC9F,MAAI,SAAS;AACb,MAAI,MAAuB;AAC3B,WAAS,YAAY,GAAG,aAAa,GAAG,aAAa;AACnD,UAAM,UAAU,MAAM,sBAAsB,QAAQ,EAAE,OAAO,YAAY,CAAC;AAC1E,QAAI,QAAQ,SAAS,CAAC,QAAQ,OAAQ,OAAM,IAAI,MAAM,QAAQ,SAAS,wBAAwB;AAC/F,UAAM,MAAM,MAAM,QAAQ,OAAO,MAAM;AAAA,MACrC,QAAQ,YAAY,QAAQ,IAAM;AAAA,MAClC,UAAU;AAAA,IACZ,CAAC;AACD,QAAI,IAAI,UAAU,OAAO,IAAI,SAAS,KAAK;AACzC,YAAM,WAAW,IAAI,QAAQ,IAAI,UAAU;AAC3C,UAAI,CAAC,SAAU,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,oCAAoC;AACrF,eAAS,IAAI,IAAI,UAAU,QAAQ,OAAO,IAAI,EAAE;AAChD,YAAM;AACN;AAAA,IACF;AACA;AAAA,EACF;AACA,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wCAAwC;AAClE,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,EAAE;AACjD,MAAI,CAAC,IAAI,KAAM,OAAM,IAAI,MAAM,qBAAqB;AAEpD,QAAM,WAAW,IAAI,QAAQ,IAAI,cAAc,GAAG,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK;AAC1E,MAAI,QAAQ,iBAAiB,CAAC,YAAY,aAAa,QAAQ,MAAM,QAAQ,eAAe;AAC1F,UAAM,IAAI,MAAM,YAAY,QAAQ,YAAY,yBAAyB,YAAY,iBAAiB,EAAE;AAAA,EAC1G;AACA,QAAM,gBAAgB,OAAO,IAAI,QAAQ,IAAI,gBAAgB,KAAK,CAAC;AACnE,MAAI,OAAO,SAAS,aAAa,KAAK,gBAAgB,UAAU;AAC9D,UAAM,IAAI,MAAM,uBAAuB,QAAQ,aAAa;AAAA,EAC9D;AAEA,MAAI,OAAO,KAAK,SAAS,QAAQ;AACjC,MAAI,YAAY,CAAC,QAAQ,QAAQ,GAAG;AAClC,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,cAAc;AAAA,MACzD,aAAa;AAAA,MAAQ,iBAAiB;AAAA,MAAQ,cAAc;AAAA,MAC5D,aAAa;AAAA,MAAQ,cAAc;AAAA,MACnC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,aAAa;AAAA,IAC1D;AACA,UAAM,MAAM,QAAQ,QAAQ;AAC5B,QAAI,IAAK,QAAO,OAAO;AAAA,EACzB;AAEA,QAAM,SAAS,kBAAkB,IAAI;AACrC,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,WAAO,KAAK,QAAQ,MAAM,QAAQ,CAAC;AACnC,WAAO,KAAK,SAAS,MAAM;AAAA,EAC7B,CAAC;AACD,MAAI,gBAAgB;AACpB,QAAM,UAAU,IAAI,UAAU;AAAA,IAC5B,UAAU,OAAe,WAAW,UAAU;AAC5C,YAAM,QAAQ,MAAM;AACpB,UAAI,gBAAgB,QAAQ,UAAU;AACpC,iBAAS,IAAI,MAAM,uBAAuB,QAAQ,aAAa,CAAC;AAChE;AAAA,MACF;AACA,UAAI,QAAQ,gBAAgB,CAAC,QAAQ,aAAa,KAAK,GAAG;AACxD,iBAAS,IAAI,MAAM,wCAAwC,CAAC;AAC5D;AAAA,MACF;AACA,uBAAiB;AACjB,eAAS,MAAM,KAAK;AAAA,IACtB;AAAA,EACF,CAAC;AACD,MAAI;AACF,UAAM,SAAS,SAAS,QAAQ,IAAI,IAA2C,GAAG,SAAS,MAAM;AAAA,EACnG,SAAS,OAAO;AACd,WAAO,QAAQ;AACf,WAAO,MAAM,EAAE,OAAO,KAAK,CAAC;AAC5B,UAAM;AAAA,EACR;AAEA,QAAM,EAAE,SAAS,IAAI,MAAM,OAAO,IAAS;AAC3C,QAAM,YAAY,SAAS,IAAI,EAAE;AAEjC,SAAO,EAAE,WAAW,MAAM,WAAW,SAAS;AAChD;AAEA,eAAsB,iBACpB,MACA,SACA,UAA+B,CAAC,GACR;AACxB,QAAM,QAAW,QAAQ,SAAS,CAAC,SAAS,SAAS,OAAO;AAC5D,QAAM,WAAW,IAAI,IAAI,KAAK;AAC9B,QAAM,eAAe,oBAAoB,MAAM,OAAO;AACtD,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,aAAa,KAAK,GAAG,CAAC;AACrE,QAAM,gBAAgB,oBAAoB,cAAc,QAAQ,UAAU,GAAK;AAC/E,QAAM,aAAa,cAAc,MAAM,GAAG,SAAS;AACnD,QAAM,cAAa,oBAAI,IAAI;AAAA,IACzB,GAAG,aAAa,IAAI,WAAS,MAAM,GAAG;AAAA,IACtC,IAAI,QAAQ,UAAU,UAAU,CAAC,GAAG,IAAI,WAAS,MAAM,GAAG;AAAA,EAC5D,CAAC,GAAE;AACH,QAAM,OAAO,WAAW,QAAQ,WAAS;AACvC,UAAM,OAAO,MAAM,QAAQ,YAAY,MAAM,GAAG;AAChD,WAAO,QAAQ,SAAS,IAAI,IAAI,IAAI,CAAC,EAAE,GAAG,OAAO,KAAK,CAAC,IAAI,CAAC;AAAA,EAC9D,CAAC;AACD,QAAM,gBAAgB,aAAa,KAAK;AACxC,QAAM,wBAAwB,cAAc,SAAS;AACrD,QAAM,YAAY,QAAQ,UAAU,cAAc,QAAQ,CAAC;AAC3D,QAAM,aAAa,wBAAwB,gBAAgB,QAAQ,UAAU,cAAc;AAC3F,QAAM,WAAW,wBACb,CAAC,yBAAyB,cAAc,MAAM,4CAA4C,SAAS,yBAAyB,IAC5H,QAAQ,WACR,QAAQ,SAAS,YAAY,CAAC,IAAI,CAAC,yCAAyC,QAAQ,SAAS,UAAU,oCAAoC,IAC3I,CAAC,+FAA+F;AAEpG,QAAM,eAA4D;AAAA,IAChE;AAAA,IACA,aAAa,aAAa;AAAA,IAC1B,eAAe,QAAQ,UAAU,OAAO,UAAU;AAAA,IAClD;AAAA,IACA;AAAA,IACA,cAAc,YAAY,aAAsB;AAAA,IAChD;AAAA,IACA;AAAA,IACA,cAAc,QAAQ,UAAU,gBAAgB;AAAA,IAChD;AAAA,IACA,UAAU;AAAA,EACZ;AAEA,MAAI,QAAQ,cAAc,MAAM;AAC9B,WAAO;AAAA,MACL,GAAG;AAAA,MACH,WAAW;AAAA,MACX,QAAQ,KAAK,IAAI,CAAC,OAAO,OAAO;AAAA,QAC9B,GAAG;AAAA,QACH,UAAU;AAAA,QAAM,UAAU,aAAa,MAAM,KAAK,CAAC;AAAA,QAAG,WAAW;AAAA,QAAM,WAAW;AAAA,QAClF,QAAQ;AAAA,QAAM,gBAAgB;AAAA,QAA0B,eAAe;AAAA,QAAM,eAAe;AAAA,MAC9F,EAAE;AAAA,IACJ;AAAA,EACF;AAEA,QAAM,UAAW,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,OAAO,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IAAE,QAAQ;AAAE,aAAO;AAAA,IAAU;AAAA,EAAE,GAAG;AACtH,QAAM,SAAU,oBAAI,KAAK,GAAE,YAAY,EAAE,QAAQ,SAAS,GAAG,EAAE,MAAM,GAAG,EAAE;AAC1E,QAAM,SAAU,QAAQ,aAAa,KAAK,QAAQ,GAAG,aAAa,eAAe,SAAS,GAAG,KAAK,IAAI,MAAM,EAAE;AAC9G,YAAU,QAAQ,EAAE,WAAW,KAAK,CAAC;AAErC,QAAM,SAAuB,CAAC;AAE9B,QAAM,QAAQ;AAAA,IACZ,KAAK,IAAI,OAAO,OAAO,MAAM;AAC3B,YAAM,WAAW,aAAa,MAAM,KAAK,CAAC;AAC1C,UAAI;AACF,cAAM,EAAE,WAAW,WAAW,SAAS,IAAI,MAAM,cAAc,MAAM,KAAK,QAAQ,QAAQ;AAC1F,cAAM,eAAe,WAAY,aAAa,QAAQ,KAAK,MAAM,OAAQ,MAAM;AAC/E,eAAO,KAAK;AAAA,UACV,GAAG;AAAA,UAAO,MAAM;AAAA,UAAc;AAAA,UAAU,UAAU,SAAS,SAAS;AAAA,UAAG;AAAA,UAAW;AAAA,UAClF,QAAQ;AAAA,UAAM,gBAAgB;AAAA,UAAc,eAAe;AAAA,UAAM,eAAe;AAAA,QAClF,CAAC;AAAA,MACH,SAAS,OAAO;AACd,eAAO,KAAK;AAAA,UACV,GAAG;AAAA,UAAO,UAAU;AAAA,UAAM;AAAA,UAAU,WAAW;AAAA,UAAM,WAAW;AAAA,UAAM,QAAQ;AAAA,UAC9E,gBAAgB;AAAA,UAAU,eAAe,iBAAiB,QAAQ,MAAM,UAAU;AAAA,UAAyB,eAAe;AAAA,QAC5H,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,KAAK,CAAC,GAAG,MAAM;AACpB,QAAI,EAAE,aAAa,CAAC,EAAE,UAAW,QAAO;AACxC,QAAI,CAAC,EAAE,aAAa,EAAE,UAAW,QAAO;AACxC,WAAO,EAAE,IAAI,cAAc,EAAE,GAAG;AAAA,EAClC,CAAC;AAED,SAAO,EAAE,GAAG,cAAc,WAAW,QAAQ,OAAO;AACtD;;;AC3eO,IAAM,+BAA+B;AACrC,IAAM,+BAA+B,IAAI,KAAK,KAAK,KAAK;AACxD,IAAM,+BAA+B,KAAK,KAAK;AAEtD,SAAS,2BAA0C;AACjD,SAAO,QAAQ,IAAI,wCAAwC,KAAK,KAC3D,QAAQ,IAAI,mCAAmC,KAAK,KACpD,QAAQ,IAAI,iCAAiC,KAAK,KAClD,QAAQ,IAAI,sCAAsC,KAAK,KACvD;AACP;AAEA,SAAS,sBAA+B;AACtC,SAAO,QAAQ,IAAI,WAAW,OAAO,QAAQ,IAAI,aAAa;AAChE;AAEA,SAAS,SAAgC;AACvC,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,eAAe;AAAA,IACf,eAAe;AAAA,IACf,OAAO,yBAAyB;AAAA,EAClC;AACF;AAEO,SAAS,2BAA2B,YAAmC;AAC5E,SAAO,uBAAuB,YAAY,4BAA4B;AACxE;AA6BA,eAAsB,sCAAsC,MAK5B;AAC9B,QAAM,UAAU,MAAM,gCAAgC;AAAA,IACpD,QAAQ,OAAO;AAAA,IACf,SAAS,KAAK;AAAA,IACd,aAAa,GAAG,KAAK,KAAK;AAAA,IAC1B,WAAW,KAAK;AAAA,IAChB,UAAU,GAAG,KAAK,KAAK;AAAA,IACvB,aAAa;AAAA,IACb,SAAS,KAAK;AAAA,EAChB,CAAC;AACD,SAAO;AAAA,IACL,KAAK,QAAQ;AAAA,IACb,KAAK,QAAQ,eAAe;AAAA,IAC5B,OAAO,QAAQ;AAAA,IACf,aAAa,QAAQ;AAAA,IACrB,UAAU,QAAQ;AAAA,IAClB,QAAQ,QAAQ;AAAA,IAChB,WAAW,QAAQ;AAAA,IACnB,sBAAsB,QAAQ;AAAA,EAChC;AACF;AAEA,eAAsB,iCAAiC,MAGsC;AAC3F,SAAO,6BAA6B;AAAA,IAClC,QAAQ,OAAO;AAAA,IACf,YAAY,KAAK;AAAA,IACjB,SAAS,KAAK;AAAA,EAChB,CAAC;AACH;AAEA,eAAsB,8BAA8B,YAA4C;AAC9F,SAAO,0BAA0B;AAAA,IAC/B,QAAQ,OAAO;AAAA,IACf;AAAA,IACA,UAAU,KAAK,OAAO;AAAA,EACxB,CAAC;AACH;AAOA,eAAsB,mCAAmC,MAG9B;AACzB,MAAI,2BAA2B,KAAK,UAAU,MAAM,KAAK,QAAS,QAAO;AACzE,SAAO,8BAA8B,KAAK,UAAU;AACtD;AAEA,eAAsB,mCAAmC,OAGrD,CAAC,GAAqG;AACxG,QAAM,MAAM,KAAK,OAAO,oBAAI,KAAK;AACjC,QAAM,QAAQ,yBAAyB;AACvC,MAAI,CAAC,MAAO,QAAO,EAAE,SAAS,GAAG,OAAO,oBAAoB,IAAI,SAAS,QAAQ;AACjF,MAAI,CAAC,KAAK,SAAS,EAAE,IAAI,YAAY,MAAM,KAAK,IAAI,cAAc,MAAM,KAAK;AAC3E,WAAO,EAAE,SAAS,GAAG,OAAO,uBAAuB,SAAS,KAAK;AAAA,EACnE;AACA,QAAM,SAAS,IAAI,QAAQ,IAAI;AAC/B,QAAM,EAAE,MAAM,IAAI,IAAI,MAAM,OAAO,cAAc;AACjD,MAAI;AACJ,MAAI,UAAU;AACd,WAAS,OAAO,GAAG,OAAO,IAAI,QAAQ,GAAG;AACvC,UAAM,SAAS,MAAM,KAAK,EAAE,QAAQ,8BAA8B,OAAO,OAAO,KAAO,OAAO,CAAC;AAC/F,UAAM,UAAU,OAAO,MAAM,OAAO,UAAQ,IAAI,KAAK,KAAK,UAAU,EAAE,QAAQ,KAAK,MAAM;AACzF,QAAI,QAAQ,SAAS,GAAG;AACtB,YAAM,IAAI,QAAQ,IAAI,UAAQ,KAAK,QAAQ,GAAG,EAAE,MAAM,CAAC;AACvD,iBAAW,QAAQ;AAAA,IACrB;AACA,QAAI,CAAC,OAAO,WAAW,CAAC,OAAO,OAAQ;AACvC,aAAS,OAAO;AAAA,EAClB;AACA,SAAO,EAAE,SAAS,OAAO,sBAAsB;AACjD;","names":[]}