mcp-scraper 0.40.1 → 0.40.2

package/dist/extract-bundle-K4PG3RZJ.js

@@ -0,0 +1,568 @@
+import {
+  createSiteExtractBundleArtifactStream,
+  downloadAsset
+} from "./chunk-QZXKQB7Y.js";
+import {
+  computeIssues,
+  renderImageSection,
+  renderIssueReport,
+  renderLinkReport
+} from "./chunk-345BQXZH.js";
+import {
+  extractJobLimitInfo
+} from "./chunk-X2LKCX6H.js";
+import "./chunk-SBLGBZZB.js";
+import "./chunk-44HZLHDV.js";
+import {
+  getDb
+} from "./chunk-GUVKHCKE.js";
+
+// src/api/extract-bundle.ts
+import { createWriteStream, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
+import { basename, join } from "path";
+import { tmpdir } from "os";
+import { createGzip } from "zlib";
+import { once } from "events";
+import { finished } from "stream/promises";
+import { ZipFile } from "yazl";
+import pLimit from "p-limit";
+import { Readable } from "stream";
+var SITE_EXTRACT_PAGE_CHUNK = 25;
+var MAX_IMAGES_PER_PAGE = 20;
+var MAX_IMAGES_PER_SITE = 500;
+var IMAGE_DOWNLOAD_CONCURRENCY = 8;
+var MAX_IMAGE_BYTES = 10 * 1024 * 1024;
+var MAX_SITE_IMAGE_BYTES = 100 * 1024 * 1024;
+var MAX_ANALYZED_LINK_EDGES = 25e4;
+function normalize(u) {
+  return u.split("#")[0].replace(/\/$/, "");
+}
+function registrableDomain(host) {
+  const h = host.replace(/^www\./, "").toLowerCase();
+  const parts = h.split(".");
+  return parts.length <= 2 ? h : parts.slice(-2).join(".");
+}
+function safeImageFilename(url, index) {
+  try {
+    const u = new URL(url);
+    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
+    return base || `image-${index}`;
+  } catch {
+    return `image-${index}`;
+  }
+}
+function slugFactory() {
+  const counts = /* @__PURE__ */ new Map();
+  return (url) => {
+    const base = url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "page";
+    const n = counts.get(base) ?? 0;
+    counts.set(base, n + 1);
+    return n ? `${base}-${n}` : base;
+  };
+}
+async function* pageChunks(jobId) {
+  const db = getDb();
+  let lastRowId = 0;
+  for (; ; ) {
+    const res = await db.execute({
+      sql: `SELECT rowid AS cursor_id, page
+            FROM site_extract_pages
+            WHERE job_id = ? AND rowid > ?
+            ORDER BY rowid
+            LIMIT ${SITE_EXTRACT_PAGE_CHUNK}`,
+      args: [jobId, lastRowId]
+    });
+    if (!res.rows.length) return;
+    lastRowId = Number(res.rows.at(-1)?.cursor_id ?? lastRowId);
+    if (!Number.isSafeInteger(lastRowId) || lastRowId <= 0) throw new Error("site extract page cursor is invalid");
+    yield res.rows.map((r) => JSON.parse(String(r.page)));
+  }
+}
+async function assembleExtractArtifacts(job, extras = {}) {
+  const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`);
+  const downloadImages = job.options.downloadImages === true;
+  mkdirSync(dir, { recursive: true });
+  if (downloadImages) mkdirSync(join(dir, "images"), { recursive: true });
+  try {
+    const metas = [];
+    const pageBodyEntries = [];
+    const bodySlug = slugFactory();
+    const waybackTimeline = job.options.waybackTimeline;
+    const statusByUrl = /* @__PURE__ */ new Map();
+    for await (const chunk of pageChunks(job.id)) {
+      for (const p of chunk) {
+        statusByUrl.set(normalize(p.url), p.status);
+        metas.push({
+          url: p.url,
+          status: p.status,
+          title: p.title,
+          titleLength: p.titleLength,
+          titlePixels: p.titlePixels,
+          metaDescription: p.metaDescription,
+          metaDescLength: p.metaDescLength,
+          h1: p.h1,
+          h1_2: p.h1_2,
+          h2Count: p.h2Count,
+          extractionStatus: p.extractionStatus,
+          indexable: p.indexable,
+          indexabilityReason: p.indexabilityReason,
+          canonicalUrl: p.canonicalUrl ? p.url : null,
+          wordCount: p.wordCount,
+          contentHash: p.contentHash,
+          imagesMissingAlt: p.imagesMissingAlt,
+          schemaTypes: p.schemaTypes.length > 0 ? ["present"] : [],
+          outlinks: []
+        });
+        if (p.bodyMarkdown) {
+          const archiveLines = p.archivedUrl ? `
+Archived URL: ${p.archivedUrl}
+Archive timestamp: ${p.archiveTimestamp ?? "unknown"}
+Requested month: ${p.archiveRequestedMonth ?? "unspecified"}` : "";
+          const content = `# ${p.title ?? p.url}
+
+URL: ${p.url}${archiveLines}
+Status: ${p.status}
+
+---
+
+${p.bodyMarkdown}
+`;
+          const monthDir = p.archiveRequestedMonth ? `${p.archiveRequestedMonth}/` : "";
+          pageBodyEntries.push({ url: p.url, name: `pages/${monthDir}${bodySlug(`${p.archiveRequestedMonth ?? ""}-${p.url}`)}.md`, size: Buffer.byteLength(content) });
+        }
+      }
+    }
+    const gz = createGzip();
+    const linksOut = createWriteStream(join(dir, "links.jsonl.gz"));
+    gz.pipe(linksOut);
+    const gzWrite = async (line) => {
+      if (!gz.write(line)) await once(gz, "drain");
+    };
+    const schemaOut = createWriteStream(join(dir, "schema.jsonl"));
+    const schemaWrite = async (line) => {
+      if (!schemaOut.write(line)) await once(schemaOut, "drain");
+    };
+    const failuresOut = createWriteStream(join(dir, "failures.jsonl"));
+    const failureWrite = async (line) => {
+      if (!failuresOut.write(line)) await once(failuresOut, "drain");
+    };
+    const slug = slugFactory();
+    const inboundFrom = /* @__PURE__ */ new Map();
+    const anchorCounts = /* @__PURE__ */ new Map();
+    const adjacency = /* @__PURE__ */ new Map();
+    const outCounts = /* @__PURE__ */ new Map();
+    const domMap = /* @__PURE__ */ new Map();
+    const brokenLinkPages = /* @__PURE__ */ new Set();
+    let internalTotal = 0;
+    let externalTotal = 0;
+    let brokenInternal = 0;
+    let streamedLinkEdges = 0;
+    let analyzedLinkEdges = 0;
+    let linkAnalysisTruncated = false;
+    let siteReg = "";
+    try {
+      siteReg = registrableDomain(new URL(job.startUrl).hostname);
+    } catch {
+      siteReg = "";
+    }
+    const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY);
+    const imageDownloads = [];
+    let imagesQueued = 0;
+    let imagesFailed = 0;
+    let imageBytesDownloaded = 0;
+    const consumeImageBytes = (bytes) => {
+      if (imageBytesDownloaded + bytes > MAX_SITE_IMAGE_BYTES) return false;
+      imageBytesDownloaded += bytes;
+      return true;
+    };
+    for await (const chunk of pageChunks(job.id)) {
+      for (const p of chunk) {
+        if (p.schema?.length) await schemaWrite(JSON.stringify({ url: p.url, schema: p.schema }) + "\n");
+        if (p.extractionStatus === "failed") {
+          await failureWrite(JSON.stringify({
+            url: p.url,
+            status: p.status,
+            via: p.via,
+            failureCode: p.failureCode ?? null,
+            failureReason: p.failureReason ?? null,
+            fetchAttempts: p.fetchAttempts ?? []
+          }) + "\n");
+        }
+        if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {
+          const pageDir = join(dir, "images", slug(p.url));
+          mkdirSync(pageDir, { recursive: true });
+          for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {
+            if (imagesQueued >= MAX_IMAGES_PER_SITE) break;
+            imagesQueued++;
+            const filename = safeImageFilename(imgUrl, idx);
+            imageDownloads.push(
+              imageLimit(() => downloadAsset(imgUrl, pageDir, filename, {
+                expectedType: "image",
+                maxBytes: MAX_IMAGE_BYTES,
+                consumeBytes: consumeImageBytes
+              })).then(() => void 0, () => {
+                imagesFailed++;
+              })
+            );
+          }
+        }
+        const from = normalize(p.url);
+        const oc = { int: 0, ext: 0 };
+        for (const l of p.outlinks ?? []) {
+          const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
+          const to = normalize(l.href);
+          const targetStatus = statusByUrl.get(to) ?? null;
+          await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + "\n");
+          streamedLinkEdges++;
+          if (analyzedLinkEdges >= MAX_ANALYZED_LINK_EDGES) {
+            linkAnalysisTruncated = true;
+            continue;
+          }
+          analyzedLinkEdges++;
+          if (l.internal) {
+            oc.int++;
+            internalTotal++;
+            if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
+            inboundFrom.get(to).add(from);
+            if (l.anchor) {
+              if (!anchorCounts.has(to)) anchorCounts.set(to, /* @__PURE__ */ new Map());
+              const ac = anchorCounts.get(to);
+              ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1);
+            }
+            if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
+              if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
+              adjacency.get(from).add(to);
+            }
+            if (targetStatus != null && targetStatus >= 400) {
+              brokenInternal++;
+              brokenLinkPages.add(p.url);
+            }
+          } else {
+            oc.ext++;
+            let domain = "";
+            try {
+              domain = registrableDomain(new URL(l.href).hostname);
+            } catch {
+              continue;
+            }
+            if (!domain || domain === siteReg) continue;
+            externalTotal++;
+            const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
+            d.links++;
+            if (nofollow) d.nofollow++;
+            d.pages.add(p.url);
+            domMap.set(domain, d);
+          }
+        }
+        outCounts.set(p.url, oc);
+      }
+    }
+    const outputFinished = Promise.all([finished(linksOut), finished(schemaOut), finished(failuresOut)]);
+    gz.end();
+    schemaOut.end();
+    failuresOut.end();
+    await outputFinished;
+    await Promise.all(imageDownloads);
+    const depth = /* @__PURE__ */ new Map();
+    const start = normalize(job.startUrl);
+    depth.set(start, 0);
+    const queue = [start];
+    while (queue.length > 0) {
+      const cur = queue.shift();
+      const d = depth.get(cur);
+      for (const next of adjacency.get(cur) ?? []) {
+        if (!depth.has(next)) {
+          depth.set(next, d + 1);
+          queue.push(next);
+        }
+      }
+    }
+    const metrics = /* @__PURE__ */ new Map();
+    for (const p of metas) {
+      const key = normalize(p.url);
+      const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
+      const ac = anchorCounts.get(key);
+      const topAnchors = ac ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]) : [];
+      const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 };
+      metrics.set(p.url, {
+        url: p.url,
+        inlinks: inbound.size,
+        uniqueInlinks: inbound.size,
+        outlinksInternal: oc.int,
+        outlinksExternal: oc.ext,
+        crawlDepth: depth.has(key) ? depth.get(key) : null,
+        orphan: inbound.size === 0 && key !== start,
+        topAnchors
+      });
+    }
+    const issues = computeIssues(metas, metrics, brokenLinkPages);
+    let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics);
+    const limits = extractJobLimitInfo(job);
+    const crawlSummary = {
+      jobId: job.id,
+      startUrl: job.startUrl,
+      status: job.successfulUrls === 0 ? "failed" : job.failedUrls > 0 || job.remainingUrls > 0 || limits.creditTruncated ? "partial" : "complete",
+      ...limits,
+      discovered: job.totalUrls,
+      attempted: job.attemptedUrls,
+      successful: job.successfulUrls,
+      failed: job.failedUrls,
+      remaining: job.remainingUrls,
+      linkAnalysis: {
+        truncated: linkAnalysisTruncated,
+        analyzedEdges: analyzedLinkEdges,
+        streamedEdges: streamedLinkEdges,
+        edgeCap: MAX_ANALYZED_LINK_EDGES
+      },
+      generatedAt: (/* @__PURE__ */ new Date()).toISOString()
+    };
+    reportMd = [
+      `## Crawl delivery`,
+      `- Discovered: ${crawlSummary.discovered}`,
+      `- Attempted: ${crawlSummary.attempted}`,
+      `- Successful: ${crawlSummary.successful}`,
+      `- Failed: ${crawlSummary.failed}`,
+      `- Remaining: ${crawlSummary.remaining}`,
+      `- Requested page cap: ${limits.requestedMaxPages}`,
+      `- Funded page cap: ${limits.effectiveMaxPages}`,
+      `- Credit-truncated: ${limits.creditTruncated ? "yes" : "no"}`,
+      `- Link analysis: ${analyzedLinkEdges}/${streamedLinkEdges} edges${linkAnalysisTruncated ? ` (capped at ${MAX_ANALYZED_LINK_EDGES})` : ""}`,
+      "",
+      reportMd
+    ].join("\n");
+    if (extras.imageAudit) reportMd += `
+
+${renderImageSection(extras.imageAudit)}`;
+    const metricValues = [...metrics.values()];
+    const round = (n) => Math.round(n * 10) / 10;
+    const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
+    let sumInlinks = 0;
+    let sumOutlinks = 0;
+    for (const m of metricValues) {
+      sumInlinks += m.inlinks;
+      sumOutlinks += m.outlinksInternal + m.outlinksExternal;
+      if (m.inlinks === 0) distribution.zero++;
+      else if (m.inlinks <= 2) distribution.oneToTwo++;
+      else if (m.inlinks <= 10) distribution.threeToTen++;
+      else distribution.elevenPlus++;
+    }
+    const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
+    const linkReport = {
+      summary: {
+        internal: {
+          totalLinks: internalTotal,
+          pages: metas.length,
+          orphans: metricValues.filter((m) => m.orphan).length,
+          brokenInternal,
+          avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,
+          avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,
+          distribution,
+          topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }))
+        },
+        external: {
+          totalLinks: externalTotal,
+          uniqueDomains: externalDomains.length,
+          topDomains: externalDomains.slice(0, 20)
+        }
+      },
+      externalDomains
+    };
+    const pagesOut = createWriteStream(join(dir, "pages.jsonl"));
+    const captureMatrixOut = waybackTimeline ? createWriteStream(join(dir, "capture-matrix.jsonl")) : null;
+    const capturedCells = /* @__PURE__ */ new Set();
+    const captureCellKey = (month, url) => {
+      let normalized = url;
+      try {
+        normalized = new URL(url).href;
+      } catch {
+      }
+      return `${month ?? ""}\0${normalized}`;
+    };
+    for await (const chunk of pageChunks(job.id)) {
+      for (const page of chunk) {
+        const { bodyMarkdown: _body, schema: _schema, outlinks: _outlinks, ...boundedPage } = page;
+        const m = metrics.get(page.url);
+        const row = { ...boundedPage, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
+        if (!pagesOut.write(JSON.stringify(row) + "\n")) await once(pagesOut, "drain");
+        if (captureMatrixOut) {
+          const cellKey = captureCellKey(page.archiveRequestedMonth, page.originalUrl ?? page.url);
+          capturedCells.add(cellKey);
+          const captureRow = {
+            requestedMonth: page.archiveRequestedMonth ?? null,
+            captureTimestamp: page.archiveTimestamp ?? null,
+            originalUrl: page.originalUrl ?? page.url,
+            archivedUrl: page.archivedUrl ?? null,
+            status: page.extractionStatus === "successful" ? "captured" : "failed",
+            httpStatus: page.status,
+            contentHash: page.contentHash || null,
+            wordCount: page.wordCount,
+            failureCode: page.failureCode ?? null,
+            failureReason: page.failureReason ?? null
+          };
+          if (!captureMatrixOut.write(JSON.stringify(captureRow) + "\n")) await once(captureMatrixOut, "drain");
+        }
+      }
+    }
+    if (captureMatrixOut && waybackTimeline?.timeline?.urls) {
+      const timeline = waybackTimeline.timeline;
+      const requestedUrls = timeline.urls;
+      const months = (() => {
+        if (timeline.months?.length) return [...new Set(timeline.months)].sort();
+        if (!timeline.from || !timeline.to) return [];
+        const values = [];
+        const cursor = /* @__PURE__ */ new Date(`${timeline.from}-01T00:00:00Z`);
+        const end = /* @__PURE__ */ new Date(`${timeline.to}-01T00:00:00Z`);
+        while (cursor <= end && values.length < 60) {
+          values.push(`${cursor.getUTCFullYear()}-${String(cursor.getUTCMonth() + 1).padStart(2, "0")}`);
+          cursor.setUTCMonth(cursor.getUTCMonth() + (timeline.intervalMonths ?? 1));
+        }
+        return values;
+      })();
+      for (const requestedMonth of months) {
+        for (const originalUrl of requestedUrls) {
+          if (capturedCells.has(captureCellKey(requestedMonth, originalUrl))) continue;
+          if (!captureMatrixOut.write(JSON.stringify({
+            requestedMonth,
+            captureTimestamp: null,
+            originalUrl,
+            archivedUrl: null,
+            status: "missing",
+            httpStatus: null,
+            contentHash: null,
+            wordCount: 0,
+            failureCode: "wayback_capture_missing",
+            failureReason: "No successful HTML capture was available in the requested month."
+          }) + "\n")) await once(captureMatrixOut, "drain");
+        }
+      }
+    }
+    const pagesFinished = finished(pagesOut);
+    const captureMatrixFinished = captureMatrixOut ? finished(captureMatrixOut) : Promise.resolve();
+    pagesOut.end();
+    captureMatrixOut?.end();
+    await Promise.all([pagesFinished, captureMatrixFinished]);
+    writeFileSync(join(dir, "report.md"), reportMd);
+    writeFileSync(join(dir, "crawl-summary.json"), JSON.stringify(crawlSummary, null, 2));
+    if (waybackTimeline) {
+      writeFileSync(join(dir, "wayback-manifest.json"), JSON.stringify({
+        rootUrl: waybackTimeline.rootUrl ?? job.startUrl,
+        requested: waybackTimeline.timeline ?? null,
+        maxPagesPerSnapshot: waybackTimeline.maxPagesPerSnapshot ?? null,
+        captureMatrix: "capture-matrix.jsonl"
+      }, null, 2));
+    }
+    writeFileSync(join(dir, "issues.json"), JSON.stringify(issues, null, 2));
+    writeFileSync(join(dir, "link-metrics.jsonl"), metricValues.map((m) => JSON.stringify(m)).join("\n"));
+    writeFileSync(join(dir, "link-report.md"), renderLinkReport(linkReport));
+    writeFileSync(join(dir, "links-summary.json"), JSON.stringify(linkReport.summary, null, 2));
+    writeFileSync(join(dir, "external-domains.json"), JSON.stringify(linkReport.externalDomains, null, 2));
+    if (extras.imageAudit) {
+      writeFileSync(join(dir, "images.jsonl"), extras.imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"));
+      writeFileSync(join(dir, "images-summary.json"), JSON.stringify(extras.imageAudit.summary, null, 2));
+    }
+    if (extras.seoAudit) writeFileSync(join(dir, "seo-audit.json"), JSON.stringify(extras.seoAudit, null, 2));
+    if (extras.branding) writeFileSync(join(dir, "branding.json"), JSON.stringify(extras.branding, null, 2));
+    if (downloadImages) {
+      writeFileSync(join(dir, "images-download-summary.json"), JSON.stringify({
+        queued: imagesQueued,
+        downloaded: imagesQueued - imagesFailed,
+        failed: imagesFailed,
+        downloadedBytes: imageBytesDownloaded,
+        perPageCap: MAX_IMAGES_PER_PAGE,
+        perSiteCap: MAX_IMAGES_PER_SITE,
+        perFileByteCap: MAX_IMAGE_BYTES,
+        totalByteCap: MAX_SITE_IMAGE_BYTES
+      }, null, 2));
+    }
+    const artifactFiles = [
+      { name: "report.md", type: "text/markdown" },
+      { name: "crawl-summary.json", type: "application/json" },
+      { name: "failures.jsonl", type: "application/x-ndjson" },
+      { name: "schema.jsonl", type: "application/x-ndjson" },
+      { name: "issues.json", type: "application/json" },
+      { name: "pages.jsonl", type: "application/x-ndjson" },
+      { name: "links.jsonl.gz", type: "application/gzip" },
+      { name: "link-metrics.jsonl", type: "application/x-ndjson" },
+      { name: "link-report.md", type: "text/markdown" },
+      { name: "links-summary.json", type: "application/json" },
+      { name: "external-domains.json", type: "application/json" }
+    ];
+    if (waybackTimeline) {
+      artifactFiles.push({ name: "wayback-manifest.json", type: "application/json" });
+      artifactFiles.push({ name: "capture-matrix.jsonl", type: "application/x-ndjson" });
+    }
+    if (extras.imageAudit) {
+      artifactFiles.push({ name: "images.jsonl", type: "application/x-ndjson" });
+      artifactFiles.push({ name: "images-summary.json", type: "application/json" });
+    }
+    if (extras.seoAudit) artifactFiles.push({ name: "seo-audit.json", type: "application/json" });
+    if (extras.branding) artifactFiles.push({ name: "branding.json", type: "application/json" });
+    if (downloadImages) artifactFiles.push({ name: "images-download-summary.json", type: "application/json" });
+    const zip = new ZipFile();
+    if (job.userId == null) throw new Error("site extract artifact owner is missing");
+    const artifactPromise = createSiteExtractBundleArtifactStream({
+      ownerId: String(job.userId),
+      jobId: job.id,
+      // Artifact retention starts when this bundle is assembled, including a
+      // later support re-finalization, rather than when the crawl was queued.
+      createdAt: /* @__PURE__ */ new Date(),
+      content: zip.outputStream
+    });
+    for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name);
+    if (pageBodyEntries.length > 0) {
+      const iterator = pageChunks(job.id)[Symbol.asyncIterator]();
+      let pendingPages = [];
+      const nextBodyPage = async () => {
+        for (; ; ) {
+          const page = pendingPages.shift();
+          if (page) {
+            if (page.bodyMarkdown) return page;
+            continue;
+          }
+          const next = await iterator.next();
+          if (next.done) throw new Error("page body stream ended before all ZIP entries were produced");
+          pendingPages = next.value;
+        }
+      };
+      for (const entry of pageBodyEntries) {
+        zip.addReadStreamLazy(entry.name, { size: entry.size }, (callback) => {
+          void nextBodyPage().then((page) => {
+            if (page.url !== entry.url) throw new Error(`page body order changed for ${entry.url}`);
+            const archiveLines = page.archivedUrl ? `
+Archived URL: ${page.archivedUrl}
+Archive timestamp: ${page.archiveTimestamp ?? "unknown"}
+Requested month: ${page.archiveRequestedMonth ?? "unspecified"}` : "";
+            callback(null, Readable.from([
+              `# ${page.title ?? page.url}
+
+URL: ${page.url}${archiveLines}
+Status: ${page.status}
+
+---
+
+${page.bodyMarkdown}
+`
+            ]));
+          }).catch((error) => callback(error instanceof Error ? error : new Error(String(error))));
+        });
+      }
+    }
+    if (downloadImages && imagesQueued > imagesFailed) {
+      const imagesDir = join(dir, "images");
+      for (const rel of readdirSync(imagesDir, { recursive: true })) {
+        const full = join(imagesDir, rel);
+        if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split("\\").join("/")}`);
+      }
+    }
+    zip.end();
+    return [await artifactPromise];
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+export {
+  MAX_ANALYZED_LINK_EDGES,
+  SITE_EXTRACT_PAGE_CHUNK,
+  assembleExtractArtifacts
+};
+//# sourceMappingURL=extract-bundle-K4PG3RZJ.js.map

package/dist/extract-bundle-K4PG3RZJ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/api/extract-bundle.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'\nimport { basename, join } from 'node:path'\nimport { tmpdir } from 'node:os'\nimport { createGzip } from 'node:zlib'\nimport { once } from 'node:events'\nimport { finished } from 'node:stream/promises'\nimport { ZipFile } from 'yazl'\nimport pLimit from 'p-limit'\nimport { Readable } from 'node:stream'\nimport { getDb } from './db.js'\nimport { computeIssues, renderIssueReport } from './seo-issues.js'\nimport { renderLinkReport, type ExtDomainRow, type LinkReport } from './seo-link-report.js'\nimport type { PageLinkMetrics } from './seo-link-graph.js'\nimport type { SeoAudit } from './seo-audit.js'\nimport type { ExtractJobArtifact, ExtractJobRow } from './site-extract-repository.js'\nimport { extractJobLimitInfo } from './site-extract-repository.js'\nimport type { PageData } from './site-extractor.js'\nimport { renderImageSection, type ImageAudit } from './image-audit.js'\nimport { downloadAsset } from '../lib/media-extractor.js'\nimport { createSiteExtractBundleArtifactStream } from './site-extract-artifacts.js'\n\nexport const SITE_EXTRACT_PAGE_CHUNK = 25\nconst MAX_IMAGES_PER_PAGE = 20\nconst MAX_IMAGES_PER_SITE = 500\nconst IMAGE_DOWNLOAD_CONCURRENCY = 8\nconst MAX_IMAGE_BYTES = 10 * 1024 * 1024\nconst MAX_SITE_IMAGE_BYTES = 100 * 1024 * 1024\nexport const MAX_ANALYZED_LINK_EDGES = 250_000\n\nfunction normalize(u: string): string {\n  return u.split('#')[0].replace(/\\/$/, '')\n}\n\nfunction registrableDomain(host: string): string {\n  const h = host.replace(/^www\\./, '').toLowerCase()\n  const parts = h.split('.')\n  return parts.length <= 2 ? h : parts.slice(-2).join('.')\n}\n\nfunction safeImageFilename(url: string, index: number): string {\n  try {\n    const u = new URL(url)\n    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n    return base || `image-${index}`\n  } catch {\n    return `image-${index}`\n  }\n}\n\nfunction slugFactory() {\n  const counts = new Map<string, number>()\n  return (url: string): string => {\n    const base = url.replace(/^https?:\\/\\//, '').replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80) || 'page'\n    const n = counts.get(base) ?? 0\n    counts.set(base, n + 1)\n    return n ? `${base}-${n}` : base\n  }\n}\n\nasync function* pageChunks(jobId: string): AsyncGenerator<PageData[]> {\n  const db = getDb()\n  let lastRowId = 0\n  for (;;) {\n    const res = await db.execute({\n      sql: `SELECT rowid AS cursor_id, page\n            FROM site_extract_pages\n            WHERE job_id = ? AND rowid > ?\n            ORDER BY rowid\n            LIMIT ${SITE_EXTRACT_PAGE_CHUNK}`,\n      args: [jobId, lastRowId],\n    })\n    if (!res.rows.length) return\n    lastRowId = Number(res.rows.at(-1)?.cursor_id ?? lastRowId)\n    if (!Number.isSafeInteger(lastRowId) || lastRowId <= 0) throw new Error('site extract page cursor is invalid')\n    yield res.rows.map(r => JSON.parse(String((r as unknown as Record<string, unknown>).page)) as PageData)\n  }\n}\n\nexport interface AssembleExtras {\n  branding?: unknown\n  imageAudit?: ImageAudit | null\n  seoAudit?: SeoAudit | null\n}\n\nexport async function assembleExtractArtifacts(job: ExtractJobRow, extras: AssembleExtras = {}): Promise<ExtractJobArtifact[]> {\n  const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`)\n  const downloadImages = job.options.downloadImages === true\n  mkdirSync(dir, { recursive: true })\n  if (downloadImages) mkdirSync(join(dir, 'images'), { recursive: true })\n  try {\n    const metas: PageData[] = []\n    const pageBodyEntries: Array<{ url: string; name: string; size: number }> = []\n    const bodySlug = slugFactory()\n    const waybackTimeline = job.options.waybackTimeline as {\n      rootUrl?: string\n      timeline?: { months?: string[]; from?: string; to?: string; intervalMonths?: number; urls?: string[] }\n      maxPagesPerSnapshot?: number\n    } | undefined\n    const statusByUrl = new Map<string, number | null>()\n    for await (const chunk of pageChunks(job.id)) {\n      for (const p of chunk) {\n        statusByUrl.set(normalize(p.url), p.status)\n        // Keep only fields needed by computeIssues/renderIssueReport. Holding\n        // every bounded PageData field for 10k pages can still consume\n        // hundreds of MiB even though DB reads are streamed in small chunks.\n        metas.push({\n          url: p.url,\n          status: p.status,\n          title: p.title,\n          titleLength: p.titleLength,\n          titlePixels: p.titlePixels,\n          metaDescription: p.metaDescription,\n          metaDescLength: p.metaDescLength,\n          h1: p.h1,\n          h1_2: p.h1_2,\n          h2Count: p.h2Count,\n          extractionStatus: p.extractionStatus,\n          indexable: p.indexable,\n          indexabilityReason: p.indexabilityReason,\n          canonicalUrl: p.canonicalUrl ? p.url : null,\n          wordCount: p.wordCount,\n          contentHash: p.contentHash,\n          imagesMissingAlt: p.imagesMissingAlt,\n          schemaTypes: p.schemaTypes.length > 0 ? ['present'] : [],\n          outlinks: [],\n        } as unknown as PageData)\n        if (p.bodyMarkdown) {\n          const archiveLines = p.archivedUrl\n            ? `\\nArchived URL: ${p.archivedUrl}\\nArchive timestamp: ${p.archiveTimestamp ?? 'unknown'}\\nRequested month: ${p.archiveRequestedMonth ?? 'unspecified'}`\n            : ''\n          const content = `# ${p.title ?? p.url}\\n\\nURL: ${p.url}${archiveLines}\\nStatus: ${p.status}\\n\\n---\\n\\n${p.bodyMarkdown}\\n`\n          const monthDir = p.archiveRequestedMonth ? `${p.archiveRequestedMonth}/` : ''\n          pageBodyEntries.push({ url: p.url, name: `pages/${monthDir}${bodySlug(`${p.archiveRequestedMonth ?? ''}-${p.url}`)}.md`, size: Buffer.byteLength(content) })\n        }\n      }\n    }\n\n    const gz = createGzip()\n    const linksOut = createWriteStream(join(dir, 'links.jsonl.gz'))\n    gz.pipe(linksOut)\n    const gzWrite = async (line: string) => { if (!gz.write(line)) await once(gz, 'drain') }\n    const schemaOut = createWriteStream(join(dir, 'schema.jsonl'))\n    const schemaWrite = async (line: string) => { if (!schemaOut.write(line)) await once(schemaOut, 'drain') }\n    const failuresOut = createWriteStream(join(dir, 'failures.jsonl'))\n    const failureWrite = async (line: string) => { if (!failuresOut.write(line)) await once(failuresOut, 'drain') }\n\n    const slug = slugFactory()\n    const inboundFrom = new Map<string, Set<string>>()\n    const anchorCounts = new Map<string, Map<string, number>>()\n    const adjacency = new Map<string, Set<string>>()\n    const outCounts = new Map<string, { int: number; ext: number }>()\n    const domMap = new Map<string, { links: number; nofollow: number; pages: Set<string> }>()\n    const brokenLinkPages = new Set<string>()\n    let internalTotal = 0\n    let externalTotal = 0\n    let brokenInternal = 0\n    let streamedLinkEdges = 0\n    let analyzedLinkEdges = 0\n    let linkAnalysisTruncated = false\n    let siteReg = ''\n    try { siteReg = registrableDomain(new URL(job.startUrl).hostname) } catch { siteReg = '' }\n\n    const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY)\n    const imageDownloads: Promise<void>[] = []\n    let imagesQueued = 0\n    let imagesFailed = 0\n    let imageBytesDownloaded = 0\n    const consumeImageBytes = (bytes: number) => {\n      if (imageBytesDownloaded + bytes > MAX_SITE_IMAGE_BYTES) return false\n      imageBytesDownloaded += bytes\n      return true\n    }\n\n    for await (const chunk of pageChunks(job.id)) {\n      for (const p of chunk) {\n        if (p.schema?.length) await schemaWrite(JSON.stringify({ url: p.url, schema: p.schema }) + '\\n')\n        if (p.extractionStatus === 'failed') {\n          await failureWrite(JSON.stringify({\n            url: p.url,\n            status: p.status,\n            via: p.via,\n            failureCode: p.failureCode ?? null,\n            failureReason: p.failureReason ?? null,\n            fetchAttempts: p.fetchAttempts ?? [],\n          }) + '\\n')\n        }\n        if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {\n          const pageDir = join(dir, 'images', slug(p.url))\n          mkdirSync(pageDir, { recursive: true })\n          for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {\n            if (imagesQueued >= MAX_IMAGES_PER_SITE) break\n            imagesQueued++\n            const filename = safeImageFilename(imgUrl, idx)\n            imageDownloads.push(\n              imageLimit(() => downloadAsset(imgUrl, pageDir, filename, {\n                expectedType: 'image',\n                maxBytes: MAX_IMAGE_BYTES,\n                consumeBytes: consumeImageBytes,\n              })).then(() => undefined, () => { imagesFailed++ }),\n            )\n          }\n        }\n        const from = normalize(p.url)\n        const oc = { int: 0, ext: 0 }\n        for (const l of p.outlinks ?? []) {\n          const nofollow = (l.rel ?? '').split(/\\s+/).includes('nofollow')\n          const to = normalize(l.href)\n          const targetStatus = statusByUrl.get(to) ?? null\n          await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + '\\n')\n          streamedLinkEdges++\n          if (analyzedLinkEdges >= MAX_ANALYZED_LINK_EDGES) {\n            linkAnalysisTruncated = true\n            continue\n          }\n          analyzedLinkEdges++\n          if (l.internal) {\n            oc.int++\n            internalTotal++\n            if (!inboundFrom.has(to)) inboundFrom.set(to, new Set())\n            inboundFrom.get(to)!.add(from)\n            if (l.anchor) {\n              if (!anchorCounts.has(to)) anchorCounts.set(to, new Map())\n              const ac = anchorCounts.get(to)!\n              ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1)\n            }\n            if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {\n              if (!adjacency.has(from)) adjacency.set(from, new Set())\n              adjacency.get(from)!.add(to)\n            }\n            if (targetStatus != null && targetStatus >= 400) {\n              brokenInternal++\n              brokenLinkPages.add(p.url)\n            }\n          } else {\n            oc.ext++\n            let domain = ''\n            try { domain = registrableDomain(new URL(l.href).hostname) } catch { continue }\n            if (!domain || domain === siteReg) continue\n            externalTotal++\n            const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: new Set<string>() }\n            d.links++\n            if (nofollow) d.nofollow++\n            d.pages.add(p.url)\n            domMap.set(domain, d)\n          }\n        }\n        outCounts.set(p.url, oc)\n      }\n    }\n    const outputFinished = Promise.all([finished(linksOut), finished(schemaOut), finished(failuresOut)])\n    gz.end()\n    schemaOut.end()\n    failuresOut.end()\n    await outputFinished\n    await Promise.all(imageDownloads)\n\n    const depth = new Map<string, number>()\n    const start = normalize(job.startUrl)\n    depth.set(start, 0)\n    const queue: string[] = [start]\n    while (queue.length > 0) {\n      const cur = queue.shift()!\n      const d = depth.get(cur)!\n      for (const next of adjacency.get(cur) ?? []) {\n        if (!depth.has(next)) { depth.set(next, d + 1); queue.push(next) }\n      }\n    }\n\n    const metrics = new Map<string, PageLinkMetrics>()\n    for (const p of metas) {\n      const key = normalize(p.url)\n      const inbound = inboundFrom.get(key) ?? new Set()\n      const ac = anchorCounts.get(key)\n      const topAnchors = ac\n        ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(e => e[0])\n        : []\n      const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 }\n      metrics.set(p.url, {\n        url: p.url,\n        inlinks: inbound.size,\n        uniqueInlinks: inbound.size,\n        outlinksInternal: oc.int,\n        outlinksExternal: oc.ext,\n        crawlDepth: depth.has(key) ? depth.get(key)! : null,\n        orphan: inbound.size === 0 && key !== start,\n        topAnchors,\n      })\n    }\n\n    const issues = computeIssues(metas, metrics, brokenLinkPages)\n    let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics)\n    const limits = extractJobLimitInfo(job)\n    const crawlSummary = {\n      jobId: job.id,\n      startUrl: job.startUrl,\n      status: job.successfulUrls === 0\n        ? 'failed'\n        : job.failedUrls > 0 || job.remainingUrls > 0 || limits.creditTruncated\n          ? 'partial'\n          : 'complete',\n      ...limits,\n      discovered: job.totalUrls,\n      attempted: job.attemptedUrls,\n      successful: job.successfulUrls,\n      failed: job.failedUrls,\n      remaining: job.remainingUrls,\n      linkAnalysis: {\n        truncated: linkAnalysisTruncated,\n        analyzedEdges: analyzedLinkEdges,\n        streamedEdges: streamedLinkEdges,\n        edgeCap: MAX_ANALYZED_LINK_EDGES,\n      },\n      generatedAt: new Date().toISOString(),\n    }\n    reportMd = [\n      `## Crawl delivery`,\n      `- Discovered: ${crawlSummary.discovered}`,\n      `- Attempted: ${crawlSummary.attempted}`,\n      `- Successful: ${crawlSummary.successful}`,\n      `- Failed: ${crawlSummary.failed}`,\n      `- Remaining: ${crawlSummary.remaining}`,\n      `- Requested page cap: ${limits.requestedMaxPages}`,\n      `- Funded page cap: ${limits.effectiveMaxPages}`,\n      `- Credit-truncated: ${limits.creditTruncated ? 'yes' : 'no'}`,\n      `- Link analysis: ${analyzedLinkEdges}/${streamedLinkEdges} edges${linkAnalysisTruncated ? ` (capped at ${MAX_ANALYZED_LINK_EDGES})` : ''}`,\n      '',\n      reportMd,\n    ].join('\\n')\n    if (extras.imageAudit) reportMd += `\\n\\n${renderImageSection(extras.imageAudit)}`\n\n    const metricValues = [...metrics.values()]\n    const round = (n: number) => Math.round(n * 10) / 10\n    const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 }\n    let sumInlinks = 0\n    let sumOutlinks = 0\n    for (const m of metricValues) {\n      sumInlinks += m.inlinks\n      sumOutlinks += m.outlinksInternal + m.outlinksExternal\n      if (m.inlinks === 0) distribution.zero++\n      else if (m.inlinks <= 2) distribution.oneToTwo++\n      else if (m.inlinks <= 10) distribution.threeToTen++\n      else distribution.elevenPlus++\n    }\n    const externalDomains: ExtDomainRow[] = [...domMap.entries()]\n      .map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size }))\n      .sort((a, b) => b.links - a.links)\n    const linkReport: LinkReport = {\n      summary: {\n        internal: {\n          totalLinks: internalTotal,\n          pages: metas.length,\n          orphans: metricValues.filter(m => m.orphan).length,\n          brokenInternal,\n          avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,\n          avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,\n          distribution,\n          topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20)\n            .map(m => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal })),\n        },\n        external: {\n          totalLinks: externalTotal,\n          uniqueDomains: externalDomains.length,\n          topDomains: externalDomains.slice(0, 20),\n        },\n      },\n      externalDomains,\n    }\n\n    const pagesOut = createWriteStream(join(dir, 'pages.jsonl'))\n    const captureMatrixOut = waybackTimeline\n      ? createWriteStream(join(dir, 'capture-matrix.jsonl'))\n      : null\n    const capturedCells = new Set<string>()\n    const captureCellKey = (month: string | null | undefined, url: string) => {\n      let normalized = url\n      try { normalized = new URL(url).href } catch { /** Keep the bounded stored value. */ }\n      return `${month ?? ''}\\0${normalized}`\n    }\n    for await (const chunk of pageChunks(job.id)) {\n      for (const page of chunk) {\n        const { bodyMarkdown: _body, schema: _schema, outlinks: _outlinks, ...boundedPage } = page\n        const m = metrics.get(page.url)\n        const row = { ...boundedPage, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false }\n        if (!pagesOut.write(JSON.stringify(row) + '\\n')) await once(pagesOut, 'drain')\n        if (captureMatrixOut) {\n          const cellKey = captureCellKey(page.archiveRequestedMonth, page.originalUrl ?? page.url)\n          capturedCells.add(cellKey)\n          const captureRow = {\n            requestedMonth: page.archiveRequestedMonth ?? null,\n            captureTimestamp: page.archiveTimestamp ?? null,\n            originalUrl: page.originalUrl ?? page.url,\n            archivedUrl: page.archivedUrl ?? null,\n            status: page.extractionStatus === 'successful' ? 'captured' : 'failed',\n            httpStatus: page.status,\n            contentHash: page.contentHash || null,\n            wordCount: page.wordCount,\n            failureCode: page.failureCode ?? null,\n            failureReason: page.failureReason ?? null,\n          }\n          if (!captureMatrixOut.write(JSON.stringify(captureRow) + '\\n')) await once(captureMatrixOut, 'drain')\n        }\n      }\n    }\n    if (captureMatrixOut && waybackTimeline?.timeline?.urls) {\n      const timeline = waybackTimeline.timeline\n      const requestedUrls = timeline.urls!\n      const months = (() => {\n        if (timeline.months?.length) return [...new Set(timeline.months)].sort()\n        if (!timeline.from || !timeline.to) return []\n        const values: string[] = []\n        const cursor = new Date(`${timeline.from}-01T00:00:00Z`)\n        const end = new Date(`${timeline.to}-01T00:00:00Z`)\n        while (cursor <= end && values.length < 60) {\n          values.push(`${cursor.getUTCFullYear()}-${String(cursor.getUTCMonth() + 1).padStart(2, '0')}`)\n          cursor.setUTCMonth(cursor.getUTCMonth() + (timeline.intervalMonths ?? 1))\n        }\n        return values\n      })()\n      for (const requestedMonth of months) {\n        for (const originalUrl of requestedUrls) {\n          if (capturedCells.has(captureCellKey(requestedMonth, originalUrl))) continue\n          if (!captureMatrixOut.write(JSON.stringify({\n            requestedMonth,\n            captureTimestamp: null,\n            originalUrl,\n            archivedUrl: null,\n            status: 'missing',\n            httpStatus: null,\n            contentHash: null,\n            wordCount: 0,\n            failureCode: 'wayback_capture_missing',\n            failureReason: 'No successful HTML capture was available in the requested month.',\n          }) + '\\n')) await once(captureMatrixOut, 'drain')\n        }\n      }\n    }\n    const pagesFinished = finished(pagesOut)\n    const captureMatrixFinished = captureMatrixOut ? finished(captureMatrixOut) : Promise.resolve()\n    pagesOut.end()\n    captureMatrixOut?.end()\n    await Promise.all([pagesFinished, captureMatrixFinished])\n\n    writeFileSync(join(dir, 'report.md'), reportMd)\n    writeFileSync(join(dir, 'crawl-summary.json'), JSON.stringify(crawlSummary, null, 2))\n    if (waybackTimeline) {\n      writeFileSync(join(dir, 'wayback-manifest.json'), JSON.stringify({\n        rootUrl: waybackTimeline.rootUrl ?? job.startUrl,\n        requested: waybackTimeline.timeline ?? null,\n        maxPagesPerSnapshot: waybackTimeline.maxPagesPerSnapshot ?? null,\n        captureMatrix: 'capture-matrix.jsonl',\n      }, null, 2))\n    }\n    writeFileSync(join(dir, 'issues.json'), JSON.stringify(issues, null, 2))\n    writeFileSync(join(dir, 'link-metrics.jsonl'), metricValues.map(m => JSON.stringify(m)).join('\\n'))\n    writeFileSync(join(dir, 'link-report.md'), renderLinkReport(linkReport))\n    writeFileSync(join(dir, 'links-summary.json'), JSON.stringify(linkReport.summary, null, 2))\n    writeFileSync(join(dir, 'external-domains.json'), JSON.stringify(linkReport.externalDomains, null, 2))\n    if (extras.imageAudit) {\n      writeFileSync(join(dir, 'images.jsonl'), extras.imageAudit.rows.map(r => JSON.stringify(r)).join('\\n'))\n      writeFileSync(join(dir, 'images-summary.json'), JSON.stringify(extras.imageAudit.summary, null, 2))\n    }\n    if (extras.seoAudit) writeFileSync(join(dir, 'seo-audit.json'), JSON.stringify(extras.seoAudit, null, 2))\n    if (extras.branding) writeFileSync(join(dir, 'branding.json'), JSON.stringify(extras.branding, null, 2))\n    if (downloadImages) {\n      writeFileSync(join(dir, 'images-download-summary.json'), JSON.stringify({\n        queued: imagesQueued,\n        downloaded: imagesQueued - imagesFailed,\n        failed: imagesFailed,\n        downloadedBytes: imageBytesDownloaded,\n        perPageCap: MAX_IMAGES_PER_PAGE,\n        perSiteCap: MAX_IMAGES_PER_SITE,\n        perFileByteCap: MAX_IMAGE_BYTES,\n        totalByteCap: MAX_SITE_IMAGE_BYTES,\n      }, null, 2))\n    }\n\n    const artifactFiles: Array<{ name: string; type: string }> = [\n      { name: 'report.md', type: 'text/markdown' },\n      { name: 'crawl-summary.json', type: 'application/json' },\n      { name: 'failures.jsonl', type: 'application/x-ndjson' },\n      { name: 'schema.jsonl', type: 'application/x-ndjson' },\n      { name: 'issues.json', type: 'application/json' },\n      { name: 'pages.jsonl', type: 'application/x-ndjson' },\n      { name: 'links.jsonl.gz', type: 'application/gzip' },\n      { name: 'link-metrics.jsonl', type: 'application/x-ndjson' },\n      { name: 'link-report.md', type: 'text/markdown' },\n      { name: 'links-summary.json', type: 'application/json' },\n      { name: 'external-domains.json', type: 'application/json' },\n    ]\n    if (waybackTimeline) {\n      artifactFiles.push({ name: 'wayback-manifest.json', type: 'application/json' })\n      artifactFiles.push({ name: 'capture-matrix.jsonl', type: 'application/x-ndjson' })\n    }\n    if (extras.imageAudit) {\n      artifactFiles.push({ name: 'images.jsonl', type: 'application/x-ndjson' })\n      artifactFiles.push({ name: 'images-summary.json', type: 'application/json' })\n    }\n    if (extras.seoAudit) artifactFiles.push({ name: 'seo-audit.json', type: 'application/json' })\n    if (extras.branding) artifactFiles.push({ name: 'branding.json', type: 'application/json' })\n    if (downloadImages) artifactFiles.push({ name: 'images-download-summary.json', type: 'application/json' })\n\n    const zip = new ZipFile()\n    if (job.userId == null) throw new Error('site extract artifact owner is missing')\n    const artifactPromise = createSiteExtractBundleArtifactStream({\n      ownerId: String(job.userId),\n      jobId: job.id,\n      // Artifact retention starts when this bundle is assembled, including a\n      // later support re-finalization, rather than when the crawl was queued.\n      createdAt: new Date(),\n      content: zip.outputStream,\n    })\n    for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name)\n    if (pageBodyEntries.length > 0) {\n      const iterator = pageChunks(job.id)[Symbol.asyncIterator]()\n      let pendingPages: PageData[] = []\n      const nextBodyPage = async (): Promise<PageData> => {\n        for (;;) {\n          const page = pendingPages.shift()\n          if (page) {\n            if (page.bodyMarkdown) return page\n            continue\n          }\n          const next = await iterator.next()\n          if (next.done) throw new Error('page body stream ended before all ZIP entries were produced')\n          pendingPages = next.value\n        }\n      }\n      for (const entry of pageBodyEntries) {\n        zip.addReadStreamLazy(entry.name, { size: entry.size }, callback => {\n          void nextBodyPage().then(page => {\n            if (page.url !== entry.url) throw new Error(`page body order changed for ${entry.url}`)\n            const archiveLines = page.archivedUrl\n              ? `\\nArchived URL: ${page.archivedUrl}\\nArchive timestamp: ${page.archiveTimestamp ?? 'unknown'}\\nRequested month: ${page.archiveRequestedMonth ?? 'unspecified'}`\n              : ''\n            callback(null, Readable.from([\n              `# ${page.title ?? page.url}\\n\\nURL: ${page.url}${archiveLines}\\nStatus: ${page.status}\\n\\n---\\n\\n${page.bodyMarkdown}\\n`,\n            ]))\n          }).catch(error => callback(error instanceof Error ? error : new Error(String(error))))\n        })\n      }\n    }\n    if (downloadImages && imagesQueued > imagesFailed) {\n      const imagesDir = join(dir, 'images')\n      for (const rel of readdirSync(imagesDir, { recursive: true }) as string[]) {\n        const full = join(imagesDir, rel)\n        if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split('\\\\').join('/')}`)\n      }\n    }\n    zip.end()\n    return [await artifactPromise]\n  } finally {\n    rmSync(dir, { recursive: true, force: true })\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,SAAS,mBAAmB,WAAW,aAAa,QAAQ,UAAU,qBAAqB;AAC3F,SAAS,UAAU,YAAY;AAC/B,SAAS,cAAc;AACvB,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AACrB,SAAS,gBAAgB;AACzB,SAAS,eAAe;AACxB,OAAO,YAAY;AACnB,SAAS,gBAAgB;AAalB,IAAM,0BAA0B;AACvC,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAC5B,IAAM,6BAA6B;AACnC,IAAM,kBAAkB,KAAK,OAAO;AACpC,IAAM,uBAAuB,MAAM,OAAO;AACnC,IAAM,0BAA0B;AAEvC,SAAS,UAAU,GAAmB;AACpC,SAAO,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,OAAO,EAAE;AAC1C;AAEA,SAAS,kBAAkB,MAAsB;AAC/C,QAAM,IAAI,KAAK,QAAQ,UAAU,EAAE,EAAE,YAAY;AACjD,QAAM,QAAQ,EAAE,MAAM,GAAG;AACzB,SAAO,MAAM,UAAU,IAAI,IAAI,MAAM,MAAM,EAAE,EAAE,KAAK,GAAG;AACzD;AAEA,SAAS,kBAAkB,KAAa,OAAuB;AAC7D,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEA,SAAS,cAAc;AACrB,QAAM,SAAS,oBAAI,IAAoB;AACvC,SAAO,CAAC,QAAwB;AAC9B,UAAM,OAAO,IAAI,QAAQ,gBAAgB,EAAE,EAAE,QAAQ,kBAAkB,GAAG,EAAE,QAAQ,YAAY,EAAE,EAAE,MAAM,GAAG,EAAE,KAAK;AACpH,UAAM,IAAI,OAAO,IAAI,IAAI,KAAK;AAC9B,WAAO,IAAI,MAAM,IAAI,CAAC;AACtB,WAAO,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK;AAAA,EAC9B;AACF;AAEA,gBAAgB,WAAW,OAA2C;AACpE,QAAM,KAAK,MAAM;AACjB,MAAI,YAAY;AAChB,aAAS;AACP,UAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,MAC3B,KAAK;AAAA;AAAA;AAAA;AAAA,oBAIS,uBAAuB;AAAA,MACrC,MAAM,CAAC,OAAO,SAAS;AAAA,IACzB,CAAC;AACD,QAAI,CAAC,IAAI,KAAK,OAAQ;AACtB,gBAAY,OAAO,IAAI,KAAK,GAAG,EAAE,GAAG,aAAa,SAAS;AAC1D,QAAI,CAAC,OAAO,cAAc,SAAS,KAAK,aAAa,EAAG,OAAM,IAAI,MAAM,qCAAqC;AAC7G,UAAM,IAAI,KAAK,IAAI,OAAK,KAAK,MAAM,OAAQ,EAAyC,IAAI,CAAC,CAAa;AAAA,EACxG;AACF;AAQA,eAAsB,yBAAyB,KAAoB,SAAyB,CAAC,GAAkC;AAC7H,QAAM,MAAM,KAAK,OAAO,GAAG,kBAAkB,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AACnE,QAAM,iBAAiB,IAAI,QAAQ,mBAAmB;AACtD,YAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAClC,MAAI,eAAgB,WAAU,KAAK,KAAK,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;AACtE,MAAI;AACF,UAAM,QAAoB,CAAC;AAC3B,UAAM,kBAAsE,CAAC;AAC7E,UAAM,WAAW,YAAY;AAC7B,UAAM,kBAAkB,IAAI,QAAQ;AAKpC,UAAM,cAAc,oBAAI,IAA2B;AACnD,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,oBAAY,IAAI,UAAU,EAAE,GAAG,GAAG,EAAE,MAAM;AAI1C,cAAM,KAAK;AAAA,UACT,KAAK,EAAE;AAAA,UACP,QAAQ,EAAE;AAAA,UACV,OAAO,EAAE;AAAA,UACT,aAAa,EAAE;AAAA,UACf,aAAa,EAAE;AAAA,UACf,iBAAiB,EAAE;AAAA,UACnB,gBAAgB,EAAE;AAAA,UAClB,IAAI,EAAE;AAAA,UACN,MAAM,EAAE;AAAA,UACR,SAAS,EAAE;AAAA,UACX,kBAAkB,EAAE;AAAA,UACpB,WAAW,EAAE;AAAA,UACb,oBAAoB,EAAE;AAAA,UACtB,cAAc,EAAE,eAAe,EAAE,MAAM;AAAA,UACvC,WAAW,EAAE;AAAA,UACb,aAAa,EAAE;AAAA,UACf,kBAAkB,EAAE;AAAA,UACpB,aAAa,EAAE,YAAY,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC;AAAA,UACvD,UAAU,CAAC;AAAA,QACb,CAAwB;AACxB,YAAI,EAAE,cAAc;AAClB,gBAAM,eAAe,EAAE,cACnB;AAAA,gBAAmB,EAAE,WAAW;AAAA,qBAAwB,EAAE,oBAAoB,SAAS;AAAA,mBAAsB,EAAE,yBAAyB,aAAa,KACrJ;AACJ,gBAAM,UAAU,KAAK,EAAE,SAAS,EAAE,GAAG;AAAA;AAAA,OAAY,EAAE,GAAG,GAAG,YAAY;AAAA,UAAa,EAAE,MAAM;AAAA;AAAA;AAAA;AAAA,EAAc,EAAE,YAAY;AAAA;AACtH,gBAAM,WAAW,EAAE,wBAAwB,GAAG,EAAE,qBAAqB,MAAM;AAC3E,0BAAgB,KAAK,EAAE,KAAK,EAAE,KAAK,MAAM,SAAS,QAAQ,GAAG,SAAS,GAAG,EAAE,yBAAyB,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,MAAM,OAAO,WAAW,OAAO,EAAE,CAAC;AAAA,QAC7J;AAAA,MACF;AAAA,IACF;AAEA,UAAM,KAAK,WAAW;AACtB,UAAM,WAAW,kBAAkB,KAAK,KAAK,gBAAgB,CAAC;AAC9D,OAAG,KAAK,QAAQ;AAChB,UAAM,UAAU,OAAO,SAAiB;AAAE,UAAI,CAAC,GAAG,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI,OAAO;AAAA,IAAE;AACvF,UAAM,YAAY,kBAAkB,KAAK,KAAK,cAAc,CAAC;AAC7D,UAAM,cAAc,OAAO,SAAiB;AAAE,UAAI,CAAC,UAAU,MAAM,IAAI,EAAG,OAAM,KAAK,WAAW,OAAO;AAAA,IAAE;AACzG,UAAM,cAAc,kBAAkB,KAAK,KAAK,gBAAgB,CAAC;AACjE,UAAM,eAAe,OAAO,SAAiB;AAAE,UAAI,CAAC,YAAY,MAAM,IAAI,EAAG,OAAM,KAAK,aAAa,OAAO;AAAA,IAAE;AAE9G,UAAM,OAAO,YAAY;AACzB,UAAM,cAAc,oBAAI,IAAyB;AACjD,UAAM,eAAe,oBAAI,IAAiC;AAC1D,UAAM,YAAY,oBAAI,IAAyB;AAC/C,UAAM,YAAY,oBAAI,IAA0C;AAChE,UAAM,SAAS,oBAAI,IAAqE;AACxF,UAAM,kBAAkB,oBAAI,IAAY;AACxC,QAAI,gBAAgB;AACpB,QAAI,gBAAgB;AACpB,QAAI,iBAAiB;AACrB,QAAI,oBAAoB;AACxB,QAAI,oBAAoB;AACxB,QAAI,wBAAwB;AAC5B,QAAI,UAAU;AACd,QAAI;AAAE,gBAAU,kBAAkB,IAAI,IAAI,IAAI,QAAQ,EAAE,QAAQ;AAAA,IAAE,QAAQ;AAAE,gBAAU;AAAA,IAAG;AAEzF,UAAM,aAAa,OAAO,0BAA0B;AACpD,UAAM,iBAAkC,CAAC;AACzC,QAAI,eAAe;AACnB,QAAI,eAAe;AACnB,QAAI,uBAAuB;AAC3B,UAAM,oBAAoB,CAAC,UAAkB;AAC3C,UAAI,uBAAuB,QAAQ,qBAAsB,QAAO;AAChE,8BAAwB;AACxB,aAAO;AAAA,IACT;AAEA,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,YAAI,EAAE,QAAQ,OAAQ,OAAM,YAAY,KAAK,UAAU,EAAE,KAAK,EAAE,KAAK,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI;AAC/F,YAAI,EAAE,qBAAqB,UAAU;AACnC,gBAAM,aAAa,KAAK,UAAU;AAAA,YAChC,KAAK,EAAE;AAAA,YACP,QAAQ,EAAE;AAAA,YACV,KAAK,EAAE;AAAA,YACP,aAAa,EAAE,eAAe;AAAA,YAC9B,eAAe,EAAE,iBAAiB;AAAA,YAClC,eAAe,EAAE,iBAAiB,CAAC;AAAA,UACrC,CAAC,IAAI,IAAI;AAAA,QACX;AACA,YAAI,kBAAkB,EAAE,YAAY,UAAU,eAAe,qBAAqB;AAChF,gBAAM,UAAU,KAAK,KAAK,UAAU,KAAK,EAAE,GAAG,CAAC;AAC/C,oBAAU,SAAS,EAAE,WAAW,KAAK,CAAC;AACtC,qBAAW,CAAC,KAAK,MAAM,KAAK,EAAE,WAAW,MAAM,GAAG,mBAAmB,EAAE,QAAQ,GAAG;AAChF,gBAAI,gBAAgB,oBAAqB;AACzC;AACA,kBAAM,WAAW,kBAAkB,QAAQ,GAAG;AAC9C,2BAAe;AAAA,cACb,WAAW,MAAM,cAAc,QAAQ,SAAS,UAAU;AAAA,gBACxD,cAAc;AAAA,gBACd,UAAU;AAAA,gBACV,cAAc;AAAA,cAChB,CAAC,CAAC,EAAE,KAAK,MAAM,QAAW,MAAM;AAAE;AAAA,cAAe,CAAC;AAAA,YACpD;AAAA,UACF;AAAA,QACF;AACA,cAAM,OAAO,UAAU,EAAE,GAAG;AAC5B,cAAM,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AAC5B,mBAAW,KAAK,EAAE,YAAY,CAAC,GAAG;AAChC,gBAAM,YAAY,EAAE,OAAO,IAAI,MAAM,KAAK,EAAE,SAAS,UAAU;AAC/D,gBAAM,KAAK,UAAU,EAAE,IAAI;AAC3B,gBAAM,eAAe,YAAY,IAAI,EAAE,KAAK;AAC5C,gBAAM,QAAQ,KAAK,UAAU,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,UAAU,EAAE,UAAU,UAAU,aAAa,CAAC,IAAI,IAAI;AAC5I;AACA,cAAI,qBAAqB,yBAAyB;AAChD,oCAAwB;AACxB;AAAA,UACF;AACA;AACA,cAAI,EAAE,UAAU;AACd,eAAG;AACH;AACA,gBAAI,CAAC,YAAY,IAAI,EAAE,EAAG,aAAY,IAAI,IAAI,oBAAI,IAAI,CAAC;AACvD,wBAAY,IAAI,EAAE,EAAG,IAAI,IAAI;AAC7B,gBAAI,EAAE,QAAQ;AACZ,kBAAI,CAAC,aAAa,IAAI,EAAE,EAAG,cAAa,IAAI,IAAI,oBAAI,IAAI,CAAC;AACzD,oBAAM,KAAK,aAAa,IAAI,EAAE;AAC9B,iBAAG,IAAI,EAAE,SAAS,GAAG,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC;AAAA,YAC9C;AACA,gBAAI,CAAC,aAAa,YAAY,IAAI,EAAE,MAAM,OAAO,CAAC,YAAY,IAAI,EAAE,IAAI;AACtE,kBAAI,CAAC,UAAU,IAAI,IAAI,EAAG,WAAU,IAAI,MAAM,oBAAI,IAAI,CAAC;AACvD,wBAAU,IAAI,IAAI,EAAG,IAAI,EAAE;AAAA,YAC7B;AACA,gBAAI,gBAAgB,QAAQ,gBAAgB,KAAK;AAC/C;AACA,8BAAgB,IAAI,EAAE,GAAG;AAAA,YAC3B;AAAA,UACF,OAAO;AACL,eAAG;AACH,gBAAI,SAAS;AACb,gBAAI;AAAE,uBAAS,kBAAkB,IAAI,IAAI,EAAE,IAAI,EAAE,QAAQ;AAAA,YAAE,QAAQ;AAAE;AAAA,YAAS;AAC9E,gBAAI,CAAC,UAAU,WAAW,QAAS;AACnC;AACA,kBAAM,IAAI,OAAO,IAAI,MAAM,KAAK,EAAE,OAAO,GAAG,UAAU,GAAG,OAAO,oBAAI,IAAY,EAAE;AAClF,cAAE;AACF,gBAAI,SAAU,GAAE;AAChB,cAAE,MAAM,IAAI,EAAE,GAAG;AACjB,mBAAO,IAAI,QAAQ,CAAC;AAAA,UACtB;AAAA,QACF;AACA,kBAAU,IAAI,EAAE,KAAK,EAAE;AAAA,MACzB;AAAA,IACF;AACA,UAAM,iBAAiB,QAAQ,IAAI,CAAC,SAAS,QAAQ,GAAG,SAAS,SAAS,GAAG,SAAS,WAAW,CAAC,CAAC;AACnG,OAAG,IAAI;AACP,cAAU,IAAI;AACd,gBAAY,IAAI;AAChB,UAAM;AACN,UAAM,QAAQ,IAAI,cAAc;AAEhC,UAAM,QAAQ,oBAAI,IAAoB;AACtC,UAAM,QAAQ,UAAU,IAAI,QAAQ;AACpC,UAAM,IAAI,OAAO,CAAC;AAClB,UAAM,QAAkB,CAAC,KAAK;AAC9B,WAAO,MAAM,SAAS,GAAG;AACvB,YAAM,MAAM,MAAM,MAAM;AACxB,YAAM,IAAI,MAAM,IAAI,GAAG;AACvB,iBAAW,QAAQ,UAAU,IAAI,GAAG,KAAK,CAAC,GAAG;AAC3C,YAAI,CAAC,MAAM,IAAI,IAAI,GAAG;AAAE,gBAAM,IAAI,MAAM,IAAI,CAAC;AAAG,gBAAM,KAAK,IAAI;AAAA,QAAE;AAAA,MACnE;AAAA,IACF;AAEA,UAAM,UAAU,oBAAI,IAA6B;AACjD,eAAW,KAAK,OAAO;AACrB,YAAM,MAAM,UAAU,EAAE,GAAG;AAC3B,YAAM,UAAU,YAAY,IAAI,GAAG,KAAK,oBAAI,IAAI;AAChD,YAAM,KAAK,aAAa,IAAI,GAAG;AAC/B,YAAM,aAAa,KACf,CAAC,GAAG,GAAG,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,IAAI,OAAK,EAAE,CAAC,CAAC,IACvE,CAAC;AACL,YAAM,KAAK,UAAU,IAAI,EAAE,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AACpD,cAAQ,IAAI,EAAE,KAAK;AAAA,QACjB,KAAK,EAAE;AAAA,QACP,SAAS,QAAQ;AAAA,QACjB,eAAe,QAAQ;AAAA,QACvB,kBAAkB,GAAG;AAAA,QACrB,kBAAkB,GAAG;AAAA,QACrB,YAAY,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,IAAK;AAAA,QAC/C,QAAQ,QAAQ,SAAS,KAAK,QAAQ;AAAA,QACtC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,cAAc,OAAO,SAAS,eAAe;AAC5D,QAAI,WAAW,kBAAkB,IAAI,UAAU,OAAO,QAAQ,OAAO;AACrE,UAAM,SAAS,oBAAoB,GAAG;AACtC,UAAM,eAAe;AAAA,MACnB,OAAO,IAAI;AAAA,MACX,UAAU,IAAI;AAAA,MACd,QAAQ,IAAI,mBAAmB,IAC3B,WACA,IAAI,aAAa,KAAK,IAAI,gBAAgB,KAAK,OAAO,kBACpD,YACA;AAAA,MACN,GAAG;AAAA,MACH,YAAY,IAAI;AAAA,MAChB,WAAW,IAAI;AAAA,MACf,YAAY,IAAI;AAAA,MAChB,QAAQ,IAAI;AAAA,MACZ,WAAW,IAAI;AAAA,MACf,cAAc;AAAA,QACZ,WAAW;AAAA,QACX,eAAe;AAAA,QACf,eAAe;AAAA,QACf,SAAS;AAAA,MACX;AAAA,MACA,cAAa,oBAAI,KAAK,GAAE,YAAY;AAAA,IACtC;AACA,eAAW;AAAA,MACT;AAAA,MACA,iBAAiB,aAAa,UAAU;AAAA,MACxC,gBAAgB,aAAa,SAAS;AAAA,MACtC,iBAAiB,aAAa,UAAU;AAAA,MACxC,aAAa,aAAa,MAAM;AAAA,MAChC,gBAAgB,aAAa,SAAS;AAAA,MACtC,yBAAyB,OAAO,iBAAiB;AAAA,MACjD,sBAAsB,OAAO,iBAAiB;AAAA,MAC9C,uBAAuB,OAAO,kBAAkB,QAAQ,IAAI;AAAA,MAC5D,oBAAoB,iBAAiB,IAAI,iBAAiB,SAAS,wBAAwB,eAAe,uBAAuB,MAAM,EAAE;AAAA,MACzI;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,QAAI,OAAO,WAAY,aAAY;AAAA;AAAA,EAAO,mBAAmB,OAAO,UAAU,CAAC;AAE/E,UAAM,eAAe,CAAC,GAAG,QAAQ,OAAO,CAAC;AACzC,UAAM,QAAQ,CAAC,MAAc,KAAK,MAAM,IAAI,EAAE,IAAI;AAClD,UAAM,eAAe,EAAE,MAAM,GAAG,UAAU,GAAG,YAAY,GAAG,YAAY,EAAE;AAC1E,QAAI,aAAa;AACjB,QAAI,cAAc;AAClB,eAAW,KAAK,cAAc;AAC5B,oBAAc,EAAE;AAChB,qBAAe,EAAE,mBAAmB,EAAE;AACtC,UAAI,EAAE,YAAY,EAAG,cAAa;AAAA,eACzB,EAAE,WAAW,EAAG,cAAa;AAAA,eAC7B,EAAE,WAAW,GAAI,cAAa;AAAA,UAClC,cAAa;AAAA,IACpB;AACA,UAAM,kBAAkC,CAAC,GAAG,OAAO,QAAQ,CAAC,EACzD,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,OAAO,EAAE,MAAM,KAAK,EAAE,EAC5F,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACnC,UAAM,aAAyB;AAAA,MAC7B,SAAS;AAAA,QACP,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,OAAO,MAAM;AAAA,UACb,SAAS,aAAa,OAAO,OAAK,EAAE,MAAM,EAAE;AAAA,UAC5C;AAAA,UACA,YAAY,MAAM,SAAS,MAAM,aAAa,MAAM,MAAM,IAAI;AAAA,UAC9D,aAAa,MAAM,SAAS,MAAM,cAAc,MAAM,MAAM,IAAI;AAAA,UAChE;AAAA,UACA,cAAc,CAAC,GAAG,YAAY,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,EAAE,EAC9E,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,SAAS,EAAE,SAAS,kBAAkB,EAAE,kBAAkB,kBAAkB,EAAE,iBAAiB,EAAE;AAAA,QAC9H;AAAA,QACA,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,eAAe,gBAAgB;AAAA,UAC/B,YAAY,gBAAgB,MAAM,GAAG,EAAE;AAAA,QACzC;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,UAAM,WAAW,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAC3D,UAAM,mBAAmB,kBACrB,kBAAkB,KAAK,KAAK,sBAAsB,CAAC,IACnD;AACJ,UAAM,gBAAgB,oBAAI,IAAY;AACtC,UAAM,iBAAiB,CAAC,OAAkC,QAAgB;AACxE,UAAI,aAAa;AACjB,UAAI;AAAE,qBAAa,IAAI,IAAI,GAAG,EAAE;AAAA,MAAK,QAAQ;AAAA,MAAwC;AACrF,aAAO,GAAG,SAAS,EAAE,KAAK,UAAU;AAAA,IACtC;AACA,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,QAAQ,OAAO;AACxB,cAAM,EAAE,cAAc,OAAO,QAAQ,SAAS,UAAU,WAAW,GAAG,YAAY,IAAI;AACtF,cAAM,IAAI,QAAQ,IAAI,KAAK,GAAG;AAC9B,cAAM,MAAM,EAAE,GAAG,aAAa,SAAS,GAAG,WAAW,GAAG,YAAY,GAAG,cAAc,MAAM,QAAQ,GAAG,UAAU,MAAM;AACtH,YAAI,CAAC,SAAS,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,EAAG,OAAM,KAAK,UAAU,OAAO;AAC7E,YAAI,kBAAkB;AACpB,gBAAM,UAAU,eAAe,KAAK,uBAAuB,KAAK,eAAe,KAAK,GAAG;AACvF,wBAAc,IAAI,OAAO;AACzB,gBAAM,aAAa;AAAA,YACjB,gBAAgB,KAAK,yBAAyB;AAAA,YAC9C,kBAAkB,KAAK,oBAAoB;AAAA,YAC3C,aAAa,KAAK,eAAe,KAAK;AAAA,YACtC,aAAa,KAAK,eAAe;AAAA,YACjC,QAAQ,KAAK,qBAAqB,eAAe,aAAa;AAAA,YAC9D,YAAY,KAAK;AAAA,YACjB,aAAa,KAAK,eAAe;AAAA,YACjC,WAAW,KAAK;AAAA,YAChB,aAAa,KAAK,eAAe;AAAA,YACjC,eAAe,KAAK,iBAAiB;AAAA,UACvC;AACA,cAAI,CAAC,iBAAiB,MAAM,KAAK,UAAU,UAAU,IAAI,IAAI,EAAG,OAAM,KAAK,kBAAkB,OAAO;AAAA,QACtG;AAAA,MACF;AAAA,IACF;AACA,QAAI,oBAAoB,iBAAiB,UAAU,MAAM;AACvD,YAAM,WAAW,gBAAgB;AACjC,YAAM,gBAAgB,SAAS;AAC/B,YAAM,UAAU,MAAM;AACpB,YAAI,SAAS,QAAQ,OAAQ,QAAO,CAAC,GAAG,IAAI,IAAI,SAAS,MAAM,CAAC,EAAE,KAAK;AACvE,YAAI,CAAC,SAAS,QAAQ,CAAC,SAAS,GAAI,QAAO,CAAC;AAC5C,cAAM,SAAmB,CAAC;AAC1B,cAAM,SAAS,oBAAI,KAAK,GAAG,SAAS,IAAI,eAAe;AACvD,cAAM,MAAM,oBAAI,KAAK,GAAG,SAAS,EAAE,eAAe;AAClD,eAAO,UAAU,OAAO,OAAO,SAAS,IAAI;AAC1C,iBAAO,KAAK,GAAG,OAAO,eAAe,CAAC,IAAI,OAAO,OAAO,YAAY,IAAI,CAAC,EAAE,SAAS,GAAG,GAAG,CAAC,EAAE;AAC7F,iBAAO,YAAY,OAAO,YAAY,KAAK,SAAS,kBAAkB,EAAE;AAAA,QAC1E;AACA,eAAO;AAAA,MACT,GAAG;AACH,iBAAW,kBAAkB,QAAQ;AACnC,mBAAW,eAAe,eAAe;AACvC,cAAI,cAAc,IAAI,eAAe,gBAAgB,WAAW,CAAC,EAAG;AACpE,cAAI,CAAC,iBAAiB,MAAM,KAAK,UAAU;AAAA,YACzC;AAAA,YACA,kBAAkB;AAAA,YAClB;AAAA,YACA,aAAa;AAAA,YACb,QAAQ;AAAA,YACR,YAAY;AAAA,YACZ,aAAa;AAAA,YACb,WAAW;AAAA,YACX,aAAa;AAAA,YACb,eAAe;AAAA,UACjB,CAAC,IAAI,IAAI,EAAG,OAAM,KAAK,kBAAkB,OAAO;AAAA,QAClD;AAAA,MACF;AAAA,IACF;AACA,UAAM,gBAAgB,SAAS,QAAQ;AACvC,UAAM,wBAAwB,mBAAmB,SAAS,gBAAgB,IAAI,QAAQ,QAAQ;AAC9F,aAAS,IAAI;AACb,sBAAkB,IAAI;AACtB,UAAM,QAAQ,IAAI,CAAC,eAAe,qBAAqB,CAAC;AAExD,kBAAc,KAAK,KAAK,WAAW,GAAG,QAAQ;AAC9C,kBAAc,KAAK,KAAK,oBAAoB,GAAG,KAAK,UAAU,cAAc,MAAM,CAAC,CAAC;AACpF,QAAI,iBAAiB;AACnB,oBAAc,KAAK,KAAK,uBAAuB,GAAG,KAAK,UAAU;AAAA,QAC/D,SAAS,gBAAgB,WAAW,IAAI;AAAA,QACxC,WAAW,gBAAgB,YAAY;AAAA,QACvC,qBAAqB,gBAAgB,uBAAuB;AAAA,QAC5D,eAAe;AAAA,MACjB,GAAG,MAAM,CAAC,CAAC;AAAA,IACb;AACA,kBAAc,KAAK,KAAK,aAAa,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,aAAa,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAClG,kBAAc,KAAK,KAAK,gBAAgB,GAAG,iBAAiB,UAAU,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,KAAK,UAAU,WAAW,SAAS,MAAM,CAAC,CAAC;AAC1F,kBAAc,KAAK,KAAK,uBAAuB,GAAG,KAAK,UAAU,WAAW,iBAAiB,MAAM,CAAC,CAAC;AACrG,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,KAAK,cAAc,GAAG,OAAO,WAAW,KAAK,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACtG,oBAAc,KAAK,KAAK,qBAAqB,GAAG,KAAK,UAAU,OAAO,WAAW,SAAS,MAAM,CAAC,CAAC;AAAA,IACpG;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,gBAAgB,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACxG,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,eAAe,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACvG,QAAI,gBAAgB;AAClB,oBAAc,KAAK,KAAK,8BAA8B,GAAG,KAAK,UAAU;AAAA,QACtE,QAAQ;AAAA,QACR,YAAY,eAAe;AAAA,QAC3B,QAAQ;AAAA,QACR,iBAAiB;AAAA,QACjB,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,cAAc;AAAA,MAChB,GAAG,MAAM,CAAC,CAAC;AAAA,IACb;AAEA,UAAM,gBAAuD;AAAA,MAC3D,EAAE,MAAM,aAAa,MAAM,gBAAgB;AAAA,MAC3C,EAAE,MAAM,sBAAsB,MAAM,mBAAmB;AAAA,MACvD,EAAE,MAAM,kBAAkB,MAAM,uBAAuB;AAAA,MACvD,EAAE,MAAM,gBAAgB,MAAM,uBAAuB;AAAA,MACrD,EAAE,MAAM,eAAe,MAAM,mBAAmB;AAAA,MAChD,EAAE,MAAM,eAAe,MAAM,uBAAuB;AAAA,MACpD,EAAE,MAAM,kBAAkB,MAAM,mBAAmB;AAAA,MACnD,EAAE,MAAM,sBAAsB,MAAM,uBAAuB;AAAA,MAC3D,EAAE,MAAM,kBAAkB,MAAM,gBAAgB;AAAA,MAChD,EAAE,MAAM,sBAAsB,MAAM,mBAAmB;AAAA,MACvD,EAAE,MAAM,yBAAyB,MAAM,mBAAmB;AAAA,IAC5D;AACA,QAAI,iBAAiB;AACnB,oBAAc,KAAK,EAAE,MAAM,yBAAyB,MAAM,mBAAmB,CAAC;AAC9E,oBAAc,KAAK,EAAE,MAAM,wBAAwB,MAAM,uBAAuB,CAAC;AAAA,IACnF;AACA,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,EAAE,MAAM,gBAAgB,MAAM,uBAAuB,CAAC;AACzE,oBAAc,KAAK,EAAE,MAAM,uBAAuB,MAAM,mBAAmB,CAAC;AAAA,IAC9E;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,kBAAkB,MAAM,mBAAmB,CAAC;AAC5F,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,iBAAiB,MAAM,mBAAmB,CAAC;AAC3F,QAAI,eAAgB,eAAc,KAAK,EAAE,MAAM,gCAAgC,MAAM,mBAAmB,CAAC;AAEzG,UAAM,MAAM,IAAI,QAAQ;AACxB,QAAI,IAAI,UAAU,KAAM,OAAM,IAAI,MAAM,wCAAwC;AAChF,UAAM,kBAAkB,sCAAsC;AAAA,MAC5D,SAAS,OAAO,IAAI,MAAM;AAAA,MAC1B,OAAO,IAAI;AAAA;AAAA;AAAA,MAGX,WAAW,oBAAI,KAAK;AAAA,MACpB,SAAS,IAAI;AAAA,IACf,CAAC;AACD,eAAW,KAAK,cAAe,KAAI,QAAQ,KAAK,KAAK,EAAE,IAAI,GAAG,EAAE,IAAI;AACpE,QAAI,gBAAgB,SAAS,GAAG;AAC9B,YAAM,WAAW,WAAW,IAAI,EAAE,EAAE,OAAO,aAAa,EAAE;AAC1D,UAAI,eAA2B,CAAC;AAChC,YAAM,eAAe,YAA+B;AAClD,mBAAS;AACP,gBAAM,OAAO,aAAa,MAAM;AAChC,cAAI,MAAM;AACR,gBAAI,KAAK,aAAc,QAAO;AAC9B;AAAA,UACF;AACA,gBAAM,OAAO,MAAM,SAAS,KAAK;AACjC,cAAI,KAAK,KAAM,OAAM,IAAI,MAAM,6DAA6D;AAC5F,yBAAe,KAAK;AAAA,QACtB;AAAA,MACF;AACA,iBAAW,SAAS,iBAAiB;AACnC,YAAI,kBAAkB,MAAM,MAAM,EAAE,MAAM,MAAM,KAAK,GAAG,cAAY;AAClE,eAAK,aAAa,EAAE,KAAK,UAAQ;AAC/B,gBAAI,KAAK,QAAQ,MAAM,IAAK,OAAM,IAAI,MAAM,+BAA+B,MAAM,GAAG,EAAE;AACtF,kBAAM,eAAe,KAAK,cACtB;AAAA,gBAAmB,KAAK,WAAW;AAAA,qBAAwB,KAAK,oBAAoB,SAAS;AAAA,mBAAsB,KAAK,yBAAyB,aAAa,KAC9J;AACJ,qBAAS,MAAM,SAAS,KAAK;AAAA,cAC3B,KAAK,KAAK,SAAS,KAAK,GAAG;AAAA;AAAA,OAAY,KAAK,GAAG,GAAG,YAAY;AAAA,UAAa,KAAK,MAAM;AAAA;AAAA;AAAA;AAAA,EAAc,KAAK,YAAY;AAAA;AAAA,YACvH,CAAC,CAAC;AAAA,UACJ,CAAC,EAAE,MAAM,WAAS,SAAS,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC,CAAC,CAAC;AAAA,QACvF,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,kBAAkB,eAAe,cAAc;AACjD,YAAM,YAAY,KAAK,KAAK,QAAQ;AACpC,iBAAW,OAAO,YAAY,WAAW,EAAE,WAAW,KAAK,CAAC,GAAe;AACzE,cAAM,OAAO,KAAK,WAAW,GAAG;AAChC,YAAI,SAAS,IAAI,EAAE,OAAO,EAAG,KAAI,QAAQ,MAAM,UAAU,IAAI,MAAM,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE;AAAA,MACtF;AAAA,IACF;AACA,QAAI,IAAI;AACR,WAAO,CAAC,MAAM,eAAe;AAAA,EAC/B,UAAE;AACA,WAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAAA,EAC9C;AACF;","names":[]}