mcp-scraper 0.38.2 → 0.40.1

package/dist/extract-bundle-KUBX6N6Z.js

@@ -1,568 +0,0 @@
-import {
-  createSiteExtractBundleArtifactStream,
-  downloadAsset
-} from "./chunk-QZXKQB7Y.js";
-import {
-  computeIssues,
-  renderImageSection,
-  renderIssueReport,
-  renderLinkReport
-} from "./chunk-345BQXZH.js";
-import {
-  extractJobLimitInfo
-} from "./chunk-YCI2PNCS.js";
-import "./chunk-AZRPG43B.js";
-import "./chunk-44HZLHDV.js";
-import {
-  getDb
-} from "./chunk-N7KUTTCC.js";
-
-// src/api/extract-bundle.ts
-import { createWriteStream, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
-import { basename, join } from "path";
-import { tmpdir } from "os";
-import { createGzip } from "zlib";
-import { once } from "events";
-import { finished } from "stream/promises";
-import { ZipFile } from "yazl";
-import pLimit from "p-limit";
-import { Readable } from "stream";
-var SITE_EXTRACT_PAGE_CHUNK = 25;
-var MAX_IMAGES_PER_PAGE = 20;
-var MAX_IMAGES_PER_SITE = 500;
-var IMAGE_DOWNLOAD_CONCURRENCY = 8;
-var MAX_IMAGE_BYTES = 10 * 1024 * 1024;
-var MAX_SITE_IMAGE_BYTES = 100 * 1024 * 1024;
-var MAX_ANALYZED_LINK_EDGES = 25e4;
-function normalize(u) {
-  return u.split("#")[0].replace(/\/$/, "");
-}
-function registrableDomain(host) {
-  const h = host.replace(/^www\./, "").toLowerCase();
-  const parts = h.split(".");
-  return parts.length <= 2 ? h : parts.slice(-2).join(".");
-}
-function safeImageFilename(url, index) {
-  try {
-    const u = new URL(url);
-    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
-    return base || `image-${index}`;
-  } catch {
-    return `image-${index}`;
-  }
-}
-function slugFactory() {
-  const counts = /* @__PURE__ */ new Map();
-  return (url) => {
-    const base = url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "page";
-    const n = counts.get(base) ?? 0;
-    counts.set(base, n + 1);
-    return n ? `${base}-${n}` : base;
-  };
-}
-async function* pageChunks(jobId) {
-  const db = getDb();
-  let lastRowId = 0;
-  for (; ; ) {
-    const res = await db.execute({
-      sql: `SELECT rowid AS cursor_id, page
-            FROM site_extract_pages
-            WHERE job_id = ? AND rowid > ?
-            ORDER BY rowid
-            LIMIT ${SITE_EXTRACT_PAGE_CHUNK}`,
-      args: [jobId, lastRowId]
-    });
-    if (!res.rows.length) return;
-    lastRowId = Number(res.rows.at(-1)?.cursor_id ?? lastRowId);
-    if (!Number.isSafeInteger(lastRowId) || lastRowId <= 0) throw new Error("site extract page cursor is invalid");
-    yield res.rows.map((r) => JSON.parse(String(r.page)));
-  }
-}
-async function assembleExtractArtifacts(job, extras = {}) {
-  const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`);
-  const downloadImages = job.options.downloadImages === true;
-  mkdirSync(dir, { recursive: true });
-  if (downloadImages) mkdirSync(join(dir, "images"), { recursive: true });
-  try {
-    const metas = [];
-    const pageBodyEntries = [];
-    const bodySlug = slugFactory();
-    const waybackTimeline = job.options.waybackTimeline;
-    const statusByUrl = /* @__PURE__ */ new Map();
-    for await (const chunk of pageChunks(job.id)) {
-      for (const p of chunk) {
-        statusByUrl.set(normalize(p.url), p.status);
-        metas.push({
-          url: p.url,
-          status: p.status,
-          title: p.title,
-          titleLength: p.titleLength,
-          titlePixels: p.titlePixels,
-          metaDescription: p.metaDescription,
-          metaDescLength: p.metaDescLength,
-          h1: p.h1,
-          h1_2: p.h1_2,
-          h2Count: p.h2Count,
-          extractionStatus: p.extractionStatus,
-          indexable: p.indexable,
-          indexabilityReason: p.indexabilityReason,
-          canonicalUrl: p.canonicalUrl ? p.url : null,
-          wordCount: p.wordCount,
-          contentHash: p.contentHash,
-          imagesMissingAlt: p.imagesMissingAlt,
-          schemaTypes: p.schemaTypes.length > 0 ? ["present"] : [],
-          outlinks: []
-        });
-        if (p.bodyMarkdown) {
-          const archiveLines = p.archivedUrl ? `
-Archived URL: ${p.archivedUrl}
-Archive timestamp: ${p.archiveTimestamp ?? "unknown"}
-Requested month: ${p.archiveRequestedMonth ?? "unspecified"}` : "";
-          const content = `# ${p.title ?? p.url}
-
-URL: ${p.url}${archiveLines}
-Status: ${p.status}
-
----
-
-${p.bodyMarkdown}
-`;
-          const monthDir = p.archiveRequestedMonth ? `${p.archiveRequestedMonth}/` : "";
-          pageBodyEntries.push({ url: p.url, name: `pages/${monthDir}${bodySlug(`${p.archiveRequestedMonth ?? ""}-${p.url}`)}.md`, size: Buffer.byteLength(content) });
-        }
-      }
-    }
-    const gz = createGzip();
-    const linksOut = createWriteStream(join(dir, "links.jsonl.gz"));
-    gz.pipe(linksOut);
-    const gzWrite = async (line) => {
-      if (!gz.write(line)) await once(gz, "drain");
-    };
-    const schemaOut = createWriteStream(join(dir, "schema.jsonl"));
-    const schemaWrite = async (line) => {
-      if (!schemaOut.write(line)) await once(schemaOut, "drain");
-    };
-    const failuresOut = createWriteStream(join(dir, "failures.jsonl"));
-    const failureWrite = async (line) => {
-      if (!failuresOut.write(line)) await once(failuresOut, "drain");
-    };
-    const slug = slugFactory();
-    const inboundFrom = /* @__PURE__ */ new Map();
-    const anchorCounts = /* @__PURE__ */ new Map();
-    const adjacency = /* @__PURE__ */ new Map();
-    const outCounts = /* @__PURE__ */ new Map();
-    const domMap = /* @__PURE__ */ new Map();
-    const brokenLinkPages = /* @__PURE__ */ new Set();
-    let internalTotal = 0;
-    let externalTotal = 0;
-    let brokenInternal = 0;
-    let streamedLinkEdges = 0;
-    let analyzedLinkEdges = 0;
-    let linkAnalysisTruncated = false;
-    let siteReg = "";
-    try {
-      siteReg = registrableDomain(new URL(job.startUrl).hostname);
-    } catch {
-      siteReg = "";
-    }
-    const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY);
-    const imageDownloads = [];
-    let imagesQueued = 0;
-    let imagesFailed = 0;
-    let imageBytesDownloaded = 0;
-    const consumeImageBytes = (bytes) => {
-      if (imageBytesDownloaded + bytes > MAX_SITE_IMAGE_BYTES) return false;
-      imageBytesDownloaded += bytes;
-      return true;
-    };
-    for await (const chunk of pageChunks(job.id)) {
-      for (const p of chunk) {
-        if (p.schema?.length) await schemaWrite(JSON.stringify({ url: p.url, schema: p.schema }) + "\n");
-        if (p.extractionStatus === "failed") {
-          await failureWrite(JSON.stringify({
-            url: p.url,
-            status: p.status,
-            via: p.via,
-            failureCode: p.failureCode ?? null,
-            failureReason: p.failureReason ?? null,
-            fetchAttempts: p.fetchAttempts ?? []
-          }) + "\n");
-        }
-        if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {
-          const pageDir = join(dir, "images", slug(p.url));
-          mkdirSync(pageDir, { recursive: true });
-          for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {
-            if (imagesQueued >= MAX_IMAGES_PER_SITE) break;
-            imagesQueued++;
-            const filename = safeImageFilename(imgUrl, idx);
-            imageDownloads.push(
-              imageLimit(() => downloadAsset(imgUrl, pageDir, filename, {
-                expectedType: "image",
-                maxBytes: MAX_IMAGE_BYTES,
-                consumeBytes: consumeImageBytes
-              })).then(() => void 0, () => {
-                imagesFailed++;
-              })
-            );
-          }
-        }
-        const from = normalize(p.url);
-        const oc = { int: 0, ext: 0 };
-        for (const l of p.outlinks ?? []) {
-          const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
-          const to = normalize(l.href);
-          const targetStatus = statusByUrl.get(to) ?? null;
-          await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + "\n");
-          streamedLinkEdges++;
-          if (analyzedLinkEdges >= MAX_ANALYZED_LINK_EDGES) {
-            linkAnalysisTruncated = true;
-            continue;
-          }
-          analyzedLinkEdges++;
-          if (l.internal) {
-            oc.int++;
-            internalTotal++;
-            if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
-            inboundFrom.get(to).add(from);
-            if (l.anchor) {
-              if (!anchorCounts.has(to)) anchorCounts.set(to, /* @__PURE__ */ new Map());
-              const ac = anchorCounts.get(to);
-              ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1);
-            }
-            if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
-              if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
-              adjacency.get(from).add(to);
-            }
-            if (targetStatus != null && targetStatus >= 400) {
-              brokenInternal++;
-              brokenLinkPages.add(p.url);
-            }
-          } else {
-            oc.ext++;
-            let domain = "";
-            try {
-              domain = registrableDomain(new URL(l.href).hostname);
-            } catch {
-              continue;
-            }
-            if (!domain || domain === siteReg) continue;
-            externalTotal++;
-            const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
-            d.links++;
-            if (nofollow) d.nofollow++;
-            d.pages.add(p.url);
-            domMap.set(domain, d);
-          }
-        }
-        outCounts.set(p.url, oc);
-      }
-    }
-    const outputFinished = Promise.all([finished(linksOut), finished(schemaOut), finished(failuresOut)]);
-    gz.end();
-    schemaOut.end();
-    failuresOut.end();
-    await outputFinished;
-    await Promise.all(imageDownloads);
-    const depth = /* @__PURE__ */ new Map();
-    const start = normalize(job.startUrl);
-    depth.set(start, 0);
-    const queue = [start];
-    while (queue.length > 0) {
-      const cur = queue.shift();
-      const d = depth.get(cur);
-      for (const next of adjacency.get(cur) ?? []) {
-        if (!depth.has(next)) {
-          depth.set(next, d + 1);
-          queue.push(next);
-        }
-      }
-    }
-    const metrics = /* @__PURE__ */ new Map();
-    for (const p of metas) {
-      const key = normalize(p.url);
-      const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
-      const ac = anchorCounts.get(key);
-      const topAnchors = ac ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]) : [];
-      const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 };
-      metrics.set(p.url, {
-        url: p.url,
-        inlinks: inbound.size,
-        uniqueInlinks: inbound.size,
-        outlinksInternal: oc.int,
-        outlinksExternal: oc.ext,
-        crawlDepth: depth.has(key) ? depth.get(key) : null,
-        orphan: inbound.size === 0 && key !== start,
-        topAnchors
-      });
-    }
-    const issues = computeIssues(metas, metrics, brokenLinkPages);
-    let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics);
-    const limits = extractJobLimitInfo(job);
-    const crawlSummary = {
-      jobId: job.id,
-      startUrl: job.startUrl,
-      status: job.successfulUrls === 0 ? "failed" : job.failedUrls > 0 || job.remainingUrls > 0 || limits.creditTruncated ? "partial" : "complete",
-      ...limits,
-      discovered: job.totalUrls,
-      attempted: job.attemptedUrls,
-      successful: job.successfulUrls,
-      failed: job.failedUrls,
-      remaining: job.remainingUrls,
-      linkAnalysis: {
-        truncated: linkAnalysisTruncated,
-        analyzedEdges: analyzedLinkEdges,
-        streamedEdges: streamedLinkEdges,
-        edgeCap: MAX_ANALYZED_LINK_EDGES
-      },
-      generatedAt: (/* @__PURE__ */ new Date()).toISOString()
-    };
-    reportMd = [
-      `## Crawl delivery`,
-      `- Discovered: ${crawlSummary.discovered}`,
-      `- Attempted: ${crawlSummary.attempted}`,
-      `- Successful: ${crawlSummary.successful}`,
-      `- Failed: ${crawlSummary.failed}`,
-      `- Remaining: ${crawlSummary.remaining}`,
-      `- Requested page cap: ${limits.requestedMaxPages}`,
-      `- Funded page cap: ${limits.effectiveMaxPages}`,
-      `- Credit-truncated: ${limits.creditTruncated ? "yes" : "no"}`,
-      `- Link analysis: ${analyzedLinkEdges}/${streamedLinkEdges} edges${linkAnalysisTruncated ? ` (capped at ${MAX_ANALYZED_LINK_EDGES})` : ""}`,
-      "",
-      reportMd
-    ].join("\n");
-    if (extras.imageAudit) reportMd += `
-
-${renderImageSection(extras.imageAudit)}`;
-    const metricValues = [...metrics.values()];
-    const round = (n) => Math.round(n * 10) / 10;
-    const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
-    let sumInlinks = 0;
-    let sumOutlinks = 0;
-    for (const m of metricValues) {
-      sumInlinks += m.inlinks;
-      sumOutlinks += m.outlinksInternal + m.outlinksExternal;
-      if (m.inlinks === 0) distribution.zero++;
-      else if (m.inlinks <= 2) distribution.oneToTwo++;
-      else if (m.inlinks <= 10) distribution.threeToTen++;
-      else distribution.elevenPlus++;
-    }
-    const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
-    const linkReport = {
-      summary: {
-        internal: {
-          totalLinks: internalTotal,
-          pages: metas.length,
-          orphans: metricValues.filter((m) => m.orphan).length,
-          brokenInternal,
-          avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,
-          avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,
-          distribution,
-          topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }))
-        },
-        external: {
-          totalLinks: externalTotal,
-          uniqueDomains: externalDomains.length,
-          topDomains: externalDomains.slice(0, 20)
-        }
-      },
-      externalDomains
-    };
-    const pagesOut = createWriteStream(join(dir, "pages.jsonl"));
-    const captureMatrixOut = waybackTimeline ? createWriteStream(join(dir, "capture-matrix.jsonl")) : null;
-    const capturedCells = /* @__PURE__ */ new Set();
-    const captureCellKey = (month, url) => {
-      let normalized = url;
-      try {
-        normalized = new URL(url).href;
-      } catch {
-      }
-      return `${month ?? ""}\0${normalized}`;
-    };
-    for await (const chunk of pageChunks(job.id)) {
-      for (const page of chunk) {
-        const { bodyMarkdown: _body, schema: _schema, outlinks: _outlinks, ...boundedPage } = page;
-        const m = metrics.get(page.url);
-        const row = { ...boundedPage, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
-        if (!pagesOut.write(JSON.stringify(row) + "\n")) await once(pagesOut, "drain");
-        if (captureMatrixOut) {
-          const cellKey = captureCellKey(page.archiveRequestedMonth, page.originalUrl ?? page.url);
-          capturedCells.add(cellKey);
-          const captureRow = {
-            requestedMonth: page.archiveRequestedMonth ?? null,
-            captureTimestamp: page.archiveTimestamp ?? null,
-            originalUrl: page.originalUrl ?? page.url,
-            archivedUrl: page.archivedUrl ?? null,
-            status: page.extractionStatus === "successful" ? "captured" : "failed",
-            httpStatus: page.status,
-            contentHash: page.contentHash || null,
-            wordCount: page.wordCount,
-            failureCode: page.failureCode ?? null,
-            failureReason: page.failureReason ?? null
-          };
-          if (!captureMatrixOut.write(JSON.stringify(captureRow) + "\n")) await once(captureMatrixOut, "drain");
-        }
-      }
-    }
-    if (captureMatrixOut && waybackTimeline?.timeline?.urls) {
-      const timeline = waybackTimeline.timeline;
-      const requestedUrls = timeline.urls;
-      const months = (() => {
-        if (timeline.months?.length) return [...new Set(timeline.months)].sort();
-        if (!timeline.from || !timeline.to) return [];
-        const values = [];
-        const cursor = /* @__PURE__ */ new Date(`${timeline.from}-01T00:00:00Z`);
-        const end = /* @__PURE__ */ new Date(`${timeline.to}-01T00:00:00Z`);
-        while (cursor <= end && values.length < 60) {
-          values.push(`${cursor.getUTCFullYear()}-${String(cursor.getUTCMonth() + 1).padStart(2, "0")}`);
-          cursor.setUTCMonth(cursor.getUTCMonth() + (timeline.intervalMonths ?? 1));
-        }
-        return values;
-      })();
-      for (const requestedMonth of months) {
-        for (const originalUrl of requestedUrls) {
-          if (capturedCells.has(captureCellKey(requestedMonth, originalUrl))) continue;
-          if (!captureMatrixOut.write(JSON.stringify({
-            requestedMonth,
-            captureTimestamp: null,
-            originalUrl,
-            archivedUrl: null,
-            status: "missing",
-            httpStatus: null,
-            contentHash: null,
-            wordCount: 0,
-            failureCode: "wayback_capture_missing",
-            failureReason: "No successful HTML capture was available in the requested month."
-          }) + "\n")) await once(captureMatrixOut, "drain");
-        }
-      }
-    }
-    const pagesFinished = finished(pagesOut);
-    const captureMatrixFinished = captureMatrixOut ? finished(captureMatrixOut) : Promise.resolve();
-    pagesOut.end();
-    captureMatrixOut?.end();
-    await Promise.all([pagesFinished, captureMatrixFinished]);
-    writeFileSync(join(dir, "report.md"), reportMd);
-    writeFileSync(join(dir, "crawl-summary.json"), JSON.stringify(crawlSummary, null, 2));
-    if (waybackTimeline) {
-      writeFileSync(join(dir, "wayback-manifest.json"), JSON.stringify({
-        rootUrl: waybackTimeline.rootUrl ?? job.startUrl,
-        requested: waybackTimeline.timeline ?? null,
-        maxPagesPerSnapshot: waybackTimeline.maxPagesPerSnapshot ?? null,
-        captureMatrix: "capture-matrix.jsonl"
-      }, null, 2));
-    }
-    writeFileSync(join(dir, "issues.json"), JSON.stringify(issues, null, 2));
-    writeFileSync(join(dir, "link-metrics.jsonl"), metricValues.map((m) => JSON.stringify(m)).join("\n"));
-    writeFileSync(join(dir, "link-report.md"), renderLinkReport(linkReport));
-    writeFileSync(join(dir, "links-summary.json"), JSON.stringify(linkReport.summary, null, 2));
-    writeFileSync(join(dir, "external-domains.json"), JSON.stringify(linkReport.externalDomains, null, 2));
-    if (extras.imageAudit) {
-      writeFileSync(join(dir, "images.jsonl"), extras.imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"));
-      writeFileSync(join(dir, "images-summary.json"), JSON.stringify(extras.imageAudit.summary, null, 2));
-    }
-    if (extras.seoAudit) writeFileSync(join(dir, "seo-audit.json"), JSON.stringify(extras.seoAudit, null, 2));
-    if (extras.branding) writeFileSync(join(dir, "branding.json"), JSON.stringify(extras.branding, null, 2));
-    if (downloadImages) {
-      writeFileSync(join(dir, "images-download-summary.json"), JSON.stringify({
-        queued: imagesQueued,
-        downloaded: imagesQueued - imagesFailed,
-        failed: imagesFailed,
-        downloadedBytes: imageBytesDownloaded,
-        perPageCap: MAX_IMAGES_PER_PAGE,
-        perSiteCap: MAX_IMAGES_PER_SITE,
-        perFileByteCap: MAX_IMAGE_BYTES,
-        totalByteCap: MAX_SITE_IMAGE_BYTES
-      }, null, 2));
-    }
-    const artifactFiles = [
-      { name: "report.md", type: "text/markdown" },
-      { name: "crawl-summary.json", type: "application/json" },
-      { name: "failures.jsonl", type: "application/x-ndjson" },
-      { name: "schema.jsonl", type: "application/x-ndjson" },
-      { name: "issues.json", type: "application/json" },
-      { name: "pages.jsonl", type: "application/x-ndjson" },
-      { name: "links.jsonl.gz", type: "application/gzip" },
-      { name: "link-metrics.jsonl", type: "application/x-ndjson" },
-      { name: "link-report.md", type: "text/markdown" },
-      { name: "links-summary.json", type: "application/json" },
-      { name: "external-domains.json", type: "application/json" }
-    ];
-    if (waybackTimeline) {
-      artifactFiles.push({ name: "wayback-manifest.json", type: "application/json" });
-      artifactFiles.push({ name: "capture-matrix.jsonl", type: "application/x-ndjson" });
-    }
-    if (extras.imageAudit) {
-      artifactFiles.push({ name: "images.jsonl", type: "application/x-ndjson" });
-      artifactFiles.push({ name: "images-summary.json", type: "application/json" });
-    }
-    if (extras.seoAudit) artifactFiles.push({ name: "seo-audit.json", type: "application/json" });
-    if (extras.branding) artifactFiles.push({ name: "branding.json", type: "application/json" });
-    if (downloadImages) artifactFiles.push({ name: "images-download-summary.json", type: "application/json" });
-    const zip = new ZipFile();
-    if (job.userId == null) throw new Error("site extract artifact owner is missing");
-    const artifactPromise = createSiteExtractBundleArtifactStream({
-      ownerId: String(job.userId),
-      jobId: job.id,
-      // Artifact retention starts when this bundle is assembled, including a
-      // later support re-finalization, rather than when the crawl was queued.
-      createdAt: /* @__PURE__ */ new Date(),
-      content: zip.outputStream
-    });
-    for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name);
-    if (pageBodyEntries.length > 0) {
-      const iterator = pageChunks(job.id)[Symbol.asyncIterator]();
-      let pendingPages = [];
-      const nextBodyPage = async () => {
-        for (; ; ) {
-          const page = pendingPages.shift();
-          if (page) {
-            if (page.bodyMarkdown) return page;
-            continue;
-          }
-          const next = await iterator.next();
-          if (next.done) throw new Error("page body stream ended before all ZIP entries were produced");
-          pendingPages = next.value;
-        }
-      };
-      for (const entry of pageBodyEntries) {
-        zip.addReadStreamLazy(entry.name, { size: entry.size }, (callback) => {
-          void nextBodyPage().then((page) => {
-            if (page.url !== entry.url) throw new Error(`page body order changed for ${entry.url}`);
-            const archiveLines = page.archivedUrl ? `
-Archived URL: ${page.archivedUrl}
-Archive timestamp: ${page.archiveTimestamp ?? "unknown"}
-Requested month: ${page.archiveRequestedMonth ?? "unspecified"}` : "";
-            callback(null, Readable.from([
-              `# ${page.title ?? page.url}
-
-URL: ${page.url}${archiveLines}
-Status: ${page.status}
-
----
-
-${page.bodyMarkdown}
-`
-            ]));
-          }).catch((error) => callback(error instanceof Error ? error : new Error(String(error))));
-        });
-      }
-    }
-    if (downloadImages && imagesQueued > imagesFailed) {
-      const imagesDir = join(dir, "images");
-      for (const rel of readdirSync(imagesDir, { recursive: true })) {
-        const full = join(imagesDir, rel);
-        if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split("\\").join("/")}`);
-      }
-    }
-    zip.end();
-    return [await artifactPromise];
-  } finally {
-    rmSync(dir, { recursive: true, force: true });
-  }
-}
-export {
-  MAX_ANALYZED_LINK_EDGES,
-  SITE_EXTRACT_PAGE_CHUNK,
-  assembleExtractArtifacts
-};
-//# sourceMappingURL=extract-bundle-KUBX6N6Z.js.map

package/dist/extract-bundle-KUBX6N6Z.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/api/extract-bundle.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'\nimport { basename, join } from 'node:path'\nimport { tmpdir } from 'node:os'\nimport { createGzip } from 'node:zlib'\nimport { once } from 'node:events'\nimport { finished } from 'node:stream/promises'\nimport { ZipFile } from 'yazl'\nimport pLimit from 'p-limit'\nimport { Readable } from 'node:stream'\nimport { getDb } from './db.js'\nimport { computeIssues, renderIssueReport } from './seo-issues.js'\nimport { renderLinkReport, type ExtDomainRow, type LinkReport } from './seo-link-report.js'\nimport type { PageLinkMetrics } from './seo-link-graph.js'\nimport type { SeoAudit } from './seo-audit.js'\nimport type { ExtractJobArtifact, ExtractJobRow } from './site-extract-repository.js'\nimport { extractJobLimitInfo } from './site-extract-repository.js'\nimport type { PageData } from './site-extractor.js'\nimport { renderImageSection, type ImageAudit } from './image-audit.js'\nimport { downloadAsset } from '../lib/media-extractor.js'\nimport { createSiteExtractBundleArtifactStream } from './site-extract-artifacts.js'\n\nexport const SITE_EXTRACT_PAGE_CHUNK = 25\nconst MAX_IMAGES_PER_PAGE = 20\nconst MAX_IMAGES_PER_SITE = 500\nconst IMAGE_DOWNLOAD_CONCURRENCY = 8\nconst MAX_IMAGE_BYTES = 10 * 1024 * 1024\nconst MAX_SITE_IMAGE_BYTES = 100 * 1024 * 1024\nexport const MAX_ANALYZED_LINK_EDGES = 250_000\n\nfunction normalize(u: string): string {\n  return u.split('#')[0].replace(/\\/$/, '')\n}\n\nfunction registrableDomain(host: string): string {\n  const h = host.replace(/^www\\./, '').toLowerCase()\n  const parts = h.split('.')\n  return parts.length <= 2 ? h : parts.slice(-2).join('.')\n}\n\nfunction safeImageFilename(url: string, index: number): string {\n  try {\n    const u = new URL(url)\n    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n    return base || `image-${index}`\n  } catch {\n    return `image-${index}`\n  }\n}\n\nfunction slugFactory() {\n  const counts = new Map<string, number>()\n  return (url: string): string => {\n    const base = url.replace(/^https?:\\/\\//, '').replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80) || 'page'\n    const n = counts.get(base) ?? 0\n    counts.set(base, n + 1)\n    return n ? `${base}-${n}` : base\n  }\n}\n\nasync function* pageChunks(jobId: string): AsyncGenerator<PageData[]> {\n  const db = getDb()\n  let lastRowId = 0\n  for (;;) {\n    const res = await db.execute({\n      sql: `SELECT rowid AS cursor_id, page\n            FROM site_extract_pages\n            WHERE job_id = ? AND rowid > ?\n            ORDER BY rowid\n            LIMIT ${SITE_EXTRACT_PAGE_CHUNK}`,\n      args: [jobId, lastRowId],\n    })\n    if (!res.rows.length) return\n    lastRowId = Number(res.rows.at(-1)?.cursor_id ?? lastRowId)\n    if (!Number.isSafeInteger(lastRowId) || lastRowId <= 0) throw new Error('site extract page cursor is invalid')\n    yield res.rows.map(r => JSON.parse(String((r as unknown as Record<string, unknown>).page)) as PageData)\n  }\n}\n\nexport interface AssembleExtras {\n  branding?: unknown\n  imageAudit?: ImageAudit | null\n  seoAudit?: SeoAudit | null\n}\n\nexport async function assembleExtractArtifacts(job: ExtractJobRow, extras: AssembleExtras = {}): Promise<ExtractJobArtifact[]> {\n  const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`)\n  const downloadImages = job.options.downloadImages === true\n  mkdirSync(dir, { recursive: true })\n  if (downloadImages) mkdirSync(join(dir, 'images'), { recursive: true })\n  try {\n    const metas: PageData[] = []\n    const pageBodyEntries: Array<{ url: string; name: string; size: number }> = []\n    const bodySlug = slugFactory()\n    const waybackTimeline = job.options.waybackTimeline as {\n      rootUrl?: string\n      timeline?: { months?: string[]; from?: string; to?: string; intervalMonths?: number; urls?: string[] }\n      maxPagesPerSnapshot?: number\n    } | undefined\n    const statusByUrl = new Map<string, number | null>()\n    for await (const chunk of pageChunks(job.id)) {\n      for (const p of chunk) {\n        statusByUrl.set(normalize(p.url), p.status)\n        // Keep only fields needed by computeIssues/renderIssueReport. Holding\n        // every bounded PageData field for 10k pages can still consume\n        // hundreds of MiB even though DB reads are streamed in small chunks.\n        metas.push({\n          url: p.url,\n          status: p.status,\n          title: p.title,\n          titleLength: p.titleLength,\n          titlePixels: p.titlePixels,\n          metaDescription: p.metaDescription,\n          metaDescLength: p.metaDescLength,\n          h1: p.h1,\n          h1_2: p.h1_2,\n          h2Count: p.h2Count,\n          extractionStatus: p.extractionStatus,\n          indexable: p.indexable,\n          indexabilityReason: p.indexabilityReason,\n          canonicalUrl: p.canonicalUrl ? p.url : null,\n          wordCount: p.wordCount,\n          contentHash: p.contentHash,\n          imagesMissingAlt: p.imagesMissingAlt,\n          schemaTypes: p.schemaTypes.length > 0 ? ['present'] : [],\n          outlinks: [],\n        } as unknown as PageData)\n        if (p.bodyMarkdown) {\n          const archiveLines = p.archivedUrl\n            ? `\\nArchived URL: ${p.archivedUrl}\\nArchive timestamp: ${p.archiveTimestamp ?? 'unknown'}\\nRequested month: ${p.archiveRequestedMonth ?? 'unspecified'}`\n            : ''\n          const content = `# ${p.title ?? p.url}\\n\\nURL: ${p.url}${archiveLines}\\nStatus: ${p.status}\\n\\n---\\n\\n${p.bodyMarkdown}\\n`\n          const monthDir = p.archiveRequestedMonth ? `${p.archiveRequestedMonth}/` : ''\n          pageBodyEntries.push({ url: p.url, name: `pages/${monthDir}${bodySlug(`${p.archiveRequestedMonth ?? ''}-${p.url}`)}.md`, size: Buffer.byteLength(content) })\n        }\n      }\n    }\n\n    const gz = createGzip()\n    const linksOut = createWriteStream(join(dir, 'links.jsonl.gz'))\n    gz.pipe(linksOut)\n    const gzWrite = async (line: string) => { if (!gz.write(line)) await once(gz, 'drain') }\n    const schemaOut = createWriteStream(join(dir, 'schema.jsonl'))\n    const schemaWrite = async (line: string) => { if (!schemaOut.write(line)) await once(schemaOut, 'drain') }\n    const failuresOut = createWriteStream(join(dir, 'failures.jsonl'))\n    const failureWrite = async (line: string) => { if (!failuresOut.write(line)) await once(failuresOut, 'drain') }\n\n    const slug = slugFactory()\n    const inboundFrom = new Map<string, Set<string>>()\n    const anchorCounts = new Map<string, Map<string, number>>()\n    const adjacency = new Map<string, Set<string>>()\n    const outCounts = new Map<string, { int: number; ext: number }>()\n    const domMap = new Map<string, { links: number; nofollow: number; pages: Set<string> }>()\n    const brokenLinkPages = new Set<string>()\n    let internalTotal = 0\n    let externalTotal = 0\n    let brokenInternal = 0\n    let streamedLinkEdges = 0\n    let analyzedLinkEdges = 0\n    let linkAnalysisTruncated = false\n    let siteReg = ''\n    try { siteReg = registrableDomain(new URL(job.startUrl).hostname) } catch { siteReg = '' }\n\n    const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY)\n    const imageDownloads: Promise<void>[] = []\n    let imagesQueued = 0\n    let imagesFailed = 0\n    let imageBytesDownloaded = 0\n    const consumeImageBytes = (bytes: number) => {\n      if (imageBytesDownloaded + bytes > MAX_SITE_IMAGE_BYTES) return false\n      imageBytesDownloaded += bytes\n      return true\n    }\n\n    for await (const chunk of pageChunks(job.id)) {\n      for (const p of chunk) {\n        if (p.schema?.length) await schemaWrite(JSON.stringify({ url: p.url, schema: p.schema }) + '\\n')\n        if (p.extractionStatus === 'failed') {\n          await failureWrite(JSON.stringify({\n            url: p.url,\n            status: p.status,\n            via: p.via,\n            failureCode: p.failureCode ?? null,\n            failureReason: p.failureReason ?? null,\n            fetchAttempts: p.fetchAttempts ?? [],\n          }) + '\\n')\n        }\n        if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {\n          const pageDir = join(dir, 'images', slug(p.url))\n          mkdirSync(pageDir, { recursive: true })\n          for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {\n            if (imagesQueued >= MAX_IMAGES_PER_SITE) break\n            imagesQueued++\n            const filename = safeImageFilename(imgUrl, idx)\n            imageDownloads.push(\n              imageLimit(() => downloadAsset(imgUrl, pageDir, filename, {\n                expectedType: 'image',\n                maxBytes: MAX_IMAGE_BYTES,\n                consumeBytes: consumeImageBytes,\n              })).then(() => undefined, () => { imagesFailed++ }),\n            )\n          }\n        }\n        const from = normalize(p.url)\n        const oc = { int: 0, ext: 0 }\n        for (const l of p.outlinks ?? []) {\n          const nofollow = (l.rel ?? '').split(/\\s+/).includes('nofollow')\n          const to = normalize(l.href)\n          const targetStatus = statusByUrl.get(to) ?? null\n          await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + '\\n')\n          streamedLinkEdges++\n          if (analyzedLinkEdges >= MAX_ANALYZED_LINK_EDGES) {\n            linkAnalysisTruncated = true\n            continue\n          }\n          analyzedLinkEdges++\n          if (l.internal) {\n            oc.int++\n            internalTotal++\n            if (!inboundFrom.has(to)) inboundFrom.set(to, new Set())\n            inboundFrom.get(to)!.add(from)\n            if (l.anchor) {\n              if (!anchorCounts.has(to)) anchorCounts.set(to, new Map())\n              const ac = anchorCounts.get(to)!\n              ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1)\n            }\n            if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {\n              if (!adjacency.has(from)) adjacency.set(from, new Set())\n              adjacency.get(from)!.add(to)\n            }\n            if (targetStatus != null && targetStatus >= 400) {\n              brokenInternal++\n              brokenLinkPages.add(p.url)\n            }\n          } else {\n            oc.ext++\n            let domain = ''\n            try { domain = registrableDomain(new URL(l.href).hostname) } catch { continue }\n            if (!domain || domain === siteReg) continue\n            externalTotal++\n            const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: new Set<string>() }\n            d.links++\n            if (nofollow) d.nofollow++\n            d.pages.add(p.url)\n            domMap.set(domain, d)\n          }\n        }\n        outCounts.set(p.url, oc)\n      }\n    }\n    const outputFinished = Promise.all([finished(linksOut), finished(schemaOut), finished(failuresOut)])\n    gz.end()\n    schemaOut.end()\n    failuresOut.end()\n    await outputFinished\n    await Promise.all(imageDownloads)\n\n    const depth = new Map<string, number>()\n    const start = normalize(job.startUrl)\n    depth.set(start, 0)\n    const queue: string[] = [start]\n    while (queue.length > 0) {\n      const cur = queue.shift()!\n      const d = depth.get(cur)!\n      for (const next of adjacency.get(cur) ?? []) {\n        if (!depth.has(next)) { depth.set(next, d + 1); queue.push(next) }\n      }\n    }\n\n    const metrics = new Map<string, PageLinkMetrics>()\n    for (const p of metas) {\n      const key = normalize(p.url)\n      const inbound = inboundFrom.get(key) ?? new Set()\n      const ac = anchorCounts.get(key)\n      const topAnchors = ac\n        ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(e => e[0])\n        : []\n      const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 }\n      metrics.set(p.url, {\n        url: p.url,\n        inlinks: inbound.size,\n        uniqueInlinks: inbound.size,\n        outlinksInternal: oc.int,\n        outlinksExternal: oc.ext,\n        crawlDepth: depth.has(key) ? depth.get(key)! : null,\n        orphan: inbound.size === 0 && key !== start,\n        topAnchors,\n      })\n    }\n\n    const issues = computeIssues(metas, metrics, brokenLinkPages)\n    let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics)\n    const limits = extractJobLimitInfo(job)\n    const crawlSummary = {\n      jobId: job.id,\n      startUrl: job.startUrl,\n      status: job.successfulUrls === 0\n        ? 'failed'\n        : job.failedUrls > 0 || job.remainingUrls > 0 || limits.creditTruncated\n          ? 'partial'\n          : 'complete',\n      ...limits,\n      discovered: job.totalUrls,\n      attempted: job.attemptedUrls,\n      successful: job.successfulUrls,\n      failed: job.failedUrls,\n      remaining: job.remainingUrls,\n      linkAnalysis: {\n        truncated: linkAnalysisTruncated,\n        analyzedEdges: analyzedLinkEdges,\n        streamedEdges: streamedLinkEdges,\n        edgeCap: MAX_ANALYZED_LINK_EDGES,\n      },\n      generatedAt: new Date().toISOString(),\n    }\n    reportMd = [\n      `## Crawl delivery`,\n      `- Discovered: ${crawlSummary.discovered}`,\n      `- Attempted: ${crawlSummary.attempted}`,\n      `- Successful: ${crawlSummary.successful}`,\n      `- Failed: ${crawlSummary.failed}`,\n      `- Remaining: ${crawlSummary.remaining}`,\n      `- Requested page cap: ${limits.requestedMaxPages}`,\n      `- Funded page cap: ${limits.effectiveMaxPages}`,\n      `- Credit-truncated: ${limits.creditTruncated ? 'yes' : 'no'}`,\n      `- Link analysis: ${analyzedLinkEdges}/${streamedLinkEdges} edges${linkAnalysisTruncated ? ` (capped at ${MAX_ANALYZED_LINK_EDGES})` : ''}`,\n      '',\n      reportMd,\n    ].join('\\n')\n    if (extras.imageAudit) reportMd += `\\n\\n${renderImageSection(extras.imageAudit)}`\n\n    const metricValues = [...metrics.values()]\n    const round = (n: number) => Math.round(n * 10) / 10\n    const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 }\n    let sumInlinks = 0\n    let sumOutlinks = 0\n    for (const m of metricValues) {\n      sumInlinks += m.inlinks\n      sumOutlinks += m.outlinksInternal + m.outlinksExternal\n      if (m.inlinks === 0) distribution.zero++\n      else if (m.inlinks <= 2) distribution.oneToTwo++\n      else if (m.inlinks <= 10) distribution.threeToTen++\n      else distribution.elevenPlus++\n    }\n    const externalDomains: ExtDomainRow[] = [...domMap.entries()]\n      .map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size }))\n      .sort((a, b) => b.links - a.links)\n    const linkReport: LinkReport = {\n      summary: {\n        internal: {\n          totalLinks: internalTotal,\n          pages: metas.length,\n          orphans: metricValues.filter(m => m.orphan).length,\n          brokenInternal,\n          avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,\n          avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,\n          distribution,\n          topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20)\n            .map(m => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal })),\n        },\n        external: {\n          totalLinks: externalTotal,\n          uniqueDomains: externalDomains.length,\n          topDomains: externalDomains.slice(0, 20),\n        },\n      },\n      externalDomains,\n    }\n\n    const pagesOut = createWriteStream(join(dir, 'pages.jsonl'))\n    const captureMatrixOut = waybackTimeline\n      ? createWriteStream(join(dir, 'capture-matrix.jsonl'))\n      : null\n    const capturedCells = new Set<string>()\n    const captureCellKey = (month: string | null | undefined, url: string) => {\n      let normalized = url\n      try { normalized = new URL(url).href } catch { /** Keep the bounded stored value. */ }\n      return `${month ?? ''}\\0${normalized}`\n    }\n    for await (const chunk of pageChunks(job.id)) {\n      for (const page of chunk) {\n        const { bodyMarkdown: _body, schema: _schema, outlinks: _outlinks, ...boundedPage } = page\n        const m = metrics.get(page.url)\n        const row = { ...boundedPage, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false }\n        if (!pagesOut.write(JSON.stringify(row) + '\\n')) await once(pagesOut, 'drain')\n        if (captureMatrixOut) {\n          const cellKey = captureCellKey(page.archiveRequestedMonth, page.originalUrl ?? page.url)\n          capturedCells.add(cellKey)\n          const captureRow = {\n            requestedMonth: page.archiveRequestedMonth ?? null,\n            captureTimestamp: page.archiveTimestamp ?? null,\n            originalUrl: page.originalUrl ?? page.url,\n            archivedUrl: page.archivedUrl ?? null,\n            status: page.extractionStatus === 'successful' ? 'captured' : 'failed',\n            httpStatus: page.status,\n            contentHash: page.contentHash || null,\n            wordCount: page.wordCount,\n            failureCode: page.failureCode ?? null,\n            failureReason: page.failureReason ?? null,\n          }\n          if (!captureMatrixOut.write(JSON.stringify(captureRow) + '\\n')) await once(captureMatrixOut, 'drain')\n        }\n      }\n    }\n    if (captureMatrixOut && waybackTimeline?.timeline?.urls) {\n      const timeline = waybackTimeline.timeline\n      const requestedUrls = timeline.urls!\n      const months = (() => {\n        if (timeline.months?.length) return [...new Set(timeline.months)].sort()\n        if (!timeline.from || !timeline.to) return []\n        const values: string[] = []\n        const cursor = new Date(`${timeline.from}-01T00:00:00Z`)\n        const end = new Date(`${timeline.to}-01T00:00:00Z`)\n        while (cursor <= end && values.length < 60) {\n          values.push(`${cursor.getUTCFullYear()}-${String(cursor.getUTCMonth() + 1).padStart(2, '0')}`)\n          cursor.setUTCMonth(cursor.getUTCMonth() + (timeline.intervalMonths ?? 1))\n        }\n        return values\n      })()\n      for (const requestedMonth of months) {\n        for (const originalUrl of requestedUrls) {\n          if (capturedCells.has(captureCellKey(requestedMonth, originalUrl))) continue\n          if (!captureMatrixOut.write(JSON.stringify({\n            requestedMonth,\n            captureTimestamp: null,\n            originalUrl,\n            archivedUrl: null,\n            status: 'missing',\n            httpStatus: null,\n            contentHash: null,\n            wordCount: 0,\n            failureCode: 'wayback_capture_missing',\n            failureReason: 'No successful HTML capture was available in the requested month.',\n          }) + '\\n')) await once(captureMatrixOut, 'drain')\n        }\n      }\n    }\n    const pagesFinished = finished(pagesOut)\n    const captureMatrixFinished = captureMatrixOut ? finished(captureMatrixOut) : Promise.resolve()\n    pagesOut.end()\n    captureMatrixOut?.end()\n    await Promise.all([pagesFinished, captureMatrixFinished])\n\n    writeFileSync(join(dir, 'report.md'), reportMd)\n    writeFileSync(join(dir, 'crawl-summary.json'), JSON.stringify(crawlSummary, null, 2))\n    if (waybackTimeline) {\n      writeFileSync(join(dir, 'wayback-manifest.json'), JSON.stringify({\n        rootUrl: waybackTimeline.rootUrl ?? job.startUrl,\n        requested: waybackTimeline.timeline ?? null,\n        maxPagesPerSnapshot: waybackTimeline.maxPagesPerSnapshot ?? null,\n        captureMatrix: 'capture-matrix.jsonl',\n      }, null, 2))\n    }\n    writeFileSync(join(dir, 'issues.json'), JSON.stringify(issues, null, 2))\n    writeFileSync(join(dir, 'link-metrics.jsonl'), metricValues.map(m => JSON.stringify(m)).join('\\n'))\n    writeFileSync(join(dir, 'link-report.md'), renderLinkReport(linkReport))\n    writeFileSync(join(dir, 'links-summary.json'), JSON.stringify(linkReport.summary, null, 2))\n    writeFileSync(join(dir, 'external-domains.json'), JSON.stringify(linkReport.externalDomains, null, 2))\n    if (extras.imageAudit) {\n      writeFileSync(join(dir, 'images.jsonl'), extras.imageAudit.rows.map(r => JSON.stringify(r)).join('\\n'))\n      writeFileSync(join(dir, 'images-summary.json'), JSON.stringify(extras.imageAudit.summary, null, 2))\n    }\n    if (extras.seoAudit) writeFileSync(join(dir, 'seo-audit.json'), JSON.stringify(extras.seoAudit, null, 2))\n    if (extras.branding) writeFileSync(join(dir, 'branding.json'), JSON.stringify(extras.branding, null, 2))\n    if (downloadImages) {\n      writeFileSync(join(dir, 'images-download-summary.json'), JSON.stringify({\n        queued: imagesQueued,\n        downloaded: imagesQueued - imagesFailed,\n        failed: imagesFailed,\n        downloadedBytes: imageBytesDownloaded,\n        perPageCap: MAX_IMAGES_PER_PAGE,\n        perSiteCap: MAX_IMAGES_PER_SITE,\n        perFileByteCap: MAX_IMAGE_BYTES,\n        totalByteCap: MAX_SITE_IMAGE_BYTES,\n      }, null, 2))\n    }\n\n    const artifactFiles: Array<{ name: string; type: string }> = [\n      { name: 'report.md', type: 'text/markdown' },\n      { name: 'crawl-summary.json', type: 'application/json' },\n      { name: 'failures.jsonl', type: 'application/x-ndjson' },\n      { name: 'schema.jsonl', type: 'application/x-ndjson' },\n      { name: 'issues.json', type: 'application/json' },\n      { name: 'pages.jsonl', type: 'application/x-ndjson' },\n      { name: 'links.jsonl.gz', type: 'application/gzip' },\n      { name: 'link-metrics.jsonl', type: 'application/x-ndjson' },\n      { name: 'link-report.md', type: 'text/markdown' },\n      { name: 'links-summary.json', type: 'application/json' },\n      { name: 'external-domains.json', type: 'application/json' },\n    ]\n    if (waybackTimeline) {\n      artifactFiles.push({ name: 'wayback-manifest.json', type: 'application/json' })\n      artifactFiles.push({ name: 'capture-matrix.jsonl', type: 'application/x-ndjson' })\n    }\n    if (extras.imageAudit) {\n      artifactFiles.push({ name: 'images.jsonl', type: 'application/x-ndjson' })\n      artifactFiles.push({ name: 'images-summary.json', type: 'application/json' })\n    }\n    if (extras.seoAudit) artifactFiles.push({ name: 'seo-audit.json', type: 'application/json' })\n    if (extras.branding) artifactFiles.push({ name: 'branding.json', type: 'application/json' })\n    if (downloadImages) artifactFiles.push({ name: 'images-download-summary.json', type: 'application/json' })\n\n    const zip = new ZipFile()\n    if (job.userId == null) throw new Error('site extract artifact owner is missing')\n    const artifactPromise = createSiteExtractBundleArtifactStream({\n      ownerId: String(job.userId),\n      jobId: job.id,\n      // Artifact retention starts when this bundle is assembled, including a\n      // later support re-finalization, rather than when the crawl was queued.\n      createdAt: new Date(),\n      content: zip.outputStream,\n    })\n    for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name)\n    if (pageBodyEntries.length > 0) {\n      const iterator = pageChunks(job.id)[Symbol.asyncIterator]()\n      let pendingPages: PageData[] = []\n      const nextBodyPage = async (): Promise<PageData> => {\n        for (;;) {\n          const page = pendingPages.shift()\n          if (page) {\n            if (page.bodyMarkdown) return page\n            continue\n          }\n          const next = await iterator.next()\n          if (next.done) throw new Error('page body stream ended before all ZIP entries were produced')\n          pendingPages = next.value\n        }\n      }\n      for (const entry of pageBodyEntries) {\n        zip.addReadStreamLazy(entry.name, { size: entry.size }, callback => {\n          void nextBodyPage().then(page => {\n            if (page.url !== entry.url) throw new Error(`page body order changed for ${entry.url}`)\n            const archiveLines = page.archivedUrl\n              ? `\\nArchived URL: ${page.archivedUrl}\\nArchive timestamp: ${page.archiveTimestamp ?? 'unknown'}\\nRequested month: ${page.archiveRequestedMonth ?? 'unspecified'}`\n              : ''\n            callback(null, Readable.from([\n              `# ${page.title ?? page.url}\\n\\nURL: ${page.url}${archiveLines}\\nStatus: ${page.status}\\n\\n---\\n\\n${page.bodyMarkdown}\\n`,\n            ]))\n          }).catch(error => callback(error instanceof Error ? error : new Error(String(error))))\n        })\n      }\n    }\n    if (downloadImages && imagesQueued > imagesFailed) {\n      const imagesDir = join(dir, 'images')\n      for (const rel of readdirSync(imagesDir, { recursive: true }) as string[]) {\n        const full = join(imagesDir, rel)\n        if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split('\\\\').join('/')}`)\n      }\n    }\n    zip.end()\n    return [await artifactPromise]\n  } finally {\n    rmSync(dir, { recursive: true, force: true })\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,SAAS,mBAAmB,WAAW,aAAa,QAAQ,UAAU,qBAAqB;AAC3F,SAAS,UAAU,YAAY;AAC/B,SAAS,cAAc;AACvB,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AACrB,SAAS,gBAAgB;AACzB,SAAS,eAAe;AACxB,OAAO,YAAY;AACnB,SAAS,gBAAgB;AAalB,IAAM,0BAA0B;AACvC,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAC5B,IAAM,6BAA6B;AACnC,IAAM,kBAAkB,KAAK,OAAO;AACpC,IAAM,uBAAuB,MAAM,OAAO;AACnC,IAAM,0BAA0B;AAEvC,SAAS,UAAU,GAAmB;AACpC,SAAO,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,OAAO,EAAE;AAC1C;AAEA,SAAS,kBAAkB,MAAsB;AAC/C,QAAM,IAAI,KAAK,QAAQ,UAAU,EAAE,EAAE,YAAY;AACjD,QAAM,QAAQ,EAAE,MAAM,GAAG;AACzB,SAAO,MAAM,UAAU,IAAI,IAAI,MAAM,MAAM,EAAE,EAAE,KAAK,GAAG;AACzD;AAEA,SAAS,kBAAkB,KAAa,OAAuB;AAC7D,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEA,SAAS,cAAc;AACrB,QAAM,SAAS,oBAAI,IAAoB;AACvC,SAAO,CAAC,QAAwB;AAC9B,UAAM,OAAO,IAAI,QAAQ,gBAAgB,EAAE,EAAE,QAAQ,kBAAkB,GAAG,EAAE,QAAQ,YAAY,EAAE,EAAE,MAAM,GAAG,EAAE,KAAK;AACpH,UAAM,IAAI,OAAO,IAAI,IAAI,KAAK;AAC9B,WAAO,IAAI,MAAM,IAAI,CAAC;AACtB,WAAO,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK;AAAA,EAC9B;AACF;AAEA,gBAAgB,WAAW,OAA2C;AACpE,QAAM,KAAK,MAAM;AACjB,MAAI,YAAY;AAChB,aAAS;AACP,UAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,MAC3B,KAAK;AAAA;AAAA;AAAA;AAAA,oBAIS,uBAAuB;AAAA,MACrC,MAAM,CAAC,OAAO,SAAS;AAAA,IACzB,CAAC;AACD,QAAI,CAAC,IAAI,KAAK,OAAQ;AACtB,gBAAY,OAAO,IAAI,KAAK,GAAG,EAAE,GAAG,aAAa,SAAS;AAC1D,QAAI,CAAC,OAAO,cAAc,SAAS,KAAK,aAAa,EAAG,OAAM,IAAI,MAAM,qCAAqC;AAC7G,UAAM,IAAI,KAAK,IAAI,OAAK,KAAK,MAAM,OAAQ,EAAyC,IAAI,CAAC,CAAa;AAAA,EACxG;AACF;AAQA,eAAsB,yBAAyB,KAAoB,SAAyB,CAAC,GAAkC;AAC7H,QAAM,MAAM,KAAK,OAAO,GAAG,kBAAkB,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AACnE,QAAM,iBAAiB,IAAI,QAAQ,mBAAmB;AACtD,YAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAClC,MAAI,eAAgB,WAAU,KAAK,KAAK,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;AACtE,MAAI;AACF,UAAM,QAAoB,CAAC;AAC3B,UAAM,kBAAsE,CAAC;AAC7E,UAAM,WAAW,YAAY;AAC7B,UAAM,kBAAkB,IAAI,QAAQ;AAKpC,UAAM,cAAc,oBAAI,IAA2B;AACnD,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,oBAAY,IAAI,UAAU,EAAE,GAAG,GAAG,EAAE,MAAM;AAI1C,cAAM,KAAK;AAAA,UACT,KAAK,EAAE;AAAA,UACP,QAAQ,EAAE;AAAA,UACV,OAAO,EAAE;AAAA,UACT,aAAa,EAAE;AAAA,UACf,aAAa,EAAE;AAAA,UACf,iBAAiB,EAAE;AAAA,UACnB,gBAAgB,EAAE;AAAA,UAClB,IAAI,EAAE;AAAA,UACN,MAAM,EAAE;AAAA,UACR,SAAS,EAAE;AAAA,UACX,kBAAkB,EAAE;AAAA,UACpB,WAAW,EAAE;AAAA,UACb,oBAAoB,EAAE;AAAA,UACtB,cAAc,EAAE,eAAe,EAAE,MAAM;AAAA,UACvC,WAAW,EAAE;AAAA,UACb,aAAa,EAAE;AAAA,UACf,kBAAkB,EAAE;AAAA,UACpB,aAAa,EAAE,YAAY,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC;AAAA,UACvD,UAAU,CAAC;AAAA,QACb,CAAwB;AACxB,YAAI,EAAE,cAAc;AAClB,gBAAM,eAAe,EAAE,cACnB;AAAA,gBAAmB,EAAE,WAAW;AAAA,qBAAwB,EAAE,oBAAoB,SAAS;AAAA,mBAAsB,EAAE,yBAAyB,aAAa,KACrJ;AACJ,gBAAM,UAAU,KAAK,EAAE,SAAS,EAAE,GAAG;AAAA;AAAA,OAAY,EAAE,GAAG,GAAG,YAAY;AAAA,UAAa,EAAE,MAAM;AAAA;AAAA;AAAA;AAAA,EAAc,EAAE,YAAY;AAAA;AACtH,gBAAM,WAAW,EAAE,wBAAwB,GAAG,EAAE,qBAAqB,MAAM;AAC3E,0BAAgB,KAAK,EAAE,KAAK,EAAE,KAAK,MAAM,SAAS,QAAQ,GAAG,SAAS,GAAG,EAAE,yBAAyB,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,MAAM,OAAO,WAAW,OAAO,EAAE,CAAC;AAAA,QAC7J;AAAA,MACF;AAAA,IACF;AAEA,UAAM,KAAK,WAAW;AACtB,UAAM,WAAW,kBAAkB,KAAK,KAAK,gBAAgB,CAAC;AAC9D,OAAG,KAAK,QAAQ;AAChB,UAAM,UAAU,OAAO,SAAiB;AAAE,UAAI,CAAC,GAAG,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI,OAAO;AAAA,IAAE;AACvF,UAAM,YAAY,kBAAkB,KAAK,KAAK,cAAc,CAAC;AAC7D,UAAM,cAAc,OAAO,SAAiB;AAAE,UAAI,CAAC,UAAU,MAAM,IAAI,EAAG,OAAM,KAAK,WAAW,OAAO;AAAA,IAAE;AACzG,UAAM,cAAc,kBAAkB,KAAK,KAAK,gBAAgB,CAAC;AACjE,UAAM,eAAe,OAAO,SAAiB;AAAE,UAAI,CAAC,YAAY,MAAM,IAAI,EAAG,OAAM,KAAK,aAAa,OAAO;AAAA,IAAE;AAE9G,UAAM,OAAO,YAAY;AACzB,UAAM,cAAc,oBAAI,IAAyB;AACjD,UAAM,eAAe,oBAAI,IAAiC;AAC1D,UAAM,YAAY,oBAAI,IAAyB;AAC/C,UAAM,YAAY,oBAAI,IAA0C;AAChE,UAAM,SAAS,oBAAI,IAAqE;AACxF,UAAM,kBAAkB,oBAAI,IAAY;AACxC,QAAI,gBAAgB;AACpB,QAAI,gBAAgB;AACpB,QAAI,iBAAiB;AACrB,QAAI,oBAAoB;AACxB,QAAI,oBAAoB;AACxB,QAAI,wBAAwB;AAC5B,QAAI,UAAU;AACd,QAAI;AAAE,gBAAU,kBAAkB,IAAI,IAAI,IAAI,QAAQ,EAAE,QAAQ;AAAA,IAAE,QAAQ;AAAE,gBAAU;AAAA,IAAG;AAEzF,UAAM,aAAa,OAAO,0BAA0B;AACpD,UAAM,iBAAkC,CAAC;AACzC,QAAI,eAAe;AACnB,QAAI,eAAe;AACnB,QAAI,uBAAuB;AAC3B,UAAM,oBAAoB,CAAC,UAAkB;AAC3C,UAAI,uBAAuB,QAAQ,qBAAsB,QAAO;AAChE,8BAAwB;AACxB,aAAO;AAAA,IACT;AAEA,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,YAAI,EAAE,QAAQ,OAAQ,OAAM,YAAY,KAAK,UAAU,EAAE,KAAK,EAAE,KAAK,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI;AAC/F,YAAI,EAAE,qBAAqB,UAAU;AACnC,gBAAM,aAAa,KAAK,UAAU;AAAA,YAChC,KAAK,EAAE;AAAA,YACP,QAAQ,EAAE;AAAA,YACV,KAAK,EAAE;AAAA,YACP,aAAa,EAAE,eAAe;AAAA,YAC9B,eAAe,EAAE,iBAAiB;AAAA,YAClC,eAAe,EAAE,iBAAiB,CAAC;AAAA,UACrC,CAAC,IAAI,IAAI;AAAA,QACX;AACA,YAAI,kBAAkB,EAAE,YAAY,UAAU,eAAe,qBAAqB;AAChF,gBAAM,UAAU,KAAK,KAAK,UAAU,KAAK,EAAE,GAAG,CAAC;AAC/C,oBAAU,SAAS,EAAE,WAAW,KAAK,CAAC;AACtC,qBAAW,CAAC,KAAK,MAAM,KAAK,EAAE,WAAW,MAAM,GAAG,mBAAmB,EAAE,QAAQ,GAAG;AAChF,gBAAI,gBAAgB,oBAAqB;AACzC;AACA,kBAAM,WAAW,kBAAkB,QAAQ,GAAG;AAC9C,2BAAe;AAAA,cACb,WAAW,MAAM,cAAc,QAAQ,SAAS,UAAU;AAAA,gBACxD,cAAc;AAAA,gBACd,UAAU;AAAA,gBACV,cAAc;AAAA,cAChB,CAAC,CAAC,EAAE,KAAK,MAAM,QAAW,MAAM;AAAE;AAAA,cAAe,CAAC;AAAA,YACpD;AAAA,UACF;AAAA,QACF;AACA,cAAM,OAAO,UAAU,EAAE,GAAG;AAC5B,cAAM,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AAC5B,mBAAW,KAAK,EAAE,YAAY,CAAC,GAAG;AAChC,gBAAM,YAAY,EAAE,OAAO,IAAI,MAAM,KAAK,EAAE,SAAS,UAAU;AAC/D,gBAAM,KAAK,UAAU,EAAE,IAAI;AAC3B,gBAAM,eAAe,YAAY,IAAI,EAAE,KAAK;AAC5C,gBAAM,QAAQ,KAAK,UAAU,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,UAAU,EAAE,UAAU,UAAU,aAAa,CAAC,IAAI,IAAI;AAC5I;AACA,cAAI,qBAAqB,yBAAyB;AAChD,oCAAwB;AACxB;AAAA,UACF;AACA;AACA,cAAI,EAAE,UAAU;AACd,eAAG;AACH;AACA,gBAAI,CAAC,YAAY,IAAI,EAAE,EAAG,aAAY,IAAI,IAAI,oBAAI,IAAI,CAAC;AACvD,wBAAY,IAAI,EAAE,EAAG,IAAI,IAAI;AAC7B,gBAAI,EAAE,QAAQ;AACZ,kBAAI,CAAC,aAAa,IAAI,EAAE,EAAG,cAAa,IAAI,IAAI,oBAAI,IAAI,CAAC;AACzD,oBAAM,KAAK,aAAa,IAAI,EAAE;AAC9B,iBAAG,IAAI,EAAE,SAAS,GAAG,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC;AAAA,YAC9C;AACA,gBAAI,CAAC,aAAa,YAAY,IAAI,EAAE,MAAM,OAAO,CAAC,YAAY,IAAI,EAAE,IAAI;AACtE,kBAAI,CAAC,UAAU,IAAI,IAAI,EAAG,WAAU,IAAI,MAAM,oBAAI,IAAI,CAAC;AACvD,wBAAU,IAAI,IAAI,EAAG,IAAI,EAAE;AAAA,YAC7B;AACA,gBAAI,gBAAgB,QAAQ,gBAAgB,KAAK;AAC/C;AACA,8BAAgB,IAAI,EAAE,GAAG;AAAA,YAC3B;AAAA,UACF,OAAO;AACL,eAAG;AACH,gBAAI,SAAS;AACb,gBAAI;AAAE,uBAAS,kBAAkB,IAAI,IAAI,EAAE,IAAI,EAAE,QAAQ;AAAA,YAAE,QAAQ;AAAE;AAAA,YAAS;AAC9E,gBAAI,CAAC,UAAU,WAAW,QAAS;AACnC;AACA,kBAAM,IAAI,OAAO,IAAI,MAAM,KAAK,EAAE,OAAO,GAAG,UAAU,GAAG,OAAO,oBAAI,IAAY,EAAE;AAClF,cAAE;AACF,gBAAI,SAAU,GAAE;AAChB,cAAE,MAAM,IAAI,EAAE,GAAG;AACjB,mBAAO,IAAI,QAAQ,CAAC;AAAA,UACtB;AAAA,QACF;AACA,kBAAU,IAAI,EAAE,KAAK,EAAE;AAAA,MACzB;AAAA,IACF;AACA,UAAM,iBAAiB,QAAQ,IAAI,CAAC,SAAS,QAAQ,GAAG,SAAS,SAAS,GAAG,SAAS,WAAW,CAAC,CAAC;AACnG,OAAG,IAAI;AACP,cAAU,IAAI;AACd,gBAAY,IAAI;AAChB,UAAM;AACN,UAAM,QAAQ,IAAI,cAAc;AAEhC,UAAM,QAAQ,oBAAI,IAAoB;AACtC,UAAM,QAAQ,UAAU,IAAI,QAAQ;AACpC,UAAM,IAAI,OAAO,CAAC;AAClB,UAAM,QAAkB,CAAC,KAAK;AAC9B,WAAO,MAAM,SAAS,GAAG;AACvB,YAAM,MAAM,MAAM,MAAM;AACxB,YAAM,IAAI,MAAM,IAAI,GAAG;AACvB,iBAAW,QAAQ,UAAU,IAAI,GAAG,KAAK,CAAC,GAAG;AAC3C,YAAI,CAAC,MAAM,IAAI,IAAI,GAAG;AAAE,gBAAM,IAAI,MAAM,IAAI,CAAC;AAAG,gBAAM,KAAK,IAAI;AAAA,QAAE;AAAA,MACnE;AAAA,IACF;AAEA,UAAM,UAAU,oBAAI,IAA6B;AACjD,eAAW,KAAK,OAAO;AACrB,YAAM,MAAM,UAAU,EAAE,GAAG;AAC3B,YAAM,UAAU,YAAY,IAAI,GAAG,KAAK,oBAAI,IAAI;AAChD,YAAM,KAAK,aAAa,IAAI,GAAG;AAC/B,YAAM,aAAa,KACf,CAAC,GAAG,GAAG,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,IAAI,OAAK,EAAE,CAAC,CAAC,IACvE,CAAC;AACL,YAAM,KAAK,UAAU,IAAI,EAAE,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AACpD,cAAQ,IAAI,EAAE,KAAK;AAAA,QACjB,KAAK,EAAE;AAAA,QACP,SAAS,QAAQ;AAAA,QACjB,eAAe,QAAQ;AAAA,QACvB,kBAAkB,GAAG;AAAA,QACrB,kBAAkB,GAAG;AAAA,QACrB,YAAY,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,IAAK;AAAA,QAC/C,QAAQ,QAAQ,SAAS,KAAK,QAAQ;AAAA,QACtC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,cAAc,OAAO,SAAS,eAAe;AAC5D,QAAI,WAAW,kBAAkB,IAAI,UAAU,OAAO,QAAQ,OAAO;AACrE,UAAM,SAAS,oBAAoB,GAAG;AACtC,UAAM,eAAe;AAAA,MACnB,OAAO,IAAI;AAAA,MACX,UAAU,IAAI;AAAA,MACd,QAAQ,IAAI,mBAAmB,IAC3B,WACA,IAAI,aAAa,KAAK,IAAI,gBAAgB,KAAK,OAAO,kBACpD,YACA;AAAA,MACN,GAAG;AAAA,MACH,YAAY,IAAI;AAAA,MAChB,WAAW,IAAI;AAAA,MACf,YAAY,IAAI;AAAA,MAChB,QAAQ,IAAI;AAAA,MACZ,WAAW,IAAI;AAAA,MACf,cAAc;AAAA,QACZ,WAAW;AAAA,QACX,eAAe;AAAA,QACf,eAAe;AAAA,QACf,SAAS;AAAA,MACX;AAAA,MACA,cAAa,oBAAI,KAAK,GAAE,YAAY;AAAA,IACtC;AACA,eAAW;AAAA,MACT;AAAA,MACA,iBAAiB,aAAa,UAAU;AAAA,MACxC,gBAAgB,aAAa,SAAS;AAAA,MACtC,iBAAiB,aAAa,UAAU;AAAA,MACxC,aAAa,aAAa,MAAM;AAAA,MAChC,gBAAgB,aAAa,SAAS;AAAA,MACtC,yBAAyB,OAAO,iBAAiB;AAAA,MACjD,sBAAsB,OAAO,iBAAiB;AAAA,MAC9C,uBAAuB,OAAO,kBAAkB,QAAQ,IAAI;AAAA,MAC5D,oBAAoB,iBAAiB,IAAI,iBAAiB,SAAS,wBAAwB,eAAe,uBAAuB,MAAM,EAAE;AAAA,MACzI;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,QAAI,OAAO,WAAY,aAAY;AAAA;AAAA,EAAO,mBAAmB,OAAO,UAAU,CAAC;AAE/E,UAAM,eAAe,CAAC,GAAG,QAAQ,OAAO,CAAC;AACzC,UAAM,QAAQ,CAAC,MAAc,KAAK,MAAM,IAAI,EAAE,IAAI;AAClD,UAAM,eAAe,EAAE,MAAM,GAAG,UAAU,GAAG,YAAY,GAAG,YAAY,EAAE;AAC1E,QAAI,aAAa;AACjB,QAAI,cAAc;AAClB,eAAW,KAAK,cAAc;AAC5B,oBAAc,EAAE;AAChB,qBAAe,EAAE,mBAAmB,EAAE;AACtC,UAAI,EAAE,YAAY,EAAG,cAAa;AAAA,eACzB,EAAE,WAAW,EAAG,cAAa;AAAA,eAC7B,EAAE,WAAW,GAAI,cAAa;AAAA,UAClC,cAAa;AAAA,IACpB;AACA,UAAM,kBAAkC,CAAC,GAAG,OAAO,QAAQ,CAAC,EACzD,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,OAAO,EAAE,MAAM,KAAK,EAAE,EAC5F,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACnC,UAAM,aAAyB;AAAA,MAC7B,SAAS;AAAA,QACP,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,OAAO,MAAM;AAAA,UACb,SAAS,aAAa,OAAO,OAAK,EAAE,MAAM,EAAE;AAAA,UAC5C;AAAA,UACA,YAAY,MAAM,SAAS,MAAM,aAAa,MAAM,MAAM,IAAI;AAAA,UAC9D,aAAa,MAAM,SAAS,MAAM,cAAc,MAAM,MAAM,IAAI;AAAA,UAChE;AAAA,UACA,cAAc,CAAC,GAAG,YAAY,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,EAAE,EAC9E,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,SAAS,EAAE,SAAS,kBAAkB,EAAE,kBAAkB,kBAAkB,EAAE,iBAAiB,EAAE;AAAA,QAC9H;AAAA,QACA,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,eAAe,gBAAgB;AAAA,UAC/B,YAAY,gBAAgB,MAAM,GAAG,EAAE;AAAA,QACzC;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,UAAM,WAAW,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAC3D,UAAM,mBAAmB,kBACrB,kBAAkB,KAAK,KAAK,sBAAsB,CAAC,IACnD;AACJ,UAAM,gBAAgB,oBAAI,IAAY;AACtC,UAAM,iBAAiB,CAAC,OAAkC,QAAgB;AACxE,UAAI,aAAa;AACjB,UAAI;AAAE,qBAAa,IAAI,IAAI,GAAG,EAAE;AAAA,MAAK,QAAQ;AAAA,MAAwC;AACrF,aAAO,GAAG,SAAS,EAAE,KAAK,UAAU;AAAA,IACtC;AACA,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,QAAQ,OAAO;AACxB,cAAM,EAAE,cAAc,OAAO,QAAQ,SAAS,UAAU,WAAW,GAAG,YAAY,IAAI;AACtF,cAAM,IAAI,QAAQ,IAAI,KAAK,GAAG;AAC9B,cAAM,MAAM,EAAE,GAAG,aAAa,SAAS,GAAG,WAAW,GAAG,YAAY,GAAG,cAAc,MAAM,QAAQ,GAAG,UAAU,MAAM;AACtH,YAAI,CAAC,SAAS,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,EAAG,OAAM,KAAK,UAAU,OAAO;AAC7E,YAAI,kBAAkB;AACpB,gBAAM,UAAU,eAAe,KAAK,uBAAuB,KAAK,eAAe,KAAK,GAAG;AACvF,wBAAc,IAAI,OAAO;AACzB,gBAAM,aAAa;AAAA,YACjB,gBAAgB,KAAK,yBAAyB;AAAA,YAC9C,kBAAkB,KAAK,oBAAoB;AAAA,YAC3C,aAAa,KAAK,eAAe,KAAK;AAAA,YACtC,aAAa,KAAK,eAAe;AAAA,YACjC,QAAQ,KAAK,qBAAqB,eAAe,aAAa;AAAA,YAC9D,YAAY,KAAK;AAAA,YACjB,aAAa,KAAK,eAAe;AAAA,YACjC,WAAW,KAAK;AAAA,YAChB,aAAa,KAAK,eAAe;AAAA,YACjC,eAAe,KAAK,iBAAiB;AAAA,UACvC;AACA,cAAI,CAAC,iBAAiB,MAAM,KAAK,UAAU,UAAU,IAAI,IAAI,EAAG,OAAM,KAAK,kBAAkB,OAAO;AAAA,QACtG;AAAA,MACF;AAAA,IACF;AACA,QAAI,oBAAoB,iBAAiB,UAAU,MAAM;AACvD,YAAM,WAAW,gBAAgB;AACjC,YAAM,gBAAgB,SAAS;AAC/B,YAAM,UAAU,MAAM;AACpB,YAAI,SAAS,QAAQ,OAAQ,QAAO,CAAC,GAAG,IAAI,IAAI,SAAS,MAAM,CAAC,EAAE,KAAK;AACvE,YAAI,CAAC,SAAS,QAAQ,CAAC,SAAS,GAAI,QAAO,CAAC;AAC5C,cAAM,SAAmB,CAAC;AAC1B,cAAM,SAAS,oBAAI,KAAK,GAAG,SAAS,IAAI,eAAe;AACvD,cAAM,MAAM,oBAAI,KAAK,GAAG,SAAS,EAAE,eAAe;AAClD,eAAO,UAAU,OAAO,OAAO,SAAS,IAAI;AAC1C,iBAAO,KAAK,GAAG,OAAO,eAAe,CAAC,IAAI,OAAO,OAAO,YAAY,IAAI,CAAC,EAAE,SAAS,GAAG,GAAG,CAAC,EAAE;AAC7F,iBAAO,YAAY,OAAO,YAAY,KAAK,SAAS,kBAAkB,EAAE;AAAA,QAC1E;AACA,eAAO;AAAA,MACT,GAAG;AACH,iBAAW,kBAAkB,QAAQ;AACnC,mBAAW,eAAe,eAAe;AACvC,cAAI,cAAc,IAAI,eAAe,gBAAgB,WAAW,CAAC,EAAG;AACpE,cAAI,CAAC,iBAAiB,MAAM,KAAK,UAAU;AAAA,YACzC;AAAA,YACA,kBAAkB;AAAA,YAClB;AAAA,YACA,aAAa;AAAA,YACb,QAAQ;AAAA,YACR,YAAY;AAAA,YACZ,aAAa;AAAA,YACb,WAAW;AAAA,YACX,aAAa;AAAA,YACb,eAAe;AAAA,UACjB,CAAC,IAAI,IAAI,EAAG,OAAM,KAAK,kBAAkB,OAAO;AAAA,QAClD;AAAA,MACF;AAAA,IACF;AACA,UAAM,gBAAgB,SAAS,QAAQ;AACvC,UAAM,wBAAwB,mBAAmB,SAAS,gBAAgB,IAAI,QAAQ,QAAQ;AAC9F,aAAS,IAAI;AACb,sBAAkB,IAAI;AACtB,UAAM,QAAQ,IAAI,CAAC,eAAe,qBAAqB,CAAC;AAExD,kBAAc,KAAK,KAAK,WAAW,GAAG,QAAQ;AAC9C,kBAAc,KAAK,KAAK,oBAAoB,GAAG,KAAK,UAAU,cAAc,MAAM,CAAC,CAAC;AACpF,QAAI,iBAAiB;AACnB,oBAAc,KAAK,KAAK,uBAAuB,GAAG,KAAK,UAAU;AAAA,QAC/D,SAAS,gBAAgB,WAAW,IAAI;AAAA,QACxC,WAAW,gBAAgB,YAAY;AAAA,QACvC,qBAAqB,gBAAgB,uBAAuB;AAAA,QAC5D,eAAe;AAAA,MACjB,GAAG,MAAM,CAAC,CAAC;AAAA,IACb;AACA,kBAAc,KAAK,KAAK,aAAa,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,aAAa,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAClG,kBAAc,KAAK,KAAK,gBAAgB,GAAG,iBAAiB,UAAU,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,KAAK,UAAU,WAAW,SAAS,MAAM,CAAC,CAAC;AAC1F,kBAAc,KAAK,KAAK,uBAAuB,GAAG,KAAK,UAAU,WAAW,iBAAiB,MAAM,CAAC,CAAC;AACrG,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,KAAK,cAAc,GAAG,OAAO,WAAW,KAAK,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACtG,oBAAc,KAAK,KAAK,qBAAqB,GAAG,KAAK,UAAU,OAAO,WAAW,SAAS,MAAM,CAAC,CAAC;AAAA,IACpG;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,gBAAgB,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACxG,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,eAAe,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACvG,QAAI,gBAAgB;AAClB,oBAAc,KAAK,KAAK,8BAA8B,GAAG,KAAK,UAAU;AAAA,QACtE,QAAQ;AAAA,QACR,YAAY,eAAe;AAAA,QAC3B,QAAQ;AAAA,QACR,iBAAiB;AAAA,QACjB,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,cAAc;AAAA,MAChB,GAAG,MAAM,CAAC,CAAC;AAAA,IACb;AAEA,UAAM,gBAAuD;AAAA,MAC3D,EAAE,MAAM,aAAa,MAAM,gBAAgB;AAAA,MAC3C,EAAE,MAAM,sBAAsB,MAAM,mBAAmB;AAAA,MACvD,EAAE,MAAM,kBAAkB,MAAM,uBAAuB;AAAA,MACvD,EAAE,MAAM,gBAAgB,MAAM,uBAAuB;AAAA,MACrD,EAAE,MAAM,eAAe,MAAM,mBAAmB;AAAA,MAChD,EAAE,MAAM,eAAe,MAAM,uBAAuB;AAAA,MACpD,EAAE,MAAM,kBAAkB,MAAM,mBAAmB;AAAA,MACnD,EAAE,MAAM,sBAAsB,MAAM,uBAAuB;AAAA,MAC3D,EAAE,MAAM,kBAAkB,MAAM,gBAAgB;AAAA,MAChD,EAAE,MAAM,sBAAsB,MAAM,mBAAmB;AAAA,MACvD,EAAE,MAAM,yBAAyB,MAAM,mBAAmB;AAAA,IAC5D;AACA,QAAI,iBAAiB;AACnB,oBAAc,KAAK,EAAE,MAAM,yBAAyB,MAAM,mBAAmB,CAAC;AAC9E,oBAAc,KAAK,EAAE,MAAM,wBAAwB,MAAM,uBAAuB,CAAC;AAAA,IACnF;AACA,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,EAAE,MAAM,gBAAgB,MAAM,uBAAuB,CAAC;AACzE,oBAAc,KAAK,EAAE,MAAM,uBAAuB,MAAM,mBAAmB,CAAC;AAAA,IAC9E;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,kBAAkB,MAAM,mBAAmB,CAAC;AAC5F,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,iBAAiB,MAAM,mBAAmB,CAAC;AAC3F,QAAI,eAAgB,eAAc,KAAK,EAAE,MAAM,gCAAgC,MAAM,mBAAmB,CAAC;AAEzG,UAAM,MAAM,IAAI,QAAQ;AACxB,QAAI,IAAI,UAAU,KAAM,OAAM,IAAI,MAAM,wCAAwC;AAChF,UAAM,kBAAkB,sCAAsC;AAAA,MAC5D,SAAS,OAAO,IAAI,MAAM;AAAA,MAC1B,OAAO,IAAI;AAAA;AAAA;AAAA,MAGX,WAAW,oBAAI,KAAK;AAAA,MACpB,SAAS,IAAI;AAAA,IACf,CAAC;AACD,eAAW,KAAK,cAAe,KAAI,QAAQ,KAAK,KAAK,EAAE,IAAI,GAAG,EAAE,IAAI;AACpE,QAAI,gBAAgB,SAAS,GAAG;AAC9B,YAAM,WAAW,WAAW,IAAI,EAAE,EAAE,OAAO,aAAa,EAAE;AAC1D,UAAI,eAA2B,CAAC;AAChC,YAAM,eAAe,YAA+B;AAClD,mBAAS;AACP,gBAAM,OAAO,aAAa,MAAM;AAChC,cAAI,MAAM;AACR,gBAAI,KAAK,aAAc,QAAO;AAC9B;AAAA,UACF;AACA,gBAAM,OAAO,MAAM,SAAS,KAAK;AACjC,cAAI,KAAK,KAAM,OAAM,IAAI,MAAM,6DAA6D;AAC5F,yBAAe,KAAK;AAAA,QACtB;AAAA,MACF;AACA,iBAAW,SAAS,iBAAiB;AACnC,YAAI,kBAAkB,MAAM,MAAM,EAAE,MAAM,MAAM,KAAK,GAAG,cAAY;AAClE,eAAK,aAAa,EAAE,KAAK,UAAQ;AAC/B,gBAAI,KAAK,QAAQ,MAAM,IAAK,OAAM,IAAI,MAAM,+BAA+B,MAAM,GAAG,EAAE;AACtF,kBAAM,eAAe,KAAK,cACtB;AAAA,gBAAmB,KAAK,WAAW;AAAA,qBAAwB,KAAK,oBAAoB,SAAS;AAAA,mBAAsB,KAAK,yBAAyB,aAAa,KAC9J;AACJ,qBAAS,MAAM,SAAS,KAAK;AAAA,cAC3B,KAAK,KAAK,SAAS,KAAK,GAAG;AAAA;AAAA,OAAY,KAAK,GAAG,GAAG,YAAY;AAAA,UAAa,KAAK,MAAM;AAAA;AAAA;AAAA;AAAA,EAAc,KAAK,YAAY;AAAA;AAAA,YACvH,CAAC,CAAC;AAAA,UACJ,CAAC,EAAE,MAAM,WAAS,SAAS,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC,CAAC,CAAC;AAAA,QACvF,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,kBAAkB,eAAe,cAAc;AACjD,YAAM,YAAY,KAAK,KAAK,QAAQ;AACpC,iBAAW,OAAO,YAAY,WAAW,EAAE,WAAW,KAAK,CAAC,GAAe;AACzE,cAAM,OAAO,KAAK,WAAW,GAAG;AAChC,YAAI,SAAS,IAAI,EAAE,OAAO,EAAG,KAAI,QAAQ,MAAM,UAAU,IAAI,MAAM,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE;AAAA,MACtF;AAAA,IACF;AACA,QAAI,IAAI;AACR,WAAO,CAAC,MAAM,eAAe;AAAA,EAC/B,UAAE;AACA,WAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAAA,EAC9C;AACF;","names":[]}