mcp-scraper 0.88.2 → 0.89.0

package/dist/extract-bundle-565SBZCR.js

@@ -1,1003 +0,0 @@
-import {
-  commonsEmbedDim,
-  commonsEmbedModel,
-  downloadAsset,
-  embedCommonsTexts,
-  publicizeExtractionFailure
-} from "./chunk-F5GQJWZU.js";
-import {
-  createSiteExtractBundleArtifactStream,
-  createSiteExtractContentReader,
-  createSiteExtractImageArtifact,
-  extractJobLimitInfo
-} from "./chunk-IHXAXYIS.js";
-import "./chunk-OZJMVCDK.js";
-import {
-  computeIssues,
-  renderImageSection,
-  renderIssueReport,
-  renderLinkReport
-} from "./chunk-PGJQDMC2.js";
-import "./chunk-DNM65UCK.js";
-import "./chunk-4QMUF6XM.js";
-import "./chunk-GGZEC22A.js";
-import {
-  getDb
-} from "./chunk-YXNDOQXN.js";
-
-// src/api/extract-bundle.ts
-import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
-import { createHash as createHash2 } from "crypto";
-import { basename, join } from "path";
-import { tmpdir } from "os";
-import { createGzip } from "zlib";
-import { once } from "events";
-import { finished } from "stream/promises";
-import { ZipFile } from "yazl";
-import pLimit from "p-limit";
-
-// src/api/site-content-similarity.ts
-import { createHash } from "crypto";
-var MAX_SIMILARITY_PAGES = 500;
-var DEFAULT_SIMILARITY_THRESHOLD = 0.9;
-var DEFAULT_SIMILARITY_MAX_PAIRS = 1e4;
-var MAX_SIMILARITY_PAIRS = 5e4;
-function round(value, places) {
-  const scale = 10 ** places;
-  return Math.round(value * scale) / scale;
-}
-function cosineSimilarity(a, b) {
-  if (a.length === 0 || a.length !== b.length) throw new Error("Similarity vectors must have the same non-zero dimension.");
-  let dot = 0;
-  let normA = 0;
-  let normB = 0;
-  for (let index = 0; index < a.length; index++) {
-    dot += a[index] * b[index];
-    normA += a[index] * a[index];
-    normB += b[index] * b[index];
-  }
-  if (normA === 0 || normB === 0) return 0;
-  return Math.max(-1, Math.min(1, dot / (Math.sqrt(normA) * Math.sqrt(normB))));
-}
-function corpusHash(pages) {
-  const digest = createHash("sha256");
-  for (const page of [...pages].sort((a, b) => a.url.localeCompare(b.url))) {
-    digest.update(page.url);
-    digest.update("\0");
-    digest.update(page.contentHash);
-    digest.update("\0");
-  }
-  return digest.digest("hex");
-}
-function markdownBlockSignature(block) {
-  const normalized = block.replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/\[([^\]]+)\]\([^)]*\)/g, "$1").replace(/https?:\/\/\S+/gi, " ").replace(/&(?:amp|nbsp|quot|#39);/gi, " ").replace(/[#>*_`~|\\-]+/g, " ").replace(/[^\p{L}\p{N}]+/gu, " ").replace(/\s+/g, " ").trim().toLowerCase();
-  return normalized.length >= 12 ? normalized : null;
-}
-function prepareEmbeddingTexts(pages) {
-  const blocksByPage = pages.map((page) => page.bodyMarkdown.split(/\n{2,}/).map((block) => block.trim()).filter(Boolean));
-  const minimumPageCount = pages.length >= 4 ? Math.max(3, Math.ceil(pages.length * 0.5)) : null;
-  const pageFrequency = /* @__PURE__ */ new Map();
-  for (const blocks of blocksByPage) {
-    const pageSignatures = new Set(blocks.map(markdownBlockSignature).filter((value) => Boolean(value)));
-    for (const signature of pageSignatures) pageFrequency.set(signature, (pageFrequency.get(signature) ?? 0) + 1);
-  }
-  const repeated = /* @__PURE__ */ new Set();
-  if (minimumPageCount != null) {
-    for (const [signature, count] of pageFrequency) {
-      if (count >= minimumPageCount) repeated.add(signature);
-    }
-  }
-  let removedCharacters = 0;
-  const texts = pages.map((page, pageIndex) => {
-    const kept = [];
-    for (const block of blocksByPage[pageIndex]) {
-      const signature = markdownBlockSignature(block);
-      if (signature && repeated.has(signature)) {
-        removedCharacters += block.length;
-        continue;
-      }
-      kept.push(block);
-    }
-    const cleaned = kept.join("\n\n").trim();
-    const content = cleaned.length >= 60 ? cleaned : page.bodyMarkdown;
-    return `${page.title?.trim() || "Untitled page"}
-
-${content}`;
-  });
-  return {
-    texts,
-    minimumPageCount,
-    removedBlockSignatures: repeated.size,
-    removedCharacters
-  };
-}
-var DisjointSet = class {
-  parent;
-  constructor(size) {
-    this.parent = Array.from({ length: size }, (_, index) => index);
-  }
-  find(value) {
-    const parent = this.parent[value];
-    if (parent !== value) this.parent[value] = this.find(parent);
-    return this.parent[value];
-  }
-  union(a, b) {
-    const rootA = this.find(a);
-    const rootB = this.find(b);
-    if (rootA !== rootB) this.parent[rootB] = rootA;
-  }
-};
-async function analyzeSiteContentSimilarity(pages, options = {}) {
-  const eligible = pages.filter((page) => page.bodyMarkdown.trim().length > 0).slice(0, MAX_SIMILARITY_PAGES);
-  const requestedThreshold = Number.isFinite(options.threshold) ? options.threshold : DEFAULT_SIMILARITY_THRESHOLD;
-  const requestedMaxPairs = Number.isFinite(options.maxPairs) ? options.maxPairs : DEFAULT_SIMILARITY_MAX_PAIRS;
-  const threshold = Math.max(0, Math.min(1, requestedThreshold));
-  const maxPairs = Math.max(1, Math.min(MAX_SIMILARITY_PAIRS, Math.floor(requestedMaxPairs)));
-  const model = options.model ?? commonsEmbedModel();
-  const dimensions = options.dimensions ?? commonsEmbedDim();
-  const embedTexts = options.embedTexts ?? embedCommonsTexts;
-  const prepared = prepareEmbeddingTexts(eligible);
-  const uniqueTexts = [];
-  const textIndexByHash = /* @__PURE__ */ new Map();
-  const pageTextIndexes = [];
-  for (const [pageIndex, page] of eligible.entries()) {
-    const key = page.contentHash || createHash("sha256").update(page.bodyMarkdown).digest("hex");
-    let textIndex = textIndexByHash.get(key);
-    if (textIndex == null) {
-      textIndex = uniqueTexts.length;
-      textIndexByHash.set(key, textIndex);
-      uniqueTexts.push(prepared.texts[pageIndex]);
-    }
-    pageTextIndexes.push(textIndex);
-  }
-  const uniqueVectors = [];
-  for (let offset = 0; offset < uniqueTexts.length; offset += 64) {
-    uniqueVectors.push(...await embedTexts(uniqueTexts.slice(offset, offset + 64)));
-  }
-  if (uniqueVectors.length !== uniqueTexts.length) {
-    throw new Error(`Embedding provider returned ${uniqueVectors.length} vectors for ${uniqueTexts.length} unique page bodies.`);
-  }
-  const vectors = pageTextIndexes.map((index) => uniqueVectors[index]);
-  const sets = new DisjointSet(eligible.length);
-  const allScores = [];
-  const qualifying = [];
-  for (let source = 0; source < eligible.length; source++) {
-    for (let target = source + 1; target < eligible.length; target++) {
-      const score = cosineSimilarity(vectors[source], vectors[target]);
-      allScores.push(score);
-      if (score < threshold) continue;
-      qualifying.push({ source, target, score });
-      sets.union(source, target);
-    }
-  }
-  qualifying.sort((a, b) => b.score - a.score || eligible[a.source].url.localeCompare(eligible[b.source].url) || eligible[a.target].url.localeCompare(eligible[b.target].url));
-  const sortedScores = [...allScores].sort((a, b) => a - b);
-  const quantile = (value) => {
-    if (!sortedScores.length) return null;
-    return round(sortedScores[Math.floor((sortedScores.length - 1) * value)], 6);
-  };
-  const membersByRoot = /* @__PURE__ */ new Map();
-  for (let index = 0; index < eligible.length; index++) {
-    const root = sets.find(index);
-    const members = membersByRoot.get(root) ?? [];
-    members.push(index);
-    membersByRoot.set(root, members);
-  }
-  const clusterIdByPage = /* @__PURE__ */ new Map();
-  const clusters = [...membersByRoot.values()].filter((members) => members.length > 1).sort((a, b) => b.length - a.length || eligible[a[0]].url.localeCompare(eligible[b[0]].url)).map((members, index) => {
-    const clusterId = `cluster-${String(index + 1).padStart(3, "0")}`;
-    for (const member of members) clusterIdByPage.set(member, clusterId);
-    return {
-      clusterId,
-      pageCount: members.length,
-      urls: members.map((member) => eligible[member].url).sort()
-    };
-  });
-  const rows = qualifying.slice(0, maxPairs).map((pair, pairIndex) => {
-    const source = eligible[pair.source];
-    const target = eligible[pair.target];
-    const similarity = round(pair.score, 6);
-    return {
-      sourceUrl: source.url,
-      sourceTitle: source.title,
-      targetUrl: target.url,
-      targetTitle: target.title,
-      similarity,
-      similarityPercent: round(similarity * 100, 2),
-      corpusPercentile: sortedScores.length <= 1 ? 100 : round((1 - pairIndex / (sortedScores.length - 1)) * 100, 2),
-      exactContentDuplicate: Boolean(source.contentHash && source.contentHash === target.contentHash),
-      sourceWordCount: source.wordCount,
-      targetWordCount: target.wordCount,
-      clusterId: clusterIdByPage.get(pair.source) ?? null
-    };
-  });
-  const corpusSha256 = corpusHash(eligible);
-  const analysisSha256 = createHash("sha256").update(JSON.stringify({
-    corpusSha256,
-    model,
-    dimensions,
-    threshold,
-    maxPairs,
-    boilerplateRemoval: {
-      method: "corpus_repeated_markdown_blocks",
-      minimumPageCount: prepared.minimumPageCount,
-      removedBlockSignatures: prepared.removedBlockSignatures
-    }
-  })).digest("hex");
-  return {
-    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
-    provider: "jina",
-    model,
-    dimensions,
-    threshold,
-    requestedMaxPairs: maxPairs,
-    comparedPages: eligible.length,
-    possiblePairs: eligible.length * Math.max(0, eligible.length - 1) / 2,
-    qualifyingPairs: qualifying.length,
-    returnedPairs: rows.length,
-    pairsTruncated: qualifying.length > rows.length,
-    corpusSha256,
-    analysisSha256,
-    scoreDistribution: {
-      min: quantile(0),
-      p25: quantile(0.25),
-      median: quantile(0.5),
-      p75: quantile(0.75),
-      p90: quantile(0.9),
-      max: quantile(1)
-    },
-    boilerplateRemoval: {
-      method: "corpus_repeated_markdown_blocks",
-      minimumPageCount: prepared.minimumPageCount,
-      removedBlockSignatures: prepared.removedBlockSignatures,
-      removedCharacters: prepared.removedCharacters
-    },
-    rows,
-    clusters
-  };
-}
-
-// src/api/extract-bundle.ts
-var SITE_EXTRACT_PAGE_CHUNK = 25;
-var MAX_IMAGES_PER_PAGE = 20;
-var MAX_IMAGES_PER_SITE = 500;
-var IMAGE_DOWNLOAD_CONCURRENCY = 8;
-var MAX_IMAGE_BYTES = 10 * 1024 * 1024;
-var MAX_SITE_IMAGE_BYTES = 100 * 1024 * 1024;
-var MAX_ANALYZED_LINK_EDGES = 25e4;
-function normalize(u) {
-  return u.split("#")[0].replace(/\/$/, "");
-}
-function registrableDomain(host) {
-  const h = host.replace(/^www\./, "").toLowerCase();
-  const parts = h.split(".");
-  return parts.length <= 2 ? h : parts.slice(-2).join(".");
-}
-function safeImageFilename(url, index) {
-  try {
-    const u = new URL(url);
-    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
-    return base || `image-${index}`;
-  } catch {
-    return `image-${index}`;
-  }
-}
-function slugFactory() {
-  const counts = /* @__PURE__ */ new Map();
-  return (url) => {
-    const base = url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "page";
-    const n = counts.get(base) ?? 0;
-    counts.set(base, n + 1);
-    return n ? `${base}-${n}` : base;
-  };
-}
-async function* pageChunks(jobId) {
-  const db = getDb();
-  let lastRowId = 0;
-  for (; ; ) {
-    const res = await db.execute({
-      sql: `SELECT rowid AS cursor_id, page
-            FROM site_extract_pages
-            WHERE job_id = ? AND rowid > ?
-            ORDER BY rowid
-            LIMIT ${SITE_EXTRACT_PAGE_CHUNK}`,
-      args: [jobId, lastRowId]
-    });
-    if (!res.rows.length) return;
-    lastRowId = Number(res.rows.at(-1)?.cursor_id ?? lastRowId);
-    if (!Number.isSafeInteger(lastRowId) || lastRowId <= 0) throw new Error("site extract page cursor is invalid");
-    yield res.rows.map((r) => JSON.parse(String(r.page)));
-  }
-}
-function csvCell(value) {
-  const text = value == null ? "" : String(value);
-  return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
-}
-function similarityTableRow(row) {
-  return {
-    source_url: row.sourceUrl,
-    source_title: row.sourceTitle,
-    target_url: row.targetUrl,
-    target_title: row.targetTitle,
-    similarity: row.similarity,
-    similarity_percent: row.similarityPercent,
-    corpus_percentile: row.corpusPercentile,
-    exact_content_duplicate: row.exactContentDuplicate,
-    source_word_count: row.sourceWordCount,
-    target_word_count: row.targetWordCount,
-    cluster_id: row.clusterId
-  };
-}
-async function assembleExtractArtifacts(job, extras = {}) {
-  const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`);
-  const downloadImages = job.options.downloadImages === true;
-  if (job.userId == null) throw new Error("site extract artifact owner is missing");
-  const readPageContent = createSiteExtractContentReader(String(job.userId));
-  mkdirSync(dir, { recursive: true });
-  if (downloadImages) mkdirSync(join(dir, "images"), { recursive: true });
-  const captureRenderedDom = job.options.captureRenderedDom === true;
-  const semanticSimilarity = job.options.semanticSimilarity === true;
-  if (captureRenderedDom) mkdirSync(join(dir, "rendered-dom"), { recursive: true });
-  try {
-    const metas = [];
-    const similarityPages = [];
-    const renderedDomManifest = [];
-    const domSlug = slugFactory();
-    const pageExportEntries = [];
-    const waybackTimeline = job.options.waybackTimeline;
-    const statusByUrl = /* @__PURE__ */ new Map();
-    for await (const chunk of pageChunks(job.id)) {
-      for (const p of chunk) {
-        statusByUrl.set(normalize(p.url), p.status);
-        metas.push({
-          url: p.url,
-          status: p.status,
-          title: p.title,
-          titleLength: p.titleLength,
-          titlePixels: p.titlePixels,
-          metaDescription: p.metaDescription,
-          metaDescLength: p.metaDescLength,
-          h1: p.h1,
-          h1_2: p.h1_2,
-          h2Count: p.h2Count,
-          extractionStatus: p.extractionStatus,
-          indexable: p.indexable,
-          indexabilityReason: p.indexabilityReason,
-          canonicalUrl: p.canonicalUrl ? p.url : null,
-          wordCount: p.wordCount,
-          contentHash: p.contentHash,
-          imagesMissingAlt: p.imagesMissingAlt,
-          schemaTypes: p.schemaTypes.length > 0 ? ["present"] : [],
-          outlinks: []
-        });
-        const pageId = p.pageId ?? createHash2("sha256").update(p.archivedUrl ?? p.url).digest("hex");
-        const pageDir = join(dir, "pages", pageId);
-        mkdirSync(pageDir, { recursive: true });
-        const content = p.contentRef ? await readPageContent(p.contentRef) : null;
-        const html = content?.html ?? null;
-        const markdown = content?.markdown ?? p.bodyMarkdown ?? null;
-        if (semanticSimilarity && p.extractionStatus === "successful" && markdown?.trim()) {
-          similarityPages.push({
-            url: p.url,
-            title: p.title,
-            bodyMarkdown: markdown,
-            contentHash: p.contentHash,
-            wordCount: p.wordCount
-          });
-        }
-        if (captureRenderedDom && html != null) {
-          const relativePath = `rendered-dom/${domSlug(p.url)}.html.txt`;
-          writeFileSync(join(dir, relativePath), html);
-          renderedDomManifest.push({
-            url: p.url,
-            path: relativePath,
-            bytes: Buffer.byteLength(html),
-            truncated: p.renderedDomTruncated === true,
-            sanitized: p.renderedDomSanitized === true
-          });
-        }
-        const htmlPath = html == null ? null : `pages/${pageId}/page.html`;
-        const markdownPath = markdown == null ? null : `pages/${pageId}/page.md`;
-        const legacyMarkdownPath = markdown == null || !p.archiveRequestedMonth ? null : `pages/${p.archiveRequestedMonth}/${pageId}.md`;
-        if (html != null && htmlPath) writeFileSync(join(dir, htmlPath), html);
-        if (markdown != null && markdownPath) writeFileSync(join(dir, markdownPath), markdown);
-        if (markdown != null && legacyMarkdownPath) {
-          mkdirSync(join(dir, "pages", p.archiveRequestedMonth), { recursive: true });
-          writeFileSync(join(dir, legacyMarkdownPath), markdown);
-        }
-        const { bodyMarkdown: _body, contentRef: _contentRef, discoveryLinks: _discovery, ...metadata } = p;
-        const pageRecord = {
-          version: "site-export-page.v1",
-          ...metadata,
-          pageId,
-          content: {
-            htmlPath,
-            markdownPath,
-            htmlBytes: html == null ? 0 : Buffer.byteLength(html),
-            markdownBytes: markdown == null ? 0 : Buffer.byteLength(markdown),
-            htmlSha256: html == null ? null : createHash2("sha256").update(html).digest("hex"),
-            markdownSha256: markdown == null ? null : createHash2("sha256").update(markdown).digest("hex")
-          }
-        };
-        const pageJson = JSON.stringify(pageRecord, null, 2);
-        const jsonPath = `pages/${pageId}/page.json`;
-        writeFileSync(join(dir, jsonPath), pageJson);
-        pageExportEntries.push({
-          pageId,
-          url: p.archivedUrl ?? p.url,
-          jsonPath,
-          htmlPath,
-          markdownPath,
-          legacyMarkdownPath,
-          jsonSha256: createHash2("sha256").update(pageJson).digest("hex"),
-          htmlSha256: pageRecord.content.htmlSha256,
-          markdownSha256: pageRecord.content.markdownSha256,
-          htmlBytes: pageRecord.content.htmlBytes,
-          markdownBytes: pageRecord.content.markdownBytes
-        });
-      }
-    }
-    const gz = createGzip();
-    const linksOut = createWriteStream(join(dir, "links.jsonl.gz"));
-    gz.pipe(linksOut);
-    const gzWrite = async (line) => {
-      if (!gz.write(line)) await once(gz, "drain");
-    };
-    const schemaOut = createWriteStream(join(dir, "schema.jsonl"));
-    const schemaWrite = async (line) => {
-      if (!schemaOut.write(line)) await once(schemaOut, "drain");
-    };
-    const failuresOut = createWriteStream(join(dir, "failures.jsonl"));
-    const failureWrite = async (line) => {
-      if (!failuresOut.write(line)) await once(failuresOut, "drain");
-    };
-    const inboundFrom = /* @__PURE__ */ new Map();
-    const anchorCounts = /* @__PURE__ */ new Map();
-    const adjacency = /* @__PURE__ */ new Map();
-    const outCounts = /* @__PURE__ */ new Map();
-    const domMap = /* @__PURE__ */ new Map();
-    const brokenLinkPages = /* @__PURE__ */ new Set();
-    let internalTotal = 0;
-    let externalTotal = 0;
-    let brokenInternal = 0;
-    let streamedLinkEdges = 0;
-    let analyzedLinkEdges = 0;
-    let linkAnalysisTruncated = false;
-    let siteReg = "";
-    try {
-      siteReg = registrableDomain(new URL(job.startUrl).hostname);
-    } catch {
-      siteReg = "";
-    }
-    const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY);
-    const imageDownloads = [];
-    const imageManifest = [];
-    const imageArtifacts = [];
-    let imagesQueued = 0;
-    let imagesFailed = 0;
-    let imageBytesDownloaded = 0;
-    const consumeImageBytes = (bytes) => {
-      if (imageBytesDownloaded + bytes > MAX_SITE_IMAGE_BYTES) return false;
-      imageBytesDownloaded += bytes;
-      return true;
-    };
-    for await (const chunk of pageChunks(job.id)) {
-      for (const p of chunk) {
-        if (p.schema?.length) await schemaWrite(JSON.stringify({ url: p.url, schema: p.schema }) + "\n");
-        if (p.extractionStatus === "failed") {
-          const { failureCode, failureReason } = publicizeExtractionFailure(p.failureCode, p.failureReason);
-          await failureWrite(JSON.stringify({
-            url: p.url,
-            status: p.status,
-            via: p.via,
-            failureCode,
-            failureReason,
-            fetchAttempts: p.fetchAttempts ?? []
-          }) + "\n");
-        }
-        if (p.imageLinks?.length) {
-          for (const [idx, imgUrl] of p.imageLinks.entries()) {
-            const imageId = createHash2("sha256").update(`${p.url}\0${imgUrl}`).digest("hex");
-            const record = {
-              imageId,
-              sourcePage: p.url,
-              sourceUrl: imgUrl,
-              path: null,
-              status: "not_requested",
-              mimeType: null,
-              bytes: null,
-              sha256: null,
-              failureCode: null,
-              failureReason: null,
-              artifactId: null
-            };
-            imageManifest.push(record);
-            if (!downloadImages) continue;
-            if (idx >= MAX_IMAGES_PER_PAGE) {
-              record.status = "skipped_page_limit";
-              record.failureCode = "page_image_limit";
-              record.failureReason = `Only the first ${MAX_IMAGES_PER_PAGE} image candidates are downloaded per page.`;
-              continue;
-            }
-            if (imagesQueued >= MAX_IMAGES_PER_SITE) {
-              record.status = "skipped_site_limit";
-              record.failureCode = "site_image_limit";
-              record.failureReason = `Only the first ${MAX_IMAGES_PER_SITE} image candidates are downloaded per site.`;
-              continue;
-            }
-            imagesQueued++;
-            const filename = `${imageId}-${safeImageFilename(imgUrl, idx)}`;
-            const outputPath = join(dir, "images");
-            imageDownloads.push(
-              imageLimit(() => downloadAsset(imgUrl, outputPath, filename, {
-                expectedType: "image",
-                maxBytes: MAX_IMAGE_BYTES,
-                consumeBytes: consumeImageBytes
-              })).then(async (result) => {
-                const bytes = readFileSync(result.savedPath);
-                const artifact = await createSiteExtractImageArtifact({
-                  ownerId: String(job.userId),
-                  jobId: job.id,
-                  imageId,
-                  filename,
-                  contentType: result.mimeType ?? "application/octet-stream",
-                  content: bytes,
-                  sourceUrl: imgUrl,
-                  sourcePage: p.url
-                });
-                imageArtifacts.push(artifact);
-                record.status = "downloaded";
-                record.path = `images/${filename}`;
-                record.artifactId = artifact.key;
-                record.mimeType = result.mimeType;
-                record.bytes = result.sizeBytes;
-                record.sha256 = createHash2("sha256").update(bytes).digest("hex");
-              }, (error) => {
-                imagesFailed++;
-                record.status = "failed";
-                record.failureCode = "image_download_failed";
-                record.failureReason = (error instanceof Error ? error.message : String(error)).replace(/\s+/g, " ").slice(0, 500);
-              })
-            );
-          }
-        }
-        const from = normalize(p.url);
-        const oc = { int: 0, ext: 0 };
-        for (const l of p.outlinks ?? []) {
-          const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
-          const to = normalize(l.href);
-          const targetStatus = statusByUrl.get(to) ?? null;
-          await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + "\n");
-          streamedLinkEdges++;
-          if (analyzedLinkEdges >= MAX_ANALYZED_LINK_EDGES) {
-            linkAnalysisTruncated = true;
-            continue;
-          }
-          analyzedLinkEdges++;
-          if (l.internal) {
-            oc.int++;
-            internalTotal++;
-            if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
-            inboundFrom.get(to).add(from);
-            if (l.anchor) {
-              if (!anchorCounts.has(to)) anchorCounts.set(to, /* @__PURE__ */ new Map());
-              const ac = anchorCounts.get(to);
-              ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1);
-            }
-            if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
-              if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
-              adjacency.get(from).add(to);
-            }
-            if (targetStatus != null && targetStatus >= 400) {
-              brokenInternal++;
-              brokenLinkPages.add(p.url);
-            }
-          } else {
-            oc.ext++;
-            let domain = "";
-            try {
-              domain = registrableDomain(new URL(l.href).hostname);
-            } catch {
-              continue;
-            }
-            if (!domain || domain === siteReg) continue;
-            externalTotal++;
-            const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
-            d.links++;
-            if (nofollow) d.nofollow++;
-            d.pages.add(p.url);
-            domMap.set(domain, d);
-          }
-        }
-        outCounts.set(p.url, oc);
-      }
-    }
-    const outputFinished = Promise.all([finished(linksOut), finished(schemaOut), finished(failuresOut)]);
-    gz.end();
-    schemaOut.end();
-    failuresOut.end();
-    await outputFinished;
-    await Promise.all(imageDownloads);
-    const depth = /* @__PURE__ */ new Map();
-    const start = normalize(job.startUrl);
-    depth.set(start, 0);
-    const queue = [start];
-    while (queue.length > 0) {
-      const cur = queue.shift();
-      const d = depth.get(cur);
-      for (const next of adjacency.get(cur) ?? []) {
-        if (!depth.has(next)) {
-          depth.set(next, d + 1);
-          queue.push(next);
-        }
-      }
-    }
-    const metrics = /* @__PURE__ */ new Map();
-    for (const p of metas) {
-      const key = normalize(p.url);
-      const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
-      const ac = anchorCounts.get(key);
-      const topAnchors = ac ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]) : [];
-      const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 };
-      metrics.set(p.url, {
-        url: p.url,
-        inlinks: inbound.size,
-        uniqueInlinks: inbound.size,
-        outlinksInternal: oc.int,
-        outlinksExternal: oc.ext,
-        crawlDepth: depth.has(key) ? depth.get(key) : null,
-        orphan: inbound.size === 0 && key !== start,
-        topAnchors
-      });
-    }
-    const issues = computeIssues(metas, metrics, brokenLinkPages);
-    let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics);
-    const limits = extractJobLimitInfo(job);
-    const crawlSummary = {
-      jobId: job.id,
-      startUrl: job.startUrl,
-      status: job.successfulUrls === 0 ? "failed" : job.failedUrls > 0 || job.remainingUrls > 0 || limits.creditTruncated ? "partial" : "complete",
-      ...limits,
-      discovered: job.totalUrls,
-      attempted: job.attemptedUrls,
-      successful: job.successfulUrls,
-      failed: job.failedUrls,
-      remaining: job.remainingUrls,
-      linkAnalysis: {
-        truncated: linkAnalysisTruncated,
-        analyzedEdges: analyzedLinkEdges,
-        streamedEdges: streamedLinkEdges,
-        edgeCap: MAX_ANALYZED_LINK_EDGES
-      },
-      generatedAt: (/* @__PURE__ */ new Date()).toISOString()
-    };
-    reportMd = [
-      `## Crawl delivery`,
-      `- Discovered: ${crawlSummary.discovered}`,
-      `- Attempted: ${crawlSummary.attempted}`,
-      `- Successful: ${crawlSummary.successful}`,
-      `- Failed: ${crawlSummary.failed}`,
-      `- Remaining: ${crawlSummary.remaining}`,
-      `- Requested page cap: ${limits.requestedMaxPages}`,
-      `- Funded page cap: ${limits.effectiveMaxPages}`,
-      `- Credit-truncated: ${limits.creditTruncated ? "yes" : "no"}`,
-      `- Link analysis: ${analyzedLinkEdges}/${streamedLinkEdges} edges${linkAnalysisTruncated ? ` (capped at ${MAX_ANALYZED_LINK_EDGES})` : ""}`,
-      "",
-      reportMd
-    ].join("\n");
-    if (extras.imageAudit) reportMd += `
-
-${renderImageSection(extras.imageAudit)}`;
-    const metricValues = [...metrics.values()];
-    const round2 = (n) => Math.round(n * 10) / 10;
-    const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
-    let sumInlinks = 0;
-    let sumOutlinks = 0;
-    for (const m of metricValues) {
-      sumInlinks += m.inlinks;
-      sumOutlinks += m.outlinksInternal + m.outlinksExternal;
-      if (m.inlinks === 0) distribution.zero++;
-      else if (m.inlinks <= 2) distribution.oneToTwo++;
-      else if (m.inlinks <= 10) distribution.threeToTen++;
-      else distribution.elevenPlus++;
-    }
-    const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
-    const linkReport = {
-      summary: {
-        internal: {
-          totalLinks: internalTotal,
-          pages: metas.length,
-          orphans: metricValues.filter((m) => m.orphan).length,
-          brokenInternal,
-          avgInlinks: metas.length ? round2(sumInlinks / metas.length) : 0,
-          avgOutlinks: metas.length ? round2(sumOutlinks / metas.length) : 0,
-          distribution,
-          topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }))
-        },
-        external: {
-          totalLinks: externalTotal,
-          uniqueDomains: externalDomains.length,
-          topDomains: externalDomains.slice(0, 20)
-        }
-      },
-      externalDomains
-    };
-    const pagesOut = createWriteStream(join(dir, "pages.jsonl"));
-    const pageExportById = new Map(pageExportEntries.map((entry) => [entry.pageId, entry]));
-    const captureMatrixOut = waybackTimeline ? createWriteStream(join(dir, "capture-matrix.jsonl")) : null;
-    const capturedCells = /* @__PURE__ */ new Set();
-    const captureCellKey = (month, url) => {
-      let normalized = url;
-      try {
-        normalized = new URL(url).href;
-      } catch {
-      }
-      return `${month ?? ""}\0${normalized}`;
-    };
-    for await (const chunk of pageChunks(job.id)) {
-      for (const page of chunk) {
-        const publicFailure = page.extractionStatus === "failed" ? publicizeExtractionFailure(page.failureCode, page.failureReason) : { failureCode: null, failureReason: null };
-        const { bodyMarkdown: _body, schema: _schema, outlinks: _outlinks, contentRef: _contentRef, ...boundedPage } = page;
-        const m = metrics.get(page.url);
-        const exported = page.pageId ? pageExportById.get(page.pageId) : void 0;
-        const row = {
-          ...boundedPage,
-          ...page.extractionStatus === "failed" ? publicFailure : {},
-          representations: exported ? {
-            jsonPath: exported.jsonPath,
-            htmlPath: exported.htmlPath,
-            markdownPath: exported.markdownPath,
-            jsonSha256: exported.jsonSha256,
-            htmlSha256: exported.htmlSha256,
-            markdownSha256: exported.markdownSha256,
-            htmlBytes: exported.htmlBytes,
-            markdownBytes: exported.markdownBytes
-          } : null,
-          inlinks: m?.inlinks ?? 0,
-          crawlDepth: m?.crawlDepth ?? null,
-          orphan: m?.orphan ?? false
-        };
-        if (!pagesOut.write(JSON.stringify(row) + "\n")) await once(pagesOut, "drain");
-        if (captureMatrixOut) {
-          const cellKey = captureCellKey(page.archiveRequestedMonth, page.originalUrl ?? page.url);
-          capturedCells.add(cellKey);
-          const captureRow = {
-            requestedMonth: page.archiveRequestedMonth ?? null,
-            captureTimestamp: page.archiveTimestamp ?? null,
-            originalUrl: page.originalUrl ?? page.url,
-            archivedUrl: page.archivedUrl ?? null,
-            status: page.extractionStatus === "successful" ? "captured" : "failed",
-            httpStatus: page.status,
-            contentHash: page.contentHash || null,
-            wordCount: page.wordCount,
-            failureCode: publicFailure.failureCode,
-            failureReason: publicFailure.failureReason
-          };
-          if (!captureMatrixOut.write(JSON.stringify(captureRow) + "\n")) await once(captureMatrixOut, "drain");
-        }
-      }
-    }
-    if (captureMatrixOut && waybackTimeline?.timeline?.urls) {
-      const timeline = waybackTimeline.timeline;
-      const requestedUrls = timeline.urls;
-      const months = (() => {
-        if (timeline.months?.length) return [...new Set(timeline.months)].sort();
-        if (!timeline.from || !timeline.to) return [];
-        const values = [];
-        const cursor = /* @__PURE__ */ new Date(`${timeline.from}-01T00:00:00Z`);
-        const end = /* @__PURE__ */ new Date(`${timeline.to}-01T00:00:00Z`);
-        while (cursor <= end && values.length < 60) {
-          values.push(`${cursor.getUTCFullYear()}-${String(cursor.getUTCMonth() + 1).padStart(2, "0")}`);
-          cursor.setUTCMonth(cursor.getUTCMonth() + (timeline.intervalMonths ?? 1));
-        }
-        return values;
-      })();
-      for (const requestedMonth of months) {
-        for (const originalUrl of requestedUrls) {
-          if (capturedCells.has(captureCellKey(requestedMonth, originalUrl))) continue;
-          if (!captureMatrixOut.write(JSON.stringify({
-            requestedMonth,
-            captureTimestamp: null,
-            originalUrl,
-            archivedUrl: null,
-            status: "missing",
-            httpStatus: null,
-            contentHash: null,
-            wordCount: 0,
-            failureCode: "wayback_capture_missing",
-            failureReason: "No successful HTML capture was available in the requested month."
-          }) + "\n")) await once(captureMatrixOut, "drain");
-        }
-      }
-    }
-    const pagesFinished = finished(pagesOut);
-    const captureMatrixFinished = captureMatrixOut ? finished(captureMatrixOut) : Promise.resolve();
-    pagesOut.end();
-    captureMatrixOut?.end();
-    await Promise.all([pagesFinished, captureMatrixFinished]);
-    if (captureRenderedDom) {
-      writeFileSync(
-        join(dir, "rendered-dom", "manifest.jsonl"),
-        renderedDomManifest.map((row) => JSON.stringify(row)).join("\n")
-      );
-    }
-    if (semanticSimilarity) {
-      const analysis = await analyzeSiteContentSimilarity(similarityPages, {
-        threshold: Number(job.options.similarityThreshold ?? void 0),
-        maxPairs: Number(job.options.similarityMaxPairs ?? void 0),
-        embedTexts: extras.embedSimilarityTexts
-      });
-      const tableRows = analysis.rows.map(similarityTableRow);
-      const columns = tableRows.length > 0 ? Object.keys(tableRows[0]) : [
-        "source_url",
-        "source_title",
-        "target_url",
-        "target_title",
-        "similarity",
-        "similarity_percent",
-        "corpus_percentile",
-        "exact_content_duplicate",
-        "source_word_count",
-        "target_word_count",
-        "cluster_id"
-      ];
-      const csv = [
-        columns.join(","),
-        ...tableRows.map((row) => columns.map((column) => csvCell(row[column] ?? null)).join(","))
-      ].join("\n") + "\n";
-      const { rows: _rows, clusters: _clusters, ...summary } = analysis;
-      writeFileSync(join(dir, "similarity-table.csv"), csv);
-      writeFileSync(join(dir, "similarity.jsonl"), tableRows.map((row) => JSON.stringify(row)).join("\n"));
-      writeFileSync(join(dir, "similarity-table-schema.json"), JSON.stringify({
-        tableNameSuggestion: `site_similarity_${new URL(job.startUrl).hostname.replace(/[^a-z0-9]+/gi, "_").replace(/^_+|_+$/g, "").toLowerCase()}`,
-        defaultSort: [{ column: "similarity", direction: "desc" }],
-        columns: {
-          source_url: "text",
-          source_title: "text",
-          target_url: "text",
-          target_title: "text",
-          similarity: "number",
-          similarity_percent: "number",
-          corpus_percentile: "number",
-          exact_content_duplicate: "boolean",
-          source_word_count: "integer",
-          target_word_count: "integer",
-          cluster_id: "text"
-        }
-      }, null, 2));
-      writeFileSync(join(dir, "similarity-summary.json"), JSON.stringify(summary, null, 2));
-      writeFileSync(join(dir, "content-clusters.json"), JSON.stringify(analysis.clusters, null, 2));
-      reportMd += [
-        "",
-        "## Rendered content similarity",
-        `- Compared pages: ${analysis.comparedPages}`,
-        `- Raw cosine threshold: ${analysis.threshold}`,
-        `- Corpus score distribution: min ${analysis.scoreDistribution.min ?? "n/a"} / median ${analysis.scoreDistribution.median ?? "n/a"} / p90 ${analysis.scoreDistribution.p90 ?? "n/a"} / max ${analysis.scoreDistribution.max ?? "n/a"}`,
-        `- Qualifying pairs: ${analysis.qualifyingPairs}`,
-        `- Returned table rows: ${analysis.returnedPairs}${analysis.pairsTruncated ? " (capped)" : ""}`,
-        `- Multi-page clusters: ${analysis.clusters.length}`,
-        `- Embedding model: ${analysis.model} (${analysis.dimensions} dimensions)`,
-        `- Corpus boilerplate removed: ${analysis.boilerplateRemoval.removedBlockSignatures} repeated block signatures / ${analysis.boilerplateRemoval.removedCharacters} characters`,
-        `- Corpus SHA-256: ${analysis.corpusSha256}`
-      ].join("\n");
-    }
-    writeFileSync(join(dir, "report.md"), reportMd);
-    writeFileSync(join(dir, "crawl-summary.json"), JSON.stringify(crawlSummary, null, 2));
-    writeFileSync(join(dir, "images-manifest.jsonl"), imageManifest.map((record) => JSON.stringify(record)).join("\n"));
-    writeFileSync(join(dir, "manifest.json"), JSON.stringify({
-      version: "site-export.v1",
-      jobId: job.id,
-      startUrl: job.startUrl,
-      generatedAt: crawlSummary.generatedAt,
-      outputProfile: {
-        json: true,
-        html: true,
-        markdown: true,
-        imageManifest: true,
-        imageBytes: downloadImages
-      },
-      pages: [...pageExportEntries].sort((a, b) => a.pageId.localeCompare(b.pageId)),
-      imagesManifestPath: "images-manifest.jsonl",
-      download: "This ZIP is the complete human export. AI clients should use site_export_read and site_export_image."
-    }, null, 2));
-    if (waybackTimeline) {
-      writeFileSync(join(dir, "wayback-manifest.json"), JSON.stringify({
-        rootUrl: waybackTimeline.rootUrl ?? job.startUrl,
-        requested: waybackTimeline.timeline ?? null,
-        maxPagesPerSnapshot: waybackTimeline.maxPagesPerSnapshot ?? null,
-        captureMatrix: "capture-matrix.jsonl"
-      }, null, 2));
-    }
-    writeFileSync(join(dir, "issues.json"), JSON.stringify(issues, null, 2));
-    writeFileSync(join(dir, "link-metrics.jsonl"), metricValues.map((m) => JSON.stringify(m)).join("\n"));
-    writeFileSync(join(dir, "link-report.md"), renderLinkReport(linkReport));
-    writeFileSync(join(dir, "links-summary.json"), JSON.stringify(linkReport.summary, null, 2));
-    writeFileSync(join(dir, "external-domains.json"), JSON.stringify(linkReport.externalDomains, null, 2));
-    if (extras.imageAudit) {
-      writeFileSync(join(dir, "images.jsonl"), extras.imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"));
-      writeFileSync(join(dir, "images-summary.json"), JSON.stringify(extras.imageAudit.summary, null, 2));
-    }
-    if (extras.seoAudit) writeFileSync(join(dir, "seo-audit.json"), JSON.stringify(extras.seoAudit, null, 2));
-    if (extras.branding) writeFileSync(join(dir, "branding.json"), JSON.stringify(extras.branding, null, 2));
-    if (downloadImages) {
-      writeFileSync(join(dir, "images-download-summary.json"), JSON.stringify({
-        queued: imagesQueued,
-        downloaded: imagesQueued - imagesFailed,
-        failed: imagesFailed,
-        downloadedBytes: imageBytesDownloaded,
-        perPageCap: MAX_IMAGES_PER_PAGE,
-        perSiteCap: MAX_IMAGES_PER_SITE,
-        perFileByteCap: MAX_IMAGE_BYTES,
-        totalByteCap: MAX_SITE_IMAGE_BYTES
-      }, null, 2));
-    }
-    const artifactFiles = [
-      { name: "manifest.json", type: "application/json" },
-      { name: "images-manifest.jsonl", type: "application/x-ndjson" },
-      { name: "report.md", type: "text/markdown" },
-      { name: "crawl-summary.json", type: "application/json" },
-      { name: "failures.jsonl", type: "application/x-ndjson" },
-      { name: "schema.jsonl", type: "application/x-ndjson" },
-      { name: "issues.json", type: "application/json" },
-      { name: "pages.jsonl", type: "application/x-ndjson" },
-      { name: "links.jsonl.gz", type: "application/gzip" },
-      { name: "link-metrics.jsonl", type: "application/x-ndjson" },
-      { name: "link-report.md", type: "text/markdown" },
-      { name: "links-summary.json", type: "application/json" },
-      { name: "external-domains.json", type: "application/json" }
-    ];
-    if (captureRenderedDom) {
-      artifactFiles.push({ name: "rendered-dom/manifest.jsonl", type: "application/x-ndjson" });
-      for (const row of renderedDomManifest) artifactFiles.push({ name: row.path, type: "text/plain" });
-    }
-    if (semanticSimilarity) {
-      artifactFiles.push({ name: "similarity-table.csv", type: "text/csv" });
-      artifactFiles.push({ name: "similarity.jsonl", type: "application/x-ndjson" });
-      artifactFiles.push({ name: "similarity-table-schema.json", type: "application/json" });
-      artifactFiles.push({ name: "similarity-summary.json", type: "application/json" });
-      artifactFiles.push({ name: "content-clusters.json", type: "application/json" });
-    }
-    if (waybackTimeline) {
-      artifactFiles.push({ name: "wayback-manifest.json", type: "application/json" });
-      artifactFiles.push({ name: "capture-matrix.jsonl", type: "application/x-ndjson" });
-    }
-    if (extras.imageAudit) {
-      artifactFiles.push({ name: "images.jsonl", type: "application/x-ndjson" });
-      artifactFiles.push({ name: "images-summary.json", type: "application/json" });
-    }
-    if (extras.seoAudit) artifactFiles.push({ name: "seo-audit.json", type: "application/json" });
-    if (extras.branding) artifactFiles.push({ name: "branding.json", type: "application/json" });
-    if (downloadImages) artifactFiles.push({ name: "images-download-summary.json", type: "application/json" });
-    const zip = new ZipFile();
-    const artifactPromise = createSiteExtractBundleArtifactStream({
-      ownerId: String(job.userId),
-      jobId: job.id,
-      // Artifact retention starts when this bundle is assembled, including a
-      // later support re-finalization, rather than when the crawl was queued.
-      createdAt: /* @__PURE__ */ new Date(),
-      content: zip.outputStream
-    });
-    for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name);
-    const orderedPageEntries = [...pageExportEntries].sort((a, b) => a.pageId.localeCompare(b.pageId));
-    for (const entry of orderedPageEntries) if (entry.markdownPath) zip.addFile(join(dir, entry.markdownPath), entry.markdownPath);
-    for (const entry of orderedPageEntries) if (entry.legacyMarkdownPath) zip.addFile(join(dir, entry.legacyMarkdownPath), entry.legacyMarkdownPath);
-    for (const entry of orderedPageEntries) if (entry.htmlPath) zip.addFile(join(dir, entry.htmlPath), entry.htmlPath);
-    for (const entry of orderedPageEntries) zip.addFile(join(dir, entry.jsonPath), entry.jsonPath);
-    if (downloadImages && imagesQueued > imagesFailed) {
-      const imagesDir = join(dir, "images");
-      for (const rel of readdirSync(imagesDir, { recursive: true })) {
-        const full = join(imagesDir, rel);
-        if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split("\\").join("/")}`);
-      }
-    }
-    zip.end();
-    return [await artifactPromise, ...imageArtifacts];
-  } finally {
-    rmSync(dir, { recursive: true, force: true });
-  }
-}
-export {
-  MAX_ANALYZED_LINK_EDGES,
-  SITE_EXTRACT_PAGE_CHUNK,
-  assembleExtractArtifacts
-};