mcp-scraper 0.19.0 → 0.20.0

package/dist/chunk-PZB3TJWK.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.20.0";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-PZB3TJWK.js.map

package/dist/chunk-PZB3TJWK.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.20.0'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{chunk-D7ZT27HY.js → chunk-UN7VMHZL.js}

@@ -1,6 +1,6 @@
 import {
   getDb
-} from "./chunk-GL4BW4CP.js";
+} from "./chunk-22VEGGTW.js";
 
 // src/lib/browser-service-env.ts
 function browserServiceApiKey() {
@@ -241,4 +241,4 @@ export {
   recordKernelSession,
   recordVendorUsage
 };
-//# sourceMappingURL=chunk-D7ZT27HY.js.map
+//# sourceMappingURL=chunk-UN7VMHZL.js.map

package/dist/{chunk-2HDMYW4B.js → chunk-XDFSLSSH.js}

@@ -5,7 +5,7 @@ import {
   currentCostContext,
   recordKernelSession,
   runWithCostContext
-} from "./chunk-D7ZT27HY.js";
+} from "./chunk-UN7VMHZL.js";
 import {
   postToMemoryLibrary
 } from "./chunk-K443GQY5.js";
@@ -3158,4 +3158,4 @@ export {
   resolveKernelProxyId,
   harvest
 };
-//# sourceMappingURL=chunk-2HDMYW4B.js.map
+//# sourceMappingURL=chunk-XDFSLSSH.js.map

package/dist/{chunk-HE2LQPJ2.js → chunk-ZRKFW5FB.js}

@@ -6,7 +6,7 @@ import {
 import {
   finishHarvestAttempt,
   startHarvestAttempt
-} from "./chunk-GL4BW4CP.js";
+} from "./chunk-22VEGGTW.js";
 
 // src/api/harvest-problems.ts
 function errorMessage(err) {
@@ -155,4 +155,4 @@ export {
   harvestProblemResponse,
   createHarvestAttemptRecorder
 };
-//# sourceMappingURL=chunk-HE2LQPJ2.js.map
+//# sourceMappingURL=chunk-ZRKFW5FB.js.map

package/dist/{db-LIOTIWVN.js → db-YHZYG7D2.js}

@@ -99,7 +99,7 @@ import {
   updateKpoJobState,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-GL4BW4CP.js";
+} from "./chunk-22VEGGTW.js";
 export {
   CREDIT_LOT_TTL,
   SiteAuditJobRowSchema,
@@ -202,4 +202,4 @@ export {
   upsertPageSnapshot,
   verifyPassword
 };
-//# sourceMappingURL=db-LIOTIWVN.js.map
+//# sourceMappingURL=db-YHZYG7D2.js.map

package/dist/{extract-bundle-U4D5LW5W.js → extract-bundle-OSUPAHCE.js}

@@ -1,3 +1,6 @@
+import {
+  downloadAsset
+} from "./chunk-JWIE5NCR.js";
 import {
   computeIssues,
   getBlobStore,
@@ -7,16 +10,20 @@ import {
 } from "./chunk-R7EETU7Z.js";
 import {
   getDb
-} from "./chunk-GL4BW4CP.js";
+} from "./chunk-22VEGGTW.js";
 
 // src/api/extract-bundle.ts
-import { createWriteStream, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs";
-import { join } from "path";
+import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
+import { basename, join } from "path";
 import { tmpdir } from "os";
 import { createGzip } from "zlib";
 import { once } from "events";
 import { ZipFile } from "yazl";
+import pLimit from "p-limit";
 var CHUNK = 400;
+var MAX_IMAGES_PER_PAGE = 20;
+var MAX_IMAGES_PER_SITE = 500;
+var IMAGE_DOWNLOAD_CONCURRENCY = 8;
 function normalize(u) {
   return u.split("#")[0].replace(/\/$/, "");
 }
@@ -25,6 +32,15 @@ function registrableDomain(host) {
   const parts = h.split(".");
   return parts.length <= 2 ? h : parts.slice(-2).join(".");
 }
+function safeImageFilename(url, index) {
+  try {
+    const u = new URL(url);
+    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
+    return base || `image-${index}`;
+  } catch {
+    return `image-${index}`;
+  }
+}
 function slugFactory() {
   const counts = /* @__PURE__ */ new Map();
   return (url) => {
@@ -47,7 +63,9 @@ async function* pageChunks(jobId) {
 }
 async function assembleExtractArtifacts(job, extras = {}) {
   const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`);
+  const downloadImages = job.options.downloadImages === true;
   mkdirSync(join(dir, "pages"), { recursive: true });
+  if (downloadImages) mkdirSync(join(dir, "images"), { recursive: true });
   try {
     const metas = [];
     const statusByUrl = /* @__PURE__ */ new Map();
@@ -81,6 +99,10 @@ async function assembleExtractArtifacts(job, extras = {}) {
     } catch {
       siteReg = "";
     }
+    const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY);
+    const imageDownloads = [];
+    let imagesQueued = 0;
+    let imagesFailed = 0;
     for await (const chunk of pageChunks(job.id)) {
       for (const p of chunk) {
         if (p.bodyMarkdown) {
@@ -98,6 +120,20 @@ ${p.bodyMarkdown}
 `
           );
         }
+        if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {
+          const pageDir = join(dir, "images", slug(p.url));
+          mkdirSync(pageDir, { recursive: true });
+          for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {
+            if (imagesQueued >= MAX_IMAGES_PER_SITE) break;
+            imagesQueued++;
+            const filename = safeImageFilename(imgUrl, idx);
+            imageDownloads.push(
+              imageLimit(() => downloadAsset(imgUrl, pageDir, filename)).then(() => void 0, () => {
+                imagesFailed++;
+              })
+            );
+          }
+        }
         const from = normalize(p.url);
         const oc = { int: 0, ext: 0 };
         for (const l of p.outlinks ?? []) {
@@ -145,6 +181,7 @@ ${p.bodyMarkdown}
     }
     gz.end();
     await once(linksOut, "finish");
+    await Promise.all(imageDownloads);
     const depth = /* @__PURE__ */ new Map();
     const start = normalize(job.startUrl);
     depth.set(start, 0);
@@ -236,6 +273,15 @@ ${renderImageSection(extras.imageAudit)}`;
     }
     if (extras.seoAudit) writeFileSync(join(dir, "seo-audit.json"), JSON.stringify(extras.seoAudit, null, 2));
     if (extras.branding) writeFileSync(join(dir, "branding.json"), JSON.stringify(extras.branding, null, 2));
+    if (downloadImages) {
+      writeFileSync(join(dir, "images-download-summary.json"), JSON.stringify({
+        queued: imagesQueued,
+        downloaded: imagesQueued - imagesFailed,
+        failed: imagesFailed,
+        perPageCap: MAX_IMAGES_PER_PAGE,
+        perSiteCap: MAX_IMAGES_PER_SITE
+      }, null, 2));
+    }
     const artifactFiles = [
       { name: "report.md", type: "text/markdown" },
       { name: "issues.json", type: "application/json" },
@@ -252,14 +298,21 @@ ${renderImageSection(extras.imageAudit)}`;
     }
     if (extras.seoAudit) artifactFiles.push({ name: "seo-audit.json", type: "application/json" });
     if (extras.branding) artifactFiles.push({ name: "branding.json", type: "application/json" });
+    if (downloadImages) artifactFiles.push({ name: "images-download-summary.json", type: "application/json" });
     const zip = new ZipFile();
     const zipOut = createWriteStream(join(dir, "bundle.zip"));
     zip.outputStream.pipe(zipOut);
     for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name);
     if (pagesWithBody > 0) {
-      const { readdirSync } = await import("fs");
       for (const f of readdirSync(join(dir, "pages"))) zip.addFile(join(dir, "pages", f), `pages/${f}`);
     }
+    if (downloadImages && imagesQueued > imagesFailed) {
+      const imagesDir = join(dir, "images");
+      for (const rel of readdirSync(imagesDir, { recursive: true })) {
+        const full = join(imagesDir, rel);
+        if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split("\\").join("/")}`);
+      }
+    }
     zip.end();
     await once(zipOut, "finish");
     const store = getBlobStore();
@@ -275,4 +328,4 @@ ${renderImageSection(extras.imageAudit)}`;
 export {
   assembleExtractArtifacts
 };
-//# sourceMappingURL=extract-bundle-U4D5LW5W.js.map
+//# sourceMappingURL=extract-bundle-OSUPAHCE.js.map

package/dist/extract-bundle-OSUPAHCE.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/api/extract-bundle.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'\nimport { basename, join } from 'node:path'\nimport { tmpdir } from 'node:os'\nimport { createGzip } from 'node:zlib'\nimport { once } from 'node:events'\nimport { ZipFile } from 'yazl'\nimport pLimit from 'p-limit'\nimport { getDb } from './db.js'\nimport { getBlobStore, type StoredArtifact } from './blob-store.js'\nimport { computeIssues, renderIssueReport } from './seo-issues.js'\nimport { renderLinkReport, type ExtDomainRow, type LinkReport } from './seo-link-report.js'\nimport type { PageLinkMetrics } from './seo-link-graph.js'\nimport type { SeoAudit } from './seo-audit.js'\nimport type { ExtractJobRow } from './site-extract-repository.js'\nimport type { PageData } from './site-extractor.js'\nimport { renderImageSection, type ImageAudit } from './image-audit.js'\nimport { downloadAsset } from '../lib/media-extractor.js'\n\nconst CHUNK = 400\nconst MAX_IMAGES_PER_PAGE = 20\nconst MAX_IMAGES_PER_SITE = 500\nconst IMAGE_DOWNLOAD_CONCURRENCY = 8\n\nfunction normalize(u: string): string {\n  return u.split('#')[0].replace(/\\/$/, '')\n}\n\nfunction registrableDomain(host: string): string {\n  const h = host.replace(/^www\\./, '').toLowerCase()\n  const parts = h.split('.')\n  return parts.length <= 2 ? h : parts.slice(-2).join('.')\n}\n\nfunction safeImageFilename(url: string, index: number): string {\n  try {\n    const u = new URL(url)\n    const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n    return base || `image-${index}`\n  } catch {\n    return `image-${index}`\n  }\n}\n\nfunction slugFactory() {\n  const counts = new Map<string, number>()\n  return (url: string): string => {\n    const base = url.replace(/^https?:\\/\\//, '').replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80) || 'page'\n    const n = counts.get(base) ?? 0\n    counts.set(base, n + 1)\n    return n ? `${base}-${n}` : base\n  }\n}\n\nasync function* pageChunks(jobId: string): AsyncGenerator<PageData[]> {\n  const db = getDb()\n  for (let offset = 0; ; offset += CHUNK) {\n    const res = await db.execute({\n      sql: `SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid LIMIT ${CHUNK} OFFSET ?`,\n      args: [jobId, offset],\n    })\n    if (!res.rows.length) return\n    yield res.rows.map(r => JSON.parse(String((r as unknown as Record<string, unknown>).page)) as PageData)\n  }\n}\n\nexport interface AssembleExtras {\n  branding?: unknown\n  imageAudit?: ImageAudit | null\n  seoAudit?: SeoAudit | null\n}\n\nexport async function assembleExtractArtifacts(job: ExtractJobRow, extras: AssembleExtras = {}): Promise<StoredArtifact[]> {\n  const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`)\n  const downloadImages = job.options.downloadImages === true\n  mkdirSync(join(dir, 'pages'), { recursive: true })\n  if (downloadImages) mkdirSync(join(dir, 'images'), { recursive: true })\n  try {\n    const metas: PageData[] = []\n    const statusByUrl = new Map<string, number | null>()\n    for await (const chunk of pageChunks(job.id)) {\n      for (const p of chunk) {\n        statusByUrl.set(normalize(p.url), p.status)\n        const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...slim } = p\n        metas.push(slim as unknown as PageData)\n      }\n    }\n\n    const gz = createGzip()\n    const linksOut = createWriteStream(join(dir, 'links.jsonl.gz'))\n    gz.pipe(linksOut)\n    const gzWrite = async (line: string) => { if (!gz.write(line)) await once(gz, 'drain') }\n\n    const slug = slugFactory()\n    const inboundFrom = new Map<string, Set<string>>()\n    const anchorCounts = new Map<string, Map<string, number>>()\n    const adjacency = new Map<string, Set<string>>()\n    const outCounts = new Map<string, { int: number; ext: number }>()\n    const domMap = new Map<string, { links: number; nofollow: number; pages: Set<string> }>()\n    const brokenLinkPages = new Set<string>()\n    let internalTotal = 0\n    let externalTotal = 0\n    let brokenInternal = 0\n    let pagesWithBody = 0\n    let siteReg = ''\n    try { siteReg = registrableDomain(new URL(job.startUrl).hostname) } catch { siteReg = '' }\n\n    const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY)\n    const imageDownloads: Promise<void>[] = []\n    let imagesQueued = 0\n    let imagesFailed = 0\n\n    for await (const chunk of pageChunks(job.id)) {\n      for (const p of chunk) {\n        if (p.bodyMarkdown) {\n          pagesWithBody++\n          writeFileSync(join(dir, 'pages', slug(p.url) + '.md'),\n            `# ${p.title ?? p.url}\\n\\nURL: ${p.url}\\nStatus: ${p.status}\\n\\n---\\n\\n${p.bodyMarkdown}\\n`)\n        }\n        if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {\n          const pageDir = join(dir, 'images', slug(p.url))\n          mkdirSync(pageDir, { recursive: true })\n          for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {\n            if (imagesQueued >= MAX_IMAGES_PER_SITE) break\n            imagesQueued++\n            const filename = safeImageFilename(imgUrl, idx)\n            imageDownloads.push(\n              imageLimit(() => downloadAsset(imgUrl, pageDir, filename)).then(() => undefined, () => { imagesFailed++ }),\n            )\n          }\n        }\n        const from = normalize(p.url)\n        const oc = { int: 0, ext: 0 }\n        for (const l of p.outlinks ?? []) {\n          const nofollow = (l.rel ?? '').split(/\\s+/).includes('nofollow')\n          const to = normalize(l.href)\n          const targetStatus = statusByUrl.get(to) ?? null\n          await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + '\\n')\n          if (l.internal) {\n            oc.int++\n            internalTotal++\n            if (!inboundFrom.has(to)) inboundFrom.set(to, new Set())\n            inboundFrom.get(to)!.add(from)\n            if (l.anchor) {\n              if (!anchorCounts.has(to)) anchorCounts.set(to, new Map())\n              const ac = anchorCounts.get(to)!\n              ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1)\n            }\n            if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {\n              if (!adjacency.has(from)) adjacency.set(from, new Set())\n              adjacency.get(from)!.add(to)\n            }\n            if (targetStatus != null && targetStatus >= 400) {\n              brokenInternal++\n              brokenLinkPages.add(p.url)\n            }\n          } else {\n            oc.ext++\n            let domain = ''\n            try { domain = registrableDomain(new URL(l.href).hostname) } catch { continue }\n            if (!domain || domain === siteReg) continue\n            externalTotal++\n            const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: new Set<string>() }\n            d.links++\n            if (nofollow) d.nofollow++\n            d.pages.add(p.url)\n            domMap.set(domain, d)\n          }\n        }\n        outCounts.set(p.url, oc)\n      }\n    }\n    gz.end()\n    await once(linksOut, 'finish')\n    await Promise.all(imageDownloads)\n\n    const depth = new Map<string, number>()\n    const start = normalize(job.startUrl)\n    depth.set(start, 0)\n    const queue: string[] = [start]\n    while (queue.length > 0) {\n      const cur = queue.shift()!\n      const d = depth.get(cur)!\n      for (const next of adjacency.get(cur) ?? []) {\n        if (!depth.has(next)) { depth.set(next, d + 1); queue.push(next) }\n      }\n    }\n\n    const metrics = new Map<string, PageLinkMetrics>()\n    for (const p of metas) {\n      const key = normalize(p.url)\n      const inbound = inboundFrom.get(key) ?? new Set()\n      const ac = anchorCounts.get(key)\n      const topAnchors = ac\n        ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(e => e[0])\n        : []\n      const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 }\n      metrics.set(p.url, {\n        url: p.url,\n        inlinks: inbound.size,\n        uniqueInlinks: inbound.size,\n        outlinksInternal: oc.int,\n        outlinksExternal: oc.ext,\n        crawlDepth: depth.has(key) ? depth.get(key)! : null,\n        orphan: inbound.size === 0 && key !== start,\n        topAnchors,\n      })\n    }\n\n    const issues = computeIssues(metas, metrics, brokenLinkPages)\n    let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics)\n    if (extras.imageAudit) reportMd += `\\n\\n${renderImageSection(extras.imageAudit)}`\n\n    const metricValues = [...metrics.values()]\n    const round = (n: number) => Math.round(n * 10) / 10\n    const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 }\n    let sumInlinks = 0\n    let sumOutlinks = 0\n    for (const m of metricValues) {\n      sumInlinks += m.inlinks\n      sumOutlinks += m.outlinksInternal + m.outlinksExternal\n      if (m.inlinks === 0) distribution.zero++\n      else if (m.inlinks <= 2) distribution.oneToTwo++\n      else if (m.inlinks <= 10) distribution.threeToTen++\n      else distribution.elevenPlus++\n    }\n    const externalDomains: ExtDomainRow[] = [...domMap.entries()]\n      .map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size }))\n      .sort((a, b) => b.links - a.links)\n    const linkReport: LinkReport = {\n      summary: {\n        internal: {\n          totalLinks: internalTotal,\n          pages: metas.length,\n          orphans: metricValues.filter(m => m.orphan).length,\n          brokenInternal,\n          avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,\n          avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,\n          distribution,\n          topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20)\n            .map(m => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal })),\n        },\n        external: {\n          totalLinks: externalTotal,\n          uniqueDomains: externalDomains.length,\n          topDomains: externalDomains.slice(0, 20),\n        },\n      },\n      externalDomains,\n    }\n\n    const pagesOut = createWriteStream(join(dir, 'pages.jsonl'))\n    for (const p of metas) {\n      const m = metrics.get(p.url)\n      const row = { ...(p as unknown as Record<string, unknown>), inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false }\n      if (!pagesOut.write(JSON.stringify(row) + '\\n')) await once(pagesOut, 'drain')\n    }\n    pagesOut.end()\n    await once(pagesOut, 'finish')\n\n    writeFileSync(join(dir, 'report.md'), reportMd)\n    writeFileSync(join(dir, 'issues.json'), JSON.stringify(issues, null, 2))\n    writeFileSync(join(dir, 'link-metrics.jsonl'), metricValues.map(m => JSON.stringify(m)).join('\\n'))\n    writeFileSync(join(dir, 'link-report.md'), renderLinkReport(linkReport))\n    writeFileSync(join(dir, 'links-summary.json'), JSON.stringify(linkReport.summary, null, 2))\n    writeFileSync(join(dir, 'external-domains.json'), JSON.stringify(linkReport.externalDomains, null, 2))\n    if (extras.imageAudit) {\n      writeFileSync(join(dir, 'images.jsonl'), extras.imageAudit.rows.map(r => JSON.stringify(r)).join('\\n'))\n      writeFileSync(join(dir, 'images-summary.json'), JSON.stringify(extras.imageAudit.summary, null, 2))\n    }\n    if (extras.seoAudit) writeFileSync(join(dir, 'seo-audit.json'), JSON.stringify(extras.seoAudit, null, 2))\n    if (extras.branding) writeFileSync(join(dir, 'branding.json'), JSON.stringify(extras.branding, null, 2))\n    if (downloadImages) {\n      writeFileSync(join(dir, 'images-download-summary.json'), JSON.stringify({\n        queued: imagesQueued,\n        downloaded: imagesQueued - imagesFailed,\n        failed: imagesFailed,\n        perPageCap: MAX_IMAGES_PER_PAGE,\n        perSiteCap: MAX_IMAGES_PER_SITE,\n      }, null, 2))\n    }\n\n    const artifactFiles: Array<{ name: string; type: string }> = [\n      { name: 'report.md', type: 'text/markdown' },\n      { name: 'issues.json', type: 'application/json' },\n      { name: 'pages.jsonl', type: 'application/x-ndjson' },\n      { name: 'links.jsonl.gz', type: 'application/gzip' },\n      { name: 'link-metrics.jsonl', type: 'application/x-ndjson' },\n      { name: 'link-report.md', type: 'text/markdown' },\n      { name: 'links-summary.json', type: 'application/json' },\n      { name: 'external-domains.json', type: 'application/json' },\n    ]\n    if (extras.imageAudit) {\n      artifactFiles.push({ name: 'images.jsonl', type: 'application/x-ndjson' })\n      artifactFiles.push({ name: 'images-summary.json', type: 'application/json' })\n    }\n    if (extras.seoAudit) artifactFiles.push({ name: 'seo-audit.json', type: 'application/json' })\n    if (extras.branding) artifactFiles.push({ name: 'branding.json', type: 'application/json' })\n    if (downloadImages) artifactFiles.push({ name: 'images-download-summary.json', type: 'application/json' })\n\n    const zip = new ZipFile()\n    const zipOut = createWriteStream(join(dir, 'bundle.zip'))\n    zip.outputStream.pipe(zipOut)\n    for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name)\n    if (pagesWithBody > 0) {\n      for (const f of readdirSync(join(dir, 'pages'))) zip.addFile(join(dir, 'pages', f), `pages/${f}`)\n    }\n    if (downloadImages && imagesQueued > imagesFailed) {\n      const imagesDir = join(dir, 'images')\n      for (const rel of readdirSync(imagesDir, { recursive: true }) as string[]) {\n        const full = join(imagesDir, rel)\n        if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split('\\\\').join('/')}`)\n      }\n    }\n    zip.end()\n    await once(zipOut, 'finish')\n\n    const store = getBlobStore()\n    const stored: StoredArtifact[] = []\n    for (const f of [...artifactFiles, { name: 'bundle.zip', type: 'application/zip' }]) {\n      stored.push(await store.put(`extract/${job.id}/${f.name}`, readFileSync(join(dir, f.name)), f.type))\n    }\n    return stored\n  } finally {\n    rmSync(dir, { recursive: true, force: true })\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc,aAAa,QAAQ,UAAU,qBAAqB;AACzG,SAAS,UAAU,YAAY;AAC/B,SAAS,cAAc;AACvB,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AACrB,SAAS,eAAe;AACxB,OAAO,YAAY;AAYnB,IAAM,QAAQ;AACd,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAC5B,IAAM,6BAA6B;AAEnC,SAAS,UAAU,GAAmB;AACpC,SAAO,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,OAAO,EAAE;AAC1C;AAEA,SAAS,kBAAkB,MAAsB;AAC/C,QAAM,IAAI,KAAK,QAAQ,UAAU,EAAE,EAAE,YAAY;AACjD,QAAM,QAAQ,EAAE,MAAM,GAAG;AACzB,SAAO,MAAM,UAAU,IAAI,IAAI,MAAM,MAAM,EAAE,EAAE,KAAK,GAAG;AACzD;AAEA,SAAS,kBAAkB,KAAa,OAAuB;AAC7D,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEA,SAAS,cAAc;AACrB,QAAM,SAAS,oBAAI,IAAoB;AACvC,SAAO,CAAC,QAAwB;AAC9B,UAAM,OAAO,IAAI,QAAQ,gBAAgB,EAAE,EAAE,QAAQ,kBAAkB,GAAG,EAAE,QAAQ,YAAY,EAAE,EAAE,MAAM,GAAG,EAAE,KAAK;AACpH,UAAM,IAAI,OAAO,IAAI,IAAI,KAAK;AAC9B,WAAO,IAAI,MAAM,IAAI,CAAC;AACtB,WAAO,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK;AAAA,EAC9B;AACF;AAEA,gBAAgB,WAAW,OAA2C;AACpE,QAAM,KAAK,MAAM;AACjB,WAAS,SAAS,KAAK,UAAU,OAAO;AACtC,UAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,MAC3B,KAAK,6EAA6E,KAAK;AAAA,MACvF,MAAM,CAAC,OAAO,MAAM;AAAA,IACtB,CAAC;AACD,QAAI,CAAC,IAAI,KAAK,OAAQ;AACtB,UAAM,IAAI,KAAK,IAAI,OAAK,KAAK,MAAM,OAAQ,EAAyC,IAAI,CAAC,CAAa;AAAA,EACxG;AACF;AAQA,eAAsB,yBAAyB,KAAoB,SAAyB,CAAC,GAA8B;AACzH,QAAM,MAAM,KAAK,OAAO,GAAG,kBAAkB,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AACnE,QAAM,iBAAiB,IAAI,QAAQ,mBAAmB;AACtD,YAAU,KAAK,KAAK,OAAO,GAAG,EAAE,WAAW,KAAK,CAAC;AACjD,MAAI,eAAgB,WAAU,KAAK,KAAK,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;AACtE,MAAI;AACF,UAAM,QAAoB,CAAC;AAC3B,UAAM,cAAc,oBAAI,IAA2B;AACnD,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,oBAAY,IAAI,UAAU,EAAE,GAAG,GAAG,EAAE,MAAM;AAC1C,cAAM,EAAE,cAAc,IAAI,QAAQ,IAAI,UAAU,IAAI,GAAG,KAAK,IAAI;AAChE,cAAM,KAAK,IAA2B;AAAA,MACxC;AAAA,IACF;AAEA,UAAM,KAAK,WAAW;AACtB,UAAM,WAAW,kBAAkB,KAAK,KAAK,gBAAgB,CAAC;AAC9D,OAAG,KAAK,QAAQ;AAChB,UAAM,UAAU,OAAO,SAAiB;AAAE,UAAI,CAAC,GAAG,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI,OAAO;AAAA,IAAE;AAEvF,UAAM,OAAO,YAAY;AACzB,UAAM,cAAc,oBAAI,IAAyB;AACjD,UAAM,eAAe,oBAAI,IAAiC;AAC1D,UAAM,YAAY,oBAAI,IAAyB;AAC/C,UAAM,YAAY,oBAAI,IAA0C;AAChE,UAAM,SAAS,oBAAI,IAAqE;AACxF,UAAM,kBAAkB,oBAAI,IAAY;AACxC,QAAI,gBAAgB;AACpB,QAAI,gBAAgB;AACpB,QAAI,iBAAiB;AACrB,QAAI,gBAAgB;AACpB,QAAI,UAAU;AACd,QAAI;AAAE,gBAAU,kBAAkB,IAAI,IAAI,IAAI,QAAQ,EAAE,QAAQ;AAAA,IAAE,QAAQ;AAAE,gBAAU;AAAA,IAAG;AAEzF,UAAM,aAAa,OAAO,0BAA0B;AACpD,UAAM,iBAAkC,CAAC;AACzC,QAAI,eAAe;AACnB,QAAI,eAAe;AAEnB,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,YAAI,EAAE,cAAc;AAClB;AACA;AAAA,YAAc,KAAK,KAAK,SAAS,KAAK,EAAE,GAAG,IAAI,KAAK;AAAA,YAClD,KAAK,EAAE,SAAS,EAAE,GAAG;AAAA;AAAA,OAAY,EAAE,GAAG;AAAA,UAAa,EAAE,MAAM;AAAA;AAAA;AAAA;AAAA,EAAc,EAAE,YAAY;AAAA;AAAA,UAAI;AAAA,QAC/F;AACA,YAAI,kBAAkB,EAAE,YAAY,UAAU,eAAe,qBAAqB;AAChF,gBAAM,UAAU,KAAK,KAAK,UAAU,KAAK,EAAE,GAAG,CAAC;AAC/C,oBAAU,SAAS,EAAE,WAAW,KAAK,CAAC;AACtC,qBAAW,CAAC,KAAK,MAAM,KAAK,EAAE,WAAW,MAAM,GAAG,mBAAmB,EAAE,QAAQ,GAAG;AAChF,gBAAI,gBAAgB,oBAAqB;AACzC;AACA,kBAAM,WAAW,kBAAkB,QAAQ,GAAG;AAC9C,2BAAe;AAAA,cACb,WAAW,MAAM,cAAc,QAAQ,SAAS,QAAQ,CAAC,EAAE,KAAK,MAAM,QAAW,MAAM;AAAE;AAAA,cAAe,CAAC;AAAA,YAC3G;AAAA,UACF;AAAA,QACF;AACA,cAAM,OAAO,UAAU,EAAE,GAAG;AAC5B,cAAM,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AAC5B,mBAAW,KAAK,EAAE,YAAY,CAAC,GAAG;AAChC,gBAAM,YAAY,EAAE,OAAO,IAAI,MAAM,KAAK,EAAE,SAAS,UAAU;AAC/D,gBAAM,KAAK,UAAU,EAAE,IAAI;AAC3B,gBAAM,eAAe,YAAY,IAAI,EAAE,KAAK;AAC5C,gBAAM,QAAQ,KAAK,UAAU,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,UAAU,EAAE,UAAU,UAAU,aAAa,CAAC,IAAI,IAAI;AAC5I,cAAI,EAAE,UAAU;AACd,eAAG;AACH;AACA,gBAAI,CAAC,YAAY,IAAI,EAAE,EAAG,aAAY,IAAI,IAAI,oBAAI,IAAI,CAAC;AACvD,wBAAY,IAAI,EAAE,EAAG,IAAI,IAAI;AAC7B,gBAAI,EAAE,QAAQ;AACZ,kBAAI,CAAC,aAAa,IAAI,EAAE,EAAG,cAAa,IAAI,IAAI,oBAAI,IAAI,CAAC;AACzD,oBAAM,KAAK,aAAa,IAAI,EAAE;AAC9B,iBAAG,IAAI,EAAE,SAAS,GAAG,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC;AAAA,YAC9C;AACA,gBAAI,CAAC,aAAa,YAAY,IAAI,EAAE,MAAM,OAAO,CAAC,YAAY,IAAI,EAAE,IAAI;AACtE,kBAAI,CAAC,UAAU,IAAI,IAAI,EAAG,WAAU,IAAI,MAAM,oBAAI,IAAI,CAAC;AACvD,wBAAU,IAAI,IAAI,EAAG,IAAI,EAAE;AAAA,YAC7B;AACA,gBAAI,gBAAgB,QAAQ,gBAAgB,KAAK;AAC/C;AACA,8BAAgB,IAAI,EAAE,GAAG;AAAA,YAC3B;AAAA,UACF,OAAO;AACL,eAAG;AACH,gBAAI,SAAS;AACb,gBAAI;AAAE,uBAAS,kBAAkB,IAAI,IAAI,EAAE,IAAI,EAAE,QAAQ;AAAA,YAAE,QAAQ;AAAE;AAAA,YAAS;AAC9E,gBAAI,CAAC,UAAU,WAAW,QAAS;AACnC;AACA,kBAAM,IAAI,OAAO,IAAI,MAAM,KAAK,EAAE,OAAO,GAAG,UAAU,GAAG,OAAO,oBAAI,IAAY,EAAE;AAClF,cAAE;AACF,gBAAI,SAAU,GAAE;AAChB,cAAE,MAAM,IAAI,EAAE,GAAG;AACjB,mBAAO,IAAI,QAAQ,CAAC;AAAA,UACtB;AAAA,QACF;AACA,kBAAU,IAAI,EAAE,KAAK,EAAE;AAAA,MACzB;AAAA,IACF;AACA,OAAG,IAAI;AACP,UAAM,KAAK,UAAU,QAAQ;AAC7B,UAAM,QAAQ,IAAI,cAAc;AAEhC,UAAM,QAAQ,oBAAI,IAAoB;AACtC,UAAM,QAAQ,UAAU,IAAI,QAAQ;AACpC,UAAM,IAAI,OAAO,CAAC;AAClB,UAAM,QAAkB,CAAC,KAAK;AAC9B,WAAO,MAAM,SAAS,GAAG;AACvB,YAAM,MAAM,MAAM,MAAM;AACxB,YAAM,IAAI,MAAM,IAAI,GAAG;AACvB,iBAAW,QAAQ,UAAU,IAAI,GAAG,KAAK,CAAC,GAAG;AAC3C,YAAI,CAAC,MAAM,IAAI,IAAI,GAAG;AAAE,gBAAM,IAAI,MAAM,IAAI,CAAC;AAAG,gBAAM,KAAK,IAAI;AAAA,QAAE;AAAA,MACnE;AAAA,IACF;AAEA,UAAM,UAAU,oBAAI,IAA6B;AACjD,eAAW,KAAK,OAAO;AACrB,YAAM,MAAM,UAAU,EAAE,GAAG;AAC3B,YAAM,UAAU,YAAY,IAAI,GAAG,KAAK,oBAAI,IAAI;AAChD,YAAM,KAAK,aAAa,IAAI,GAAG;AAC/B,YAAM,aAAa,KACf,CAAC,GAAG,GAAG,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,IAAI,OAAK,EAAE,CAAC,CAAC,IACvE,CAAC;AACL,YAAM,KAAK,UAAU,IAAI,EAAE,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AACpD,cAAQ,IAAI,EAAE,KAAK;AAAA,QACjB,KAAK,EAAE;AAAA,QACP,SAAS,QAAQ;AAAA,QACjB,eAAe,QAAQ;AAAA,QACvB,kBAAkB,GAAG;AAAA,QACrB,kBAAkB,GAAG;AAAA,QACrB,YAAY,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,IAAK;AAAA,QAC/C,QAAQ,QAAQ,SAAS,KAAK,QAAQ;AAAA,QACtC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,cAAc,OAAO,SAAS,eAAe;AAC5D,QAAI,WAAW,kBAAkB,IAAI,UAAU,OAAO,QAAQ,OAAO;AACrE,QAAI,OAAO,WAAY,aAAY;AAAA;AAAA,EAAO,mBAAmB,OAAO,UAAU,CAAC;AAE/E,UAAM,eAAe,CAAC,GAAG,QAAQ,OAAO,CAAC;AACzC,UAAM,QAAQ,CAAC,MAAc,KAAK,MAAM,IAAI,EAAE,IAAI;AAClD,UAAM,eAAe,EAAE,MAAM,GAAG,UAAU,GAAG,YAAY,GAAG,YAAY,EAAE;AAC1E,QAAI,aAAa;AACjB,QAAI,cAAc;AAClB,eAAW,KAAK,cAAc;AAC5B,oBAAc,EAAE;AAChB,qBAAe,EAAE,mBAAmB,EAAE;AACtC,UAAI,EAAE,YAAY,EAAG,cAAa;AAAA,eACzB,EAAE,WAAW,EAAG,cAAa;AAAA,eAC7B,EAAE,WAAW,GAAI,cAAa;AAAA,UAClC,cAAa;AAAA,IACpB;AACA,UAAM,kBAAkC,CAAC,GAAG,OAAO,QAAQ,CAAC,EACzD,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,OAAO,EAAE,MAAM,KAAK,EAAE,EAC5F,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACnC,UAAM,aAAyB;AAAA,MAC7B,SAAS;AAAA,QACP,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,OAAO,MAAM;AAAA,UACb,SAAS,aAAa,OAAO,OAAK,EAAE,MAAM,EAAE;AAAA,UAC5C;AAAA,UACA,YAAY,MAAM,SAAS,MAAM,aAAa,MAAM,MAAM,IAAI;AAAA,UAC9D,aAAa,MAAM,SAAS,MAAM,cAAc,MAAM,MAAM,IAAI;AAAA,UAChE;AAAA,UACA,cAAc,CAAC,GAAG,YAAY,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,EAAE,EAC9E,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,SAAS,EAAE,SAAS,kBAAkB,EAAE,kBAAkB,kBAAkB,EAAE,iBAAiB,EAAE;AAAA,QAC9H;AAAA,QACA,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,eAAe,gBAAgB;AAAA,UAC/B,YAAY,gBAAgB,MAAM,GAAG,EAAE;AAAA,QACzC;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,UAAM,WAAW,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAC3D,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,IAAI,EAAE,GAAG;AAC3B,YAAM,MAAM,EAAE,GAAI,GAA0C,SAAS,GAAG,WAAW,GAAG,YAAY,GAAG,cAAc,MAAM,QAAQ,GAAG,UAAU,MAAM;AACpJ,UAAI,CAAC,SAAS,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,EAAG,OAAM,KAAK,UAAU,OAAO;AAAA,IAC/E;AACA,aAAS,IAAI;AACb,UAAM,KAAK,UAAU,QAAQ;AAE7B,kBAAc,KAAK,KAAK,WAAW,GAAG,QAAQ;AAC9C,kBAAc,KAAK,KAAK,aAAa,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,aAAa,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAClG,kBAAc,KAAK,KAAK,gBAAgB,GAAG,iBAAiB,UAAU,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,KAAK,UAAU,WAAW,SAAS,MAAM,CAAC,CAAC;AAC1F,kBAAc,KAAK,KAAK,uBAAuB,GAAG,KAAK,UAAU,WAAW,iBAAiB,MAAM,CAAC,CAAC;AACrG,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,KAAK,cAAc,GAAG,OAAO,WAAW,KAAK,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACtG,oBAAc,KAAK,KAAK,qBAAqB,GAAG,KAAK,UAAU,OAAO,WAAW,SAAS,MAAM,CAAC,CAAC;AAAA,IACpG;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,gBAAgB,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACxG,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,eAAe,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACvG,QAAI,gBAAgB;AAClB,oBAAc,KAAK,KAAK,8BAA8B,GAAG,KAAK,UAAU;AAAA,QACtE,QAAQ;AAAA,QACR,YAAY,eAAe;AAAA,QAC3B,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,YAAY;AAAA,MACd,GAAG,MAAM,CAAC,CAAC;AAAA,IACb;AAEA,UAAM,gBAAuD;AAAA,MAC3D,EAAE,MAAM,aAAa,MAAM,gBAAgB;AAAA,MAC3C,EAAE,MAAM,eAAe,MAAM,mBAAmB;AAAA,MAChD,EAAE,MAAM,eAAe,MAAM,uBAAuB;AAAA,MACpD,EAAE,MAAM,kBAAkB,MAAM,mBAAmB;AAAA,MACnD,EAAE,MAAM,sBAAsB,MAAM,uBAAuB;AAAA,MAC3D,EAAE,MAAM,kBAAkB,MAAM,gBAAgB;AAAA,MAChD,EAAE,MAAM,sBAAsB,MAAM,mBAAmB;AAAA,MACvD,EAAE,MAAM,yBAAyB,MAAM,mBAAmB;AAAA,IAC5D;AACA,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,EAAE,MAAM,gBAAgB,MAAM,uBAAuB,CAAC;AACzE,oBAAc,KAAK,EAAE,MAAM,uBAAuB,MAAM,mBAAmB,CAAC;AAAA,IAC9E;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,kBAAkB,MAAM,mBAAmB,CAAC;AAC5F,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,iBAAiB,MAAM,mBAAmB,CAAC;AAC3F,QAAI,eAAgB,eAAc,KAAK,EAAE,MAAM,gCAAgC,MAAM,mBAAmB,CAAC;AAEzG,UAAM,MAAM,IAAI,QAAQ;AACxB,UAAM,SAAS,kBAAkB,KAAK,KAAK,YAAY,CAAC;AACxD,QAAI,aAAa,KAAK,MAAM;AAC5B,eAAW,KAAK,cAAe,KAAI,QAAQ,KAAK,KAAK,EAAE,IAAI,GAAG,EAAE,IAAI;AACpE,QAAI,gBAAgB,GAAG;AACrB,iBAAW,KAAK,YAAY,KAAK,KAAK,OAAO,CAAC,EAAG,KAAI,QAAQ,KAAK,KAAK,SAAS,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAClG;AACA,QAAI,kBAAkB,eAAe,cAAc;AACjD,YAAM,YAAY,KAAK,KAAK,QAAQ;AACpC,iBAAW,OAAO,YAAY,WAAW,EAAE,WAAW,KAAK,CAAC,GAAe;AACzE,cAAM,OAAO,KAAK,WAAW,GAAG;AAChC,YAAI,SAAS,IAAI,EAAE,OAAO,EAAG,KAAI,QAAQ,MAAM,UAAU,IAAI,MAAM,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE;AAAA,MACtF;AAAA,IACF;AACA,QAAI,IAAI;AACR,UAAM,KAAK,QAAQ,QAAQ;AAE3B,UAAM,QAAQ,aAAa;AAC3B,UAAM,SAA2B,CAAC;AAClC,eAAW,KAAK,CAAC,GAAG,eAAe,EAAE,MAAM,cAAc,MAAM,kBAAkB,CAAC,GAAG;AACnF,aAAO,KAAK,MAAM,MAAM,IAAI,WAAW,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,aAAa,KAAK,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC;AAAA,IACrG;AACA,WAAO;AAAA,EACT,UAAE;AACA,WAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAAA,EAC9C;AACF;","names":[]}