mcp-scraper 0.34.1 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +10 -9
  2. package/dist/bin/api-server.cjs +24342 -17931
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +51 -7
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +48 -5
  8. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs +2 -2
  10. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-install.js +2 -2
  12. package/dist/bin/mcp-stdio-server.cjs +2997 -2070
  13. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-stdio-server.js +8 -8
  15. package/dist/bin/paa-harvest.cjs +125 -70
  16. package/dist/bin/paa-harvest.cjs.map +1 -1
  17. package/dist/bin/paa-harvest.js +4 -4
  18. package/dist/chunk-345BQXZH.js +712 -0
  19. package/dist/chunk-345BQXZH.js.map +1 -0
  20. package/dist/{chunk-M2S27J6Z.js → chunk-44HZLHDV.js} +10 -1
  21. package/dist/chunk-44HZLHDV.js.map +1 -0
  22. package/dist/chunk-4HO66323.js +7 -0
  23. package/dist/chunk-4HO66323.js.map +1 -0
  24. package/dist/{chunk-BWXLTWF7.js → chunk-4ZIJ3BKZ.js} +6 -4
  25. package/dist/chunk-4ZIJ3BKZ.js.map +1 -0
  26. package/dist/{chunk-6OPHG76G.js → chunk-5RULXBJ7.js} +3166 -2322
  27. package/dist/chunk-5RULXBJ7.js.map +1 -0
  28. package/dist/chunk-AN3VQARU.js +684 -0
  29. package/dist/chunk-AN3VQARU.js.map +1 -0
  30. package/dist/{chunk-XVVNKASZ.js → chunk-ANCGXUQJ.js} +118 -73
  31. package/dist/chunk-ANCGXUQJ.js.map +1 -0
  32. package/dist/{chunk-3HBPKR5G.js → chunk-D7LM5QZN.js} +3 -3
  33. package/dist/{chunk-4ZB3X6BQ.js → chunk-E5UEELA7.js} +16 -2
  34. package/dist/{chunk-4ZB3X6BQ.js.map → chunk-E5UEELA7.js.map} +1 -1
  35. package/dist/{chunk-ZID3WQID.js → chunk-FQI5PFE7.js} +9 -71
  36. package/dist/chunk-FQI5PFE7.js.map +1 -0
  37. package/dist/chunk-G3P3ZDB4.js +69 -0
  38. package/dist/chunk-G3P3ZDB4.js.map +1 -0
  39. package/dist/{chunk-7AYRWAEK.js → chunk-G7KAVJ3F.js} +2 -2
  40. package/dist/{chunk-7AYRWAEK.js.map → chunk-G7KAVJ3F.js.map} +1 -1
  41. package/dist/{chunk-62DQAWPF.js → chunk-IFYER7O4.js} +367 -42
  42. package/dist/chunk-IFYER7O4.js.map +1 -0
  43. package/dist/chunk-O2MCWFXQ.js +499 -0
  44. package/dist/chunk-O2MCWFXQ.js.map +1 -0
  45. package/dist/{chunk-JWIE5NCR.js → chunk-QZXKQB7Y.js} +140 -10
  46. package/dist/chunk-QZXKQB7Y.js.map +1 -0
  47. package/dist/{db-YAI5AQOI.js → db-N6MPVMEF.js} +8 -2
  48. package/dist/extract-bundle-M4SDJG3V.js +568 -0
  49. package/dist/extract-bundle-M4SDJG3V.js.map +1 -0
  50. package/dist/index.cjs +129 -70
  51. package/dist/index.cjs.map +1 -1
  52. package/dist/index.d.cts +11 -0
  53. package/dist/index.d.ts +11 -0
  54. package/dist/index.js +4 -4
  55. package/dist/location-data-repository-Z4NQOU5Y.js +35 -0
  56. package/dist/{server-IWDHTES2.js → server-ZIGAFKOL.js} +10844 -7480
  57. package/dist/server-ZIGAFKOL.js.map +1 -0
  58. package/dist/site-extract-repository-PGQZNW6V.js +62 -0
  59. package/dist/site-extract-repository-PGQZNW6V.js.map +1 -0
  60. package/dist/{worker-645BZPEK.js → worker-EBB6CTGW.js} +7 -7
  61. package/docs/hosted-location-data.md +108 -0
  62. package/docs/mcp-tool-craft-lint.generated.md +6 -3
  63. package/docs/mcp-tool-manifest.generated.json +2073 -551
  64. package/docs/mcp-tool-quality-spec.md +1 -1
  65. package/docs/specs/connected-services-control-plane-decoupling-spec.md +1044 -0
  66. package/docs/specs/kernel-stealth-captcha-test-matrix.md +278 -0
  67. package/docs/specs/multimodal-image-memory-architecture-spec.md +1022 -0
  68. package/docs/specs/query-fanout-transport-contract-fix.md +9 -1
  69. package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +36 -27
  70. package/package.json +6 -5
  71. package/dist/chunk-5PZ6N2QM.js +0 -130
  72. package/dist/chunk-5PZ6N2QM.js.map +0 -1
  73. package/dist/chunk-62DQAWPF.js.map +0 -1
  74. package/dist/chunk-6OPHG76G.js.map +0 -1
  75. package/dist/chunk-7N2KYL4U.js +0 -7
  76. package/dist/chunk-7N2KYL4U.js.map +0 -1
  77. package/dist/chunk-BWXLTWF7.js.map +0 -1
  78. package/dist/chunk-JWIE5NCR.js.map +0 -1
  79. package/dist/chunk-M2S27J6Z.js.map +0 -1
  80. package/dist/chunk-R7EETU7Z.js +0 -419
  81. package/dist/chunk-R7EETU7Z.js.map +0 -1
  82. package/dist/chunk-XVVNKASZ.js.map +0 -1
  83. package/dist/chunk-ZID3WQID.js.map +0 -1
  84. package/dist/extract-bundle-GPZRLBVY.js +0 -331
  85. package/dist/extract-bundle-GPZRLBVY.js.map +0 -1
  86. package/dist/server-IWDHTES2.js.map +0 -1
  87. package/dist/site-extract-repository-OLVWMOU2.js +0 -30
  88. /package/dist/{chunk-3HBPKR5G.js.map → chunk-D7LM5QZN.js.map} +0 -0
  89. /package/dist/{db-YAI5AQOI.js.map → db-N6MPVMEF.js.map} +0 -0
  90. /package/dist/{site-extract-repository-OLVWMOU2.js.map → location-data-repository-Z4NQOU5Y.js.map} +0 -0
  91. /package/dist/{worker-645BZPEK.js.map → worker-EBB6CTGW.js.map} +0 -0
@@ -1,331 +0,0 @@
1
- import {
2
- downloadAsset
3
- } from "./chunk-JWIE5NCR.js";
4
- import {
5
- computeIssues,
6
- getBlobStore,
7
- renderImageSection,
8
- renderIssueReport,
9
- renderLinkReport
10
- } from "./chunk-R7EETU7Z.js";
11
- import {
12
- getDb
13
- } from "./chunk-62DQAWPF.js";
14
-
15
- // src/api/extract-bundle.ts
16
- import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
17
- import { basename, join } from "path";
18
- import { tmpdir } from "os";
19
- import { createGzip } from "zlib";
20
- import { once } from "events";
21
- import { ZipFile } from "yazl";
22
- import pLimit from "p-limit";
23
- var CHUNK = 400;
24
- var MAX_IMAGES_PER_PAGE = 20;
25
- var MAX_IMAGES_PER_SITE = 500;
26
- var IMAGE_DOWNLOAD_CONCURRENCY = 8;
27
- function normalize(u) {
28
- return u.split("#")[0].replace(/\/$/, "");
29
- }
30
- function registrableDomain(host) {
31
- const h = host.replace(/^www\./, "").toLowerCase();
32
- const parts = h.split(".");
33
- return parts.length <= 2 ? h : parts.slice(-2).join(".");
34
- }
35
- function safeImageFilename(url, index) {
36
- try {
37
- const u = new URL(url);
38
- const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
39
- return base || `image-${index}`;
40
- } catch {
41
- return `image-${index}`;
42
- }
43
- }
44
- function slugFactory() {
45
- const counts = /* @__PURE__ */ new Map();
46
- return (url) => {
47
- const base = url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "page";
48
- const n = counts.get(base) ?? 0;
49
- counts.set(base, n + 1);
50
- return n ? `${base}-${n}` : base;
51
- };
52
- }
53
- async function* pageChunks(jobId) {
54
- const db = getDb();
55
- for (let offset = 0; ; offset += CHUNK) {
56
- const res = await db.execute({
57
- sql: `SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid LIMIT ${CHUNK} OFFSET ?`,
58
- args: [jobId, offset]
59
- });
60
- if (!res.rows.length) return;
61
- yield res.rows.map((r) => JSON.parse(String(r.page)));
62
- }
63
- }
64
- async function assembleExtractArtifacts(job, extras = {}) {
65
- const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`);
66
- const downloadImages = job.options.downloadImages === true;
67
- mkdirSync(join(dir, "pages"), { recursive: true });
68
- if (downloadImages) mkdirSync(join(dir, "images"), { recursive: true });
69
- try {
70
- const metas = [];
71
- const statusByUrl = /* @__PURE__ */ new Map();
72
- for await (const chunk of pageChunks(job.id)) {
73
- for (const p of chunk) {
74
- statusByUrl.set(normalize(p.url), p.status);
75
- const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...slim } = p;
76
- metas.push(slim);
77
- }
78
- }
79
- const gz = createGzip();
80
- const linksOut = createWriteStream(join(dir, "links.jsonl.gz"));
81
- gz.pipe(linksOut);
82
- const gzWrite = async (line) => {
83
- if (!gz.write(line)) await once(gz, "drain");
84
- };
85
- const slug = slugFactory();
86
- const inboundFrom = /* @__PURE__ */ new Map();
87
- const anchorCounts = /* @__PURE__ */ new Map();
88
- const adjacency = /* @__PURE__ */ new Map();
89
- const outCounts = /* @__PURE__ */ new Map();
90
- const domMap = /* @__PURE__ */ new Map();
91
- const brokenLinkPages = /* @__PURE__ */ new Set();
92
- let internalTotal = 0;
93
- let externalTotal = 0;
94
- let brokenInternal = 0;
95
- let pagesWithBody = 0;
96
- let siteReg = "";
97
- try {
98
- siteReg = registrableDomain(new URL(job.startUrl).hostname);
99
- } catch {
100
- siteReg = "";
101
- }
102
- const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY);
103
- const imageDownloads = [];
104
- let imagesQueued = 0;
105
- let imagesFailed = 0;
106
- for await (const chunk of pageChunks(job.id)) {
107
- for (const p of chunk) {
108
- if (p.bodyMarkdown) {
109
- pagesWithBody++;
110
- writeFileSync(
111
- join(dir, "pages", slug(p.url) + ".md"),
112
- `# ${p.title ?? p.url}
113
-
114
- URL: ${p.url}
115
- Status: ${p.status}
116
-
117
- ---
118
-
119
- ${p.bodyMarkdown}
120
- `
121
- );
122
- }
123
- if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {
124
- const pageDir = join(dir, "images", slug(p.url));
125
- mkdirSync(pageDir, { recursive: true });
126
- for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {
127
- if (imagesQueued >= MAX_IMAGES_PER_SITE) break;
128
- imagesQueued++;
129
- const filename = safeImageFilename(imgUrl, idx);
130
- imageDownloads.push(
131
- imageLimit(() => downloadAsset(imgUrl, pageDir, filename)).then(() => void 0, () => {
132
- imagesFailed++;
133
- })
134
- );
135
- }
136
- }
137
- const from = normalize(p.url);
138
- const oc = { int: 0, ext: 0 };
139
- for (const l of p.outlinks ?? []) {
140
- const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
141
- const to = normalize(l.href);
142
- const targetStatus = statusByUrl.get(to) ?? null;
143
- await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + "\n");
144
- if (l.internal) {
145
- oc.int++;
146
- internalTotal++;
147
- if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
148
- inboundFrom.get(to).add(from);
149
- if (l.anchor) {
150
- if (!anchorCounts.has(to)) anchorCounts.set(to, /* @__PURE__ */ new Map());
151
- const ac = anchorCounts.get(to);
152
- ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1);
153
- }
154
- if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
155
- if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
156
- adjacency.get(from).add(to);
157
- }
158
- if (targetStatus != null && targetStatus >= 400) {
159
- brokenInternal++;
160
- brokenLinkPages.add(p.url);
161
- }
162
- } else {
163
- oc.ext++;
164
- let domain = "";
165
- try {
166
- domain = registrableDomain(new URL(l.href).hostname);
167
- } catch {
168
- continue;
169
- }
170
- if (!domain || domain === siteReg) continue;
171
- externalTotal++;
172
- const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
173
- d.links++;
174
- if (nofollow) d.nofollow++;
175
- d.pages.add(p.url);
176
- domMap.set(domain, d);
177
- }
178
- }
179
- outCounts.set(p.url, oc);
180
- }
181
- }
182
- gz.end();
183
- await once(linksOut, "finish");
184
- await Promise.all(imageDownloads);
185
- const depth = /* @__PURE__ */ new Map();
186
- const start = normalize(job.startUrl);
187
- depth.set(start, 0);
188
- const queue = [start];
189
- while (queue.length > 0) {
190
- const cur = queue.shift();
191
- const d = depth.get(cur);
192
- for (const next of adjacency.get(cur) ?? []) {
193
- if (!depth.has(next)) {
194
- depth.set(next, d + 1);
195
- queue.push(next);
196
- }
197
- }
198
- }
199
- const metrics = /* @__PURE__ */ new Map();
200
- for (const p of metas) {
201
- const key = normalize(p.url);
202
- const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
203
- const ac = anchorCounts.get(key);
204
- const topAnchors = ac ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]) : [];
205
- const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 };
206
- metrics.set(p.url, {
207
- url: p.url,
208
- inlinks: inbound.size,
209
- uniqueInlinks: inbound.size,
210
- outlinksInternal: oc.int,
211
- outlinksExternal: oc.ext,
212
- crawlDepth: depth.has(key) ? depth.get(key) : null,
213
- orphan: inbound.size === 0 && key !== start,
214
- topAnchors
215
- });
216
- }
217
- const issues = computeIssues(metas, metrics, brokenLinkPages);
218
- let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics);
219
- if (extras.imageAudit) reportMd += `
220
-
221
- ${renderImageSection(extras.imageAudit)}`;
222
- const metricValues = [...metrics.values()];
223
- const round = (n) => Math.round(n * 10) / 10;
224
- const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
225
- let sumInlinks = 0;
226
- let sumOutlinks = 0;
227
- for (const m of metricValues) {
228
- sumInlinks += m.inlinks;
229
- sumOutlinks += m.outlinksInternal + m.outlinksExternal;
230
- if (m.inlinks === 0) distribution.zero++;
231
- else if (m.inlinks <= 2) distribution.oneToTwo++;
232
- else if (m.inlinks <= 10) distribution.threeToTen++;
233
- else distribution.elevenPlus++;
234
- }
235
- const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
236
- const linkReport = {
237
- summary: {
238
- internal: {
239
- totalLinks: internalTotal,
240
- pages: metas.length,
241
- orphans: metricValues.filter((m) => m.orphan).length,
242
- brokenInternal,
243
- avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,
244
- avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,
245
- distribution,
246
- topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }))
247
- },
248
- external: {
249
- totalLinks: externalTotal,
250
- uniqueDomains: externalDomains.length,
251
- topDomains: externalDomains.slice(0, 20)
252
- }
253
- },
254
- externalDomains
255
- };
256
- const pagesOut = createWriteStream(join(dir, "pages.jsonl"));
257
- for (const p of metas) {
258
- const m = metrics.get(p.url);
259
- const row = { ...p, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
260
- if (!pagesOut.write(JSON.stringify(row) + "\n")) await once(pagesOut, "drain");
261
- }
262
- pagesOut.end();
263
- await once(pagesOut, "finish");
264
- writeFileSync(join(dir, "report.md"), reportMd);
265
- writeFileSync(join(dir, "issues.json"), JSON.stringify(issues, null, 2));
266
- writeFileSync(join(dir, "link-metrics.jsonl"), metricValues.map((m) => JSON.stringify(m)).join("\n"));
267
- writeFileSync(join(dir, "link-report.md"), renderLinkReport(linkReport));
268
- writeFileSync(join(dir, "links-summary.json"), JSON.stringify(linkReport.summary, null, 2));
269
- writeFileSync(join(dir, "external-domains.json"), JSON.stringify(linkReport.externalDomains, null, 2));
270
- if (extras.imageAudit) {
271
- writeFileSync(join(dir, "images.jsonl"), extras.imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"));
272
- writeFileSync(join(dir, "images-summary.json"), JSON.stringify(extras.imageAudit.summary, null, 2));
273
- }
274
- if (extras.seoAudit) writeFileSync(join(dir, "seo-audit.json"), JSON.stringify(extras.seoAudit, null, 2));
275
- if (extras.branding) writeFileSync(join(dir, "branding.json"), JSON.stringify(extras.branding, null, 2));
276
- if (downloadImages) {
277
- writeFileSync(join(dir, "images-download-summary.json"), JSON.stringify({
278
- queued: imagesQueued,
279
- downloaded: imagesQueued - imagesFailed,
280
- failed: imagesFailed,
281
- perPageCap: MAX_IMAGES_PER_PAGE,
282
- perSiteCap: MAX_IMAGES_PER_SITE
283
- }, null, 2));
284
- }
285
- const artifactFiles = [
286
- { name: "report.md", type: "text/markdown" },
287
- { name: "issues.json", type: "application/json" },
288
- { name: "pages.jsonl", type: "application/x-ndjson" },
289
- { name: "links.jsonl.gz", type: "application/gzip" },
290
- { name: "link-metrics.jsonl", type: "application/x-ndjson" },
291
- { name: "link-report.md", type: "text/markdown" },
292
- { name: "links-summary.json", type: "application/json" },
293
- { name: "external-domains.json", type: "application/json" }
294
- ];
295
- if (extras.imageAudit) {
296
- artifactFiles.push({ name: "images.jsonl", type: "application/x-ndjson" });
297
- artifactFiles.push({ name: "images-summary.json", type: "application/json" });
298
- }
299
- if (extras.seoAudit) artifactFiles.push({ name: "seo-audit.json", type: "application/json" });
300
- if (extras.branding) artifactFiles.push({ name: "branding.json", type: "application/json" });
301
- if (downloadImages) artifactFiles.push({ name: "images-download-summary.json", type: "application/json" });
302
- const zip = new ZipFile();
303
- const zipOut = createWriteStream(join(dir, "bundle.zip"));
304
- zip.outputStream.pipe(zipOut);
305
- for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name);
306
- if (pagesWithBody > 0) {
307
- for (const f of readdirSync(join(dir, "pages"))) zip.addFile(join(dir, "pages", f), `pages/${f}`);
308
- }
309
- if (downloadImages && imagesQueued > imagesFailed) {
310
- const imagesDir = join(dir, "images");
311
- for (const rel of readdirSync(imagesDir, { recursive: true })) {
312
- const full = join(imagesDir, rel);
313
- if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split("\\").join("/")}`);
314
- }
315
- }
316
- zip.end();
317
- await once(zipOut, "finish");
318
- const store = getBlobStore();
319
- const stored = [];
320
- for (const f of [...artifactFiles, { name: "bundle.zip", type: "application/zip" }]) {
321
- stored.push(await store.put(`extract/${job.id}/${f.name}`, readFileSync(join(dir, f.name)), f.type));
322
- }
323
- return stored;
324
- } finally {
325
- rmSync(dir, { recursive: true, force: true });
326
- }
327
- }
328
- export {
329
- assembleExtractArtifacts
330
- };
331
- //# sourceMappingURL=extract-bundle-GPZRLBVY.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/api/extract-bundle.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'\nimport { basename, join } from 'node:path'\nimport { tmpdir } from 'node:os'\nimport { createGzip } from 'node:zlib'\nimport { once } from 'node:events'\nimport { ZipFile } from 'yazl'\nimport pLimit from 'p-limit'\nimport { getDb } from './db.js'\nimport { getBlobStore, type StoredArtifact } from './blob-store.js'\nimport { computeIssues, renderIssueReport } from './seo-issues.js'\nimport { renderLinkReport, type ExtDomainRow, type LinkReport } from './seo-link-report.js'\nimport type { PageLinkMetrics } from './seo-link-graph.js'\nimport type { SeoAudit } from './seo-audit.js'\nimport type { ExtractJobRow } from './site-extract-repository.js'\nimport type { PageData } from './site-extractor.js'\nimport { renderImageSection, type ImageAudit } from './image-audit.js'\nimport { downloadAsset } from '../lib/media-extractor.js'\n\nconst CHUNK = 400\nconst MAX_IMAGES_PER_PAGE = 20\nconst MAX_IMAGES_PER_SITE = 500\nconst IMAGE_DOWNLOAD_CONCURRENCY = 8\n\nfunction normalize(u: string): string {\n return u.split('#')[0].replace(/\\/$/, '')\n}\n\nfunction registrableDomain(host: string): string {\n const h = host.replace(/^www\\./, '').toLowerCase()\n const parts = h.split('.')\n return parts.length <= 2 ? h : parts.slice(-2).join('.')\n}\n\nfunction safeImageFilename(url: string, index: number): string {\n try {\n const u = new URL(url)\n const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n return base || `image-${index}`\n } catch {\n return `image-${index}`\n }\n}\n\nfunction slugFactory() {\n const counts = new Map<string, number>()\n return (url: string): string => {\n const base = url.replace(/^https?:\\/\\//, '').replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80) || 'page'\n const n = counts.get(base) ?? 0\n counts.set(base, n + 1)\n return n ? `${base}-${n}` : base\n }\n}\n\nasync function* pageChunks(jobId: string): AsyncGenerator<PageData[]> {\n const db = getDb()\n for (let offset = 0; ; offset += CHUNK) {\n const res = await db.execute({\n sql: `SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid LIMIT ${CHUNK} OFFSET ?`,\n args: [jobId, offset],\n })\n if (!res.rows.length) return\n yield res.rows.map(r => JSON.parse(String((r as unknown as Record<string, unknown>).page)) as PageData)\n }\n}\n\nexport interface AssembleExtras {\n branding?: unknown\n imageAudit?: ImageAudit | null\n seoAudit?: SeoAudit | null\n}\n\nexport async function assembleExtractArtifacts(job: ExtractJobRow, extras: AssembleExtras = {}): Promise<StoredArtifact[]> {\n const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`)\n const downloadImages = job.options.downloadImages === true\n mkdirSync(join(dir, 'pages'), { recursive: true })\n if (downloadImages) mkdirSync(join(dir, 'images'), { recursive: true })\n try {\n const metas: PageData[] = []\n const statusByUrl = new Map<string, number | null>()\n for await (const chunk of pageChunks(job.id)) {\n for (const p of chunk) {\n statusByUrl.set(normalize(p.url), p.status)\n const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...slim } = p\n metas.push(slim as unknown as PageData)\n }\n }\n\n const gz = createGzip()\n const linksOut = createWriteStream(join(dir, 'links.jsonl.gz'))\n gz.pipe(linksOut)\n const gzWrite = async (line: string) => { if (!gz.write(line)) await once(gz, 'drain') }\n\n const slug = slugFactory()\n const inboundFrom = new Map<string, Set<string>>()\n const anchorCounts = new Map<string, Map<string, number>>()\n const adjacency = new Map<string, Set<string>>()\n const outCounts = new Map<string, { int: number; ext: number }>()\n const domMap = new Map<string, { links: number; nofollow: number; pages: Set<string> }>()\n const brokenLinkPages = new Set<string>()\n let internalTotal = 0\n let externalTotal = 0\n let brokenInternal = 0\n let pagesWithBody = 0\n let siteReg = ''\n try { siteReg = registrableDomain(new URL(job.startUrl).hostname) } catch { siteReg = '' }\n\n const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY)\n const imageDownloads: Promise<void>[] = []\n let imagesQueued = 0\n let imagesFailed = 0\n\n for await (const chunk of pageChunks(job.id)) {\n for (const p of chunk) {\n if (p.bodyMarkdown) {\n pagesWithBody++\n writeFileSync(join(dir, 'pages', slug(p.url) + '.md'),\n `# ${p.title ?? p.url}\\n\\nURL: ${p.url}\\nStatus: ${p.status}\\n\\n---\\n\\n${p.bodyMarkdown}\\n`)\n }\n if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {\n const pageDir = join(dir, 'images', slug(p.url))\n mkdirSync(pageDir, { recursive: true })\n for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {\n if (imagesQueued >= MAX_IMAGES_PER_SITE) break\n imagesQueued++\n const filename = safeImageFilename(imgUrl, idx)\n imageDownloads.push(\n imageLimit(() => downloadAsset(imgUrl, pageDir, filename)).then(() => undefined, () => { imagesFailed++ }),\n )\n }\n }\n const from = normalize(p.url)\n const oc = { int: 0, ext: 0 }\n for (const l of p.outlinks ?? []) {\n const nofollow = (l.rel ?? '').split(/\\s+/).includes('nofollow')\n const to = normalize(l.href)\n const targetStatus = statusByUrl.get(to) ?? null\n await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + '\\n')\n if (l.internal) {\n oc.int++\n internalTotal++\n if (!inboundFrom.has(to)) inboundFrom.set(to, new Set())\n inboundFrom.get(to)!.add(from)\n if (l.anchor) {\n if (!anchorCounts.has(to)) anchorCounts.set(to, new Map())\n const ac = anchorCounts.get(to)!\n ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1)\n }\n if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {\n if (!adjacency.has(from)) adjacency.set(from, new Set())\n adjacency.get(from)!.add(to)\n }\n if (targetStatus != null && targetStatus >= 400) {\n brokenInternal++\n brokenLinkPages.add(p.url)\n }\n } else {\n oc.ext++\n let domain = ''\n try { domain = registrableDomain(new URL(l.href).hostname) } catch { continue }\n if (!domain || domain === siteReg) continue\n externalTotal++\n const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: new Set<string>() }\n d.links++\n if (nofollow) d.nofollow++\n d.pages.add(p.url)\n domMap.set(domain, d)\n }\n }\n outCounts.set(p.url, oc)\n }\n }\n gz.end()\n await once(linksOut, 'finish')\n await Promise.all(imageDownloads)\n\n const depth = new Map<string, number>()\n const start = normalize(job.startUrl)\n depth.set(start, 0)\n const queue: string[] = [start]\n while (queue.length > 0) {\n const cur = queue.shift()!\n const d = depth.get(cur)!\n for (const next of adjacency.get(cur) ?? []) {\n if (!depth.has(next)) { depth.set(next, d + 1); queue.push(next) }\n }\n }\n\n const metrics = new Map<string, PageLinkMetrics>()\n for (const p of metas) {\n const key = normalize(p.url)\n const inbound = inboundFrom.get(key) ?? new Set()\n const ac = anchorCounts.get(key)\n const topAnchors = ac\n ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(e => e[0])\n : []\n const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 }\n metrics.set(p.url, {\n url: p.url,\n inlinks: inbound.size,\n uniqueInlinks: inbound.size,\n outlinksInternal: oc.int,\n outlinksExternal: oc.ext,\n crawlDepth: depth.has(key) ? depth.get(key)! : null,\n orphan: inbound.size === 0 && key !== start,\n topAnchors,\n })\n }\n\n const issues = computeIssues(metas, metrics, brokenLinkPages)\n let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics)\n if (extras.imageAudit) reportMd += `\\n\\n${renderImageSection(extras.imageAudit)}`\n\n const metricValues = [...metrics.values()]\n const round = (n: number) => Math.round(n * 10) / 10\n const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 }\n let sumInlinks = 0\n let sumOutlinks = 0\n for (const m of metricValues) {\n sumInlinks += m.inlinks\n sumOutlinks += m.outlinksInternal + m.outlinksExternal\n if (m.inlinks === 0) distribution.zero++\n else if (m.inlinks <= 2) distribution.oneToTwo++\n else if (m.inlinks <= 10) distribution.threeToTen++\n else distribution.elevenPlus++\n }\n const externalDomains: ExtDomainRow[] = [...domMap.entries()]\n .map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size }))\n .sort((a, b) => b.links - a.links)\n const linkReport: LinkReport = {\n summary: {\n internal: {\n totalLinks: internalTotal,\n pages: metas.length,\n orphans: metricValues.filter(m => m.orphan).length,\n brokenInternal,\n avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,\n avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,\n distribution,\n topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20)\n .map(m => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal })),\n },\n external: {\n totalLinks: externalTotal,\n uniqueDomains: externalDomains.length,\n topDomains: externalDomains.slice(0, 20),\n },\n },\n externalDomains,\n }\n\n const pagesOut = createWriteStream(join(dir, 'pages.jsonl'))\n for (const p of metas) {\n const m = metrics.get(p.url)\n const row = { ...(p as unknown as Record<string, unknown>), inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false }\n if (!pagesOut.write(JSON.stringify(row) + '\\n')) await once(pagesOut, 'drain')\n }\n pagesOut.end()\n await once(pagesOut, 'finish')\n\n writeFileSync(join(dir, 'report.md'), reportMd)\n writeFileSync(join(dir, 'issues.json'), JSON.stringify(issues, null, 2))\n writeFileSync(join(dir, 'link-metrics.jsonl'), metricValues.map(m => JSON.stringify(m)).join('\\n'))\n writeFileSync(join(dir, 'link-report.md'), renderLinkReport(linkReport))\n writeFileSync(join(dir, 'links-summary.json'), JSON.stringify(linkReport.summary, null, 2))\n writeFileSync(join(dir, 'external-domains.json'), JSON.stringify(linkReport.externalDomains, null, 2))\n if (extras.imageAudit) {\n writeFileSync(join(dir, 'images.jsonl'), extras.imageAudit.rows.map(r => JSON.stringify(r)).join('\\n'))\n writeFileSync(join(dir, 'images-summary.json'), JSON.stringify(extras.imageAudit.summary, null, 2))\n }\n if (extras.seoAudit) writeFileSync(join(dir, 'seo-audit.json'), JSON.stringify(extras.seoAudit, null, 2))\n if (extras.branding) writeFileSync(join(dir, 'branding.json'), JSON.stringify(extras.branding, null, 2))\n if (downloadImages) {\n writeFileSync(join(dir, 'images-download-summary.json'), JSON.stringify({\n queued: imagesQueued,\n downloaded: imagesQueued - imagesFailed,\n failed: imagesFailed,\n perPageCap: MAX_IMAGES_PER_PAGE,\n perSiteCap: MAX_IMAGES_PER_SITE,\n }, null, 2))\n }\n\n const artifactFiles: Array<{ name: string; type: string }> = [\n { name: 'report.md', type: 'text/markdown' },\n { name: 'issues.json', type: 'application/json' },\n { name: 'pages.jsonl', type: 'application/x-ndjson' },\n { name: 'links.jsonl.gz', type: 'application/gzip' },\n { name: 'link-metrics.jsonl', type: 'application/x-ndjson' },\n { name: 'link-report.md', type: 'text/markdown' },\n { name: 'links-summary.json', type: 'application/json' },\n { name: 'external-domains.json', type: 'application/json' },\n ]\n if (extras.imageAudit) {\n artifactFiles.push({ name: 'images.jsonl', type: 'application/x-ndjson' })\n artifactFiles.push({ name: 'images-summary.json', type: 'application/json' })\n }\n if (extras.seoAudit) artifactFiles.push({ name: 'seo-audit.json', type: 'application/json' })\n if (extras.branding) artifactFiles.push({ name: 'branding.json', type: 'application/json' })\n if (downloadImages) artifactFiles.push({ name: 'images-download-summary.json', type: 'application/json' })\n\n const zip = new ZipFile()\n const zipOut = createWriteStream(join(dir, 'bundle.zip'))\n zip.outputStream.pipe(zipOut)\n for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name)\n if (pagesWithBody > 0) {\n for (const f of readdirSync(join(dir, 'pages'))) zip.addFile(join(dir, 'pages', f), `pages/${f}`)\n }\n if (downloadImages && imagesQueued > imagesFailed) {\n const imagesDir = join(dir, 'images')\n for (const rel of readdirSync(imagesDir, { recursive: true }) as string[]) {\n const full = join(imagesDir, rel)\n if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split('\\\\').join('/')}`)\n }\n }\n zip.end()\n await once(zipOut, 'finish')\n\n const store = getBlobStore()\n const stored: StoredArtifact[] = []\n for (const f of [...artifactFiles, { name: 'bundle.zip', type: 'application/zip' }]) {\n stored.push(await store.put(`extract/${job.id}/${f.name}`, readFileSync(join(dir, f.name)), f.type))\n }\n return stored\n } finally {\n rmSync(dir, { recursive: true, force: true })\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc,aAAa,QAAQ,UAAU,qBAAqB;AACzG,SAAS,UAAU,YAAY;AAC/B,SAAS,cAAc;AACvB,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AACrB,SAAS,eAAe;AACxB,OAAO,YAAY;AAYnB,IAAM,QAAQ;AACd,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAC5B,IAAM,6BAA6B;AAEnC,SAAS,UAAU,GAAmB;AACpC,SAAO,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,OAAO,EAAE;AAC1C;AAEA,SAAS,kBAAkB,MAAsB;AAC/C,QAAM,IAAI,KAAK,QAAQ,UAAU,EAAE,EAAE,YAAY;AACjD,QAAM,QAAQ,EAAE,MAAM,GAAG;AACzB,SAAO,MAAM,UAAU,IAAI,IAAI,MAAM,MAAM,EAAE,EAAE,KAAK,GAAG;AACzD;AAEA,SAAS,kBAAkB,KAAa,OAAuB;AAC7D,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEA,SAAS,cAAc;AACrB,QAAM,SAAS,oBAAI,IAAoB;AACvC,SAAO,CAAC,QAAwB;AAC9B,UAAM,OAAO,IAAI,QAAQ,gBAAgB,EAAE,EAAE,QAAQ,kBAAkB,GAAG,EAAE,QAAQ,YAAY,EAAE,EAAE,MAAM,GAAG,EAAE,KAAK;AACpH,UAAM,IAAI,OAAO,IAAI,IAAI,KAAK;AAC9B,WAAO,IAAI,MAAM,IAAI,CAAC;AACtB,WAAO,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK;AAAA,EAC9B;AACF;AAEA,gBAAgB,WAAW,OAA2C;AACpE,QAAM,KAAK,MAAM;AACjB,WAAS,SAAS,KAAK,UAAU,OAAO;AACtC,UAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,MAC3B,KAAK,6EAA6E,KAAK;AAAA,MACvF,MAAM,CAAC,OAAO,MAAM;AAAA,IACtB,CAAC;AACD,QAAI,CAAC,IAAI,KAAK,OAAQ;AACtB,UAAM,IAAI,KAAK,IAAI,OAAK,KAAK,MAAM,OAAQ,EAAyC,IAAI,CAAC,CAAa;AAAA,EACxG;AACF;AAQA,eAAsB,yBAAyB,KAAoB,SAAyB,CAAC,GAA8B;AACzH,QAAM,MAAM,KAAK,OAAO,GAAG,kBAAkB,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AACnE,QAAM,iBAAiB,IAAI,QAAQ,mBAAmB;AACtD,YAAU,KAAK,KAAK,OAAO,GAAG,EAAE,WAAW,KAAK,CAAC;AACjD,MAAI,eAAgB,WAAU,KAAK,KAAK,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;AACtE,MAAI;AACF,UAAM,QAAoB,CAAC;AAC3B,UAAM,cAAc,oBAAI,IAA2B;AACnD,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,oBAAY,IAAI,UAAU,EAAE,GAAG,GAAG,EAAE,MAAM;AAC1C,cAAM,EAAE,cAAc,IAAI,QAAQ,IAAI,UAAU,IAAI,GAAG,KAAK,IAAI;AAChE,cAAM,KAAK,IAA2B;AAAA,MACxC;AAAA,IACF;AAEA,UAAM,KAAK,WAAW;AACtB,UAAM,WAAW,kBAAkB,KAAK,KAAK,gBAAgB,CAAC;AAC9D,OAAG,KAAK,QAAQ;AAChB,UAAM,UAAU,OAAO,SAAiB;AAAE,UAAI,CAAC,GAAG,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI,OAAO;AAAA,IAAE;AAEvF,UAAM,OAAO,YAAY;AACzB,UAAM,cAAc,oBAAI,IAAyB;AACjD,UAAM,eAAe,oBAAI,IAAiC;AAC1D,UAAM,YAAY,oBAAI,IAAyB;AAC/C,UAAM,YAAY,oBAAI,IAA0C;AAChE,UAAM,SAAS,oBAAI,IAAqE;AACxF,UAAM,kBAAkB,oBAAI,IAAY;AACxC,QAAI,gBAAgB;AACpB,QAAI,gBAAgB;AACpB,QAAI,iBAAiB;AACrB,QAAI,gBAAgB;AACpB,QAAI,UAAU;AACd,QAAI;AAAE,gBAAU,kBAAkB,IAAI,IAAI,IAAI,QAAQ,EAAE,QAAQ;AAAA,IAAE,QAAQ;AAAE,gBAAU;AAAA,IAAG;AAEzF,UAAM,aAAa,OAAO,0BAA0B;AACpD,UAAM,iBAAkC,CAAC;AACzC,QAAI,eAAe;AACnB,QAAI,eAAe;AAEnB,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,YAAI,EAAE,cAAc;AAClB;AACA;AAAA,YAAc,KAAK,KAAK,SAAS,KAAK,EAAE,GAAG,IAAI,KAAK;AAAA,YAClD,KAAK,EAAE,SAAS,EAAE,GAAG;AAAA;AAAA,OAAY,EAAE,GAAG;AAAA,UAAa,EAAE,MAAM;AAAA;AAAA;AAAA;AAAA,EAAc,EAAE,YAAY;AAAA;AAAA,UAAI;AAAA,QAC/F;AACA,YAAI,kBAAkB,EAAE,YAAY,UAAU,eAAe,qBAAqB;AAChF,gBAAM,UAAU,KAAK,KAAK,UAAU,KAAK,EAAE,GAAG,CAAC;AAC/C,oBAAU,SAAS,EAAE,WAAW,KAAK,CAAC;AACtC,qBAAW,CAAC,KAAK,MAAM,KAAK,EAAE,WAAW,MAAM,GAAG,mBAAmB,EAAE,QAAQ,GAAG;AAChF,gBAAI,gBAAgB,oBAAqB;AACzC;AACA,kBAAM,WAAW,kBAAkB,QAAQ,GAAG;AAC9C,2BAAe;AAAA,cACb,WAAW,MAAM,cAAc,QAAQ,SAAS,QAAQ,CAAC,EAAE,KAAK,MAAM,QAAW,MAAM;AAAE;AAAA,cAAe,CAAC;AAAA,YAC3G;AAAA,UACF;AAAA,QACF;AACA,cAAM,OAAO,UAAU,EAAE,GAAG;AAC5B,cAAM,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AAC5B,mBAAW,KAAK,EAAE,YAAY,CAAC,GAAG;AAChC,gBAAM,YAAY,EAAE,OAAO,IAAI,MAAM,KAAK,EAAE,SAAS,UAAU;AAC/D,gBAAM,KAAK,UAAU,EAAE,IAAI;AAC3B,gBAAM,eAAe,YAAY,IAAI,EAAE,KAAK;AAC5C,gBAAM,QAAQ,KAAK,UAAU,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,UAAU,EAAE,UAAU,UAAU,aAAa,CAAC,IAAI,IAAI;AAC5I,cAAI,EAAE,UAAU;AACd,eAAG;AACH;AACA,gBAAI,CAAC,YAAY,IAAI,EAAE,EAAG,aAAY,IAAI,IAAI,oBAAI,IAAI,CAAC;AACvD,wBAAY,IAAI,EAAE,EAAG,IAAI,IAAI;AAC7B,gBAAI,EAAE,QAAQ;AACZ,kBAAI,CAAC,aAAa,IAAI,EAAE,EAAG,cAAa,IAAI,IAAI,oBAAI,IAAI,CAAC;AACzD,oBAAM,KAAK,aAAa,IAAI,EAAE;AAC9B,iBAAG,IAAI,EAAE,SAAS,GAAG,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC;AAAA,YAC9C;AACA,gBAAI,CAAC,aAAa,YAAY,IAAI,EAAE,MAAM,OAAO,CAAC,YAAY,IAAI,EAAE,IAAI;AACtE,kBAAI,CAAC,UAAU,IAAI,IAAI,EAAG,WAAU,IAAI,MAAM,oBAAI,IAAI,CAAC;AACvD,wBAAU,IAAI,IAAI,EAAG,IAAI,EAAE;AAAA,YAC7B;AACA,gBAAI,gBAAgB,QAAQ,gBAAgB,KAAK;AAC/C;AACA,8BAAgB,IAAI,EAAE,GAAG;AAAA,YAC3B;AAAA,UACF,OAAO;AACL,eAAG;AACH,gBAAI,SAAS;AACb,gBAAI;AAAE,uBAAS,kBAAkB,IAAI,IAAI,EAAE,IAAI,EAAE,QAAQ;AAAA,YAAE,QAAQ;AAAE;AAAA,YAAS;AAC9E,gBAAI,CAAC,UAAU,WAAW,QAAS;AACnC;AACA,kBAAM,IAAI,OAAO,IAAI,MAAM,KAAK,EAAE,OAAO,GAAG,UAAU,GAAG,OAAO,oBAAI,IAAY,EAAE;AAClF,cAAE;AACF,gBAAI,SAAU,GAAE;AAChB,cAAE,MAAM,IAAI,EAAE,GAAG;AACjB,mBAAO,IAAI,QAAQ,CAAC;AAAA,UACtB;AAAA,QACF;AACA,kBAAU,IAAI,EAAE,KAAK,EAAE;AAAA,MACzB;AAAA,IACF;AACA,OAAG,IAAI;AACP,UAAM,KAAK,UAAU,QAAQ;AAC7B,UAAM,QAAQ,IAAI,cAAc;AAEhC,UAAM,QAAQ,oBAAI,IAAoB;AACtC,UAAM,QAAQ,UAAU,IAAI,QAAQ;AACpC,UAAM,IAAI,OAAO,CAAC;AAClB,UAAM,QAAkB,CAAC,KAAK;AAC9B,WAAO,MAAM,SAAS,GAAG;AACvB,YAAM,MAAM,MAAM,MAAM;AACxB,YAAM,IAAI,MAAM,IAAI,GAAG;AACvB,iBAAW,QAAQ,UAAU,IAAI,GAAG,KAAK,CAAC,GAAG;AAC3C,YAAI,CAAC,MAAM,IAAI,IAAI,GAAG;AAAE,gBAAM,IAAI,MAAM,IAAI,CAAC;AAAG,gBAAM,KAAK,IAAI;AAAA,QAAE;AAAA,MACnE;AAAA,IACF;AAEA,UAAM,UAAU,oBAAI,IAA6B;AACjD,eAAW,KAAK,OAAO;AACrB,YAAM,MAAM,UAAU,EAAE,GAAG;AAC3B,YAAM,UAAU,YAAY,IAAI,GAAG,KAAK,oBAAI,IAAI;AAChD,YAAM,KAAK,aAAa,IAAI,GAAG;AAC/B,YAAM,aAAa,KACf,CAAC,GAAG,GAAG,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,IAAI,OAAK,EAAE,CAAC,CAAC,IACvE,CAAC;AACL,YAAM,KAAK,UAAU,IAAI,EAAE,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AACpD,cAAQ,IAAI,EAAE,KAAK;AAAA,QACjB,KAAK,EAAE;AAAA,QACP,SAAS,QAAQ;AAAA,QACjB,eAAe,QAAQ;AAAA,QACvB,kBAAkB,GAAG;AAAA,QACrB,kBAAkB,GAAG;AAAA,QACrB,YAAY,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,IAAK;AAAA,QAC/C,QAAQ,QAAQ,SAAS,KAAK,QAAQ;AAAA,QACtC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,cAAc,OAAO,SAAS,eAAe;AAC5D,QAAI,WAAW,kBAAkB,IAAI,UAAU,OAAO,QAAQ,OAAO;AACrE,QAAI,OAAO,WAAY,aAAY;AAAA;AAAA,EAAO,mBAAmB,OAAO,UAAU,CAAC;AAE/E,UAAM,eAAe,CAAC,GAAG,QAAQ,OAAO,CAAC;AACzC,UAAM,QAAQ,CAAC,MAAc,KAAK,MAAM,IAAI,EAAE,IAAI;AAClD,UAAM,eAAe,EAAE,MAAM,GAAG,UAAU,GAAG,YAAY,GAAG,YAAY,EAAE;AAC1E,QAAI,aAAa;AACjB,QAAI,cAAc;AAClB,eAAW,KAAK,cAAc;AAC5B,oBAAc,EAAE;AAChB,qBAAe,EAAE,mBAAmB,EAAE;AACtC,UAAI,EAAE,YAAY,EAAG,cAAa;AAAA,eACzB,EAAE,WAAW,EAAG,cAAa;AAAA,eAC7B,EAAE,WAAW,GAAI,cAAa;AAAA,UAClC,cAAa;AAAA,IACpB;AACA,UAAM,kBAAkC,CAAC,GAAG,OAAO,QAAQ,CAAC,EACzD,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,OAAO,EAAE,MAAM,KAAK,EAAE,EAC5F,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACnC,UAAM,aAAyB;AAAA,MAC7B,SAAS;AAAA,QACP,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,OAAO,MAAM;AAAA,UACb,SAAS,aAAa,OAAO,OAAK,EAAE,MAAM,EAAE;AAAA,UAC5C;AAAA,UACA,YAAY,MAAM,SAAS,MAAM,aAAa,MAAM,MAAM,IAAI;AAAA,UAC9D,aAAa,MAAM,SAAS,MAAM,cAAc,MAAM,MAAM,IAAI;AAAA,UAChE;AAAA,UACA,cAAc,CAAC,GAAG,YAAY,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,EAAE,EAC9E,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,SAAS,EAAE,SAAS,kBAAkB,EAAE,kBAAkB,kBAAkB,EAAE,iBAAiB,EAAE;AAAA,QAC9H;AAAA,QACA,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,eAAe,gBAAgB;AAAA,UAC/B,YAAY,gBAAgB,MAAM,GAAG,EAAE;AAAA,QACzC;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,UAAM,WAAW,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAC3D,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,IAAI,EAAE,GAAG;AAC3B,YAAM,MAAM,EAAE,GAAI,GAA0C,SAAS,GAAG,WAAW,GAAG,YAAY,GAAG,cAAc,MAAM,QAAQ,GAAG,UAAU,MAAM;AACpJ,UAAI,CAAC,SAAS,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,EAAG,OAAM,KAAK,UAAU,OAAO;AAAA,IAC/E;AACA,aAAS,IAAI;AACb,UAAM,KAAK,UAAU,QAAQ;AAE7B,kBAAc,KAAK,KAAK,WAAW,GAAG,QAAQ;AAC9C,kBAAc,KAAK,KAAK,aAAa,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,aAAa,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAClG,kBAAc,KAAK,KAAK,gBAAgB,GAAG,iBAAiB,UAAU,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,KAAK,UAAU,WAAW,SAAS,MAAM,CAAC,CAAC;AAC1F,kBAAc,KAAK,KAAK,uBAAuB,GAAG,KAAK,UAAU,WAAW,iBAAiB,MAAM,CAAC,CAAC;AACrG,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,KAAK,cAAc,GAAG,OAAO,WAAW,KAAK,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACtG,oBAAc,KAAK,KAAK,qBAAqB,GAAG,KAAK,UAAU,OAAO,WAAW,SAAS,MAAM,CAAC,CAAC;AAAA,IACpG;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,gBAAgB,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACxG,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,eAAe,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACvG,QAAI,gBAAgB;AAClB,oBAAc,KAAK,KAAK,8BAA8B,GAAG,KAAK,UAAU;AAAA,QACtE,QAAQ;AAAA,QACR,YAAY,eAAe;AAAA,QAC3B,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,YAAY;AAAA,MACd,GAAG,MAAM,CAAC,CAAC;AAAA,IACb;AAEA,UAAM,gBAAuD;AAAA,MAC3D,EAAE,MAAM,aAAa,MAAM,gBAAgB;AAAA,MAC3C,EAAE,MAAM,eAAe,MAAM,mBAAmB;AAAA,MAChD,EAAE,MAAM,eAAe,MAAM,uBAAuB;AAAA,MACpD,EAAE,MAAM,kBAAkB,MAAM,mBAAmB;AAAA,MACnD,EAAE,MAAM,sBAAsB,MAAM,uBAAuB;AAAA,MAC3D,EAAE,MAAM,kBAAkB,MAAM,gBAAgB;AAAA,MAChD,EAAE,MAAM,sBAAsB,MAAM,mBAAmB;AAAA,MACvD,EAAE,MAAM,yBAAyB,MAAM,mBAAmB;AAAA,IAC5D;AACA,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,EAAE,MAAM,gBAAgB,MAAM,uBAAuB,CAAC;AACzE,oBAAc,KAAK,EAAE,MAAM,uBAAuB,MAAM,mBAAmB,CAAC;AAAA,IAC9E;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,kBAAkB,MAAM,mBAAmB,CAAC;AAC5F,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,iBAAiB,MAAM,mBAAmB,CAAC;AAC3F,QAAI,eAAgB,eAAc,KAAK,EAAE,MAAM,gCAAgC,MAAM,mBAAmB,CAAC;AAEzG,UAAM,MAAM,IAAI,QAAQ;AACxB,UAAM,SAAS,kBAAkB,KAAK,KAAK,YAAY,CAAC;AACxD,QAAI,aAAa,KAAK,MAAM;AAC5B,eAAW,KAAK,cAAe,KAAI,QAAQ,KAAK,KAAK,EAAE,IAAI,GAAG,EAAE,IAAI;AACpE,QAAI,gBAAgB,GAAG;AACrB,iBAAW,KAAK,YAAY,KAAK,KAAK,OAAO,CAAC,EAAG,KAAI,QAAQ,KAAK,KAAK,SAAS,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAClG;AACA,QAAI,kBAAkB,eAAe,cAAc;AACjD,YAAM,YAAY,KAAK,KAAK,QAAQ;AACpC,iBAAW,OAAO,YAAY,WAAW,EAAE,WAAW,KAAK,CAAC,GAAe;AACzE,cAAM,OAAO,KAAK,WAAW,GAAG;AAChC,YAAI,SAAS,IAAI,EAAE,OAAO,EAAG,KAAI,QAAQ,MAAM,UAAU,IAAI,MAAM,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE;AAAA,MACtF;AAAA,IACF;AACA,QAAI,IAAI;AACR,UAAM,KAAK,QAAQ,QAAQ;AAE3B,UAAM,QAAQ,aAAa;AAC3B,UAAM,SAA2B,CAAC;AAClC,eAAW,KAAK,CAAC,GAAG,eAAe,EAAE,MAAM,cAAc,MAAM,kBAAkB,CAAC,GAAG;AACnF,aAAO,KAAK,MAAM,MAAM,IAAI,WAAW,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,aAAa,KAAK,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC;AAAA,IACrG;AACA,WAAO;AAAA,EACT,UAAE;AACA,WAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAAA,EAC9C;AACF;","names":[]}