mcp-scraper 0.34.1 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -9
- package/dist/bin/api-server.cjs +24342 -17931
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +51 -7
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +48 -5
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +2997 -2070
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +8 -8
- package/dist/bin/paa-harvest.cjs +125 -70
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +4 -4
- package/dist/chunk-345BQXZH.js +712 -0
- package/dist/chunk-345BQXZH.js.map +1 -0
- package/dist/{chunk-M2S27J6Z.js → chunk-44HZLHDV.js} +10 -1
- package/dist/chunk-44HZLHDV.js.map +1 -0
- package/dist/chunk-4HO66323.js +7 -0
- package/dist/chunk-4HO66323.js.map +1 -0
- package/dist/{chunk-BWXLTWF7.js → chunk-4ZIJ3BKZ.js} +6 -4
- package/dist/chunk-4ZIJ3BKZ.js.map +1 -0
- package/dist/{chunk-6OPHG76G.js → chunk-5RULXBJ7.js} +3166 -2322
- package/dist/chunk-5RULXBJ7.js.map +1 -0
- package/dist/chunk-AN3VQARU.js +684 -0
- package/dist/chunk-AN3VQARU.js.map +1 -0
- package/dist/{chunk-XVVNKASZ.js → chunk-ANCGXUQJ.js} +118 -73
- package/dist/chunk-ANCGXUQJ.js.map +1 -0
- package/dist/{chunk-3HBPKR5G.js → chunk-D7LM5QZN.js} +3 -3
- package/dist/{chunk-4ZB3X6BQ.js → chunk-E5UEELA7.js} +16 -2
- package/dist/{chunk-4ZB3X6BQ.js.map → chunk-E5UEELA7.js.map} +1 -1
- package/dist/{chunk-ZID3WQID.js → chunk-FQI5PFE7.js} +9 -71
- package/dist/chunk-FQI5PFE7.js.map +1 -0
- package/dist/chunk-G3P3ZDB4.js +69 -0
- package/dist/chunk-G3P3ZDB4.js.map +1 -0
- package/dist/{chunk-7AYRWAEK.js → chunk-G7KAVJ3F.js} +2 -2
- package/dist/{chunk-7AYRWAEK.js.map → chunk-G7KAVJ3F.js.map} +1 -1
- package/dist/{chunk-62DQAWPF.js → chunk-IFYER7O4.js} +367 -42
- package/dist/chunk-IFYER7O4.js.map +1 -0
- package/dist/chunk-O2MCWFXQ.js +499 -0
- package/dist/chunk-O2MCWFXQ.js.map +1 -0
- package/dist/{chunk-JWIE5NCR.js → chunk-QZXKQB7Y.js} +140 -10
- package/dist/chunk-QZXKQB7Y.js.map +1 -0
- package/dist/{db-YAI5AQOI.js → db-N6MPVMEF.js} +8 -2
- package/dist/extract-bundle-M4SDJG3V.js +568 -0
- package/dist/extract-bundle-M4SDJG3V.js.map +1 -0
- package/dist/index.cjs +129 -70
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +11 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +4 -4
- package/dist/location-data-repository-Z4NQOU5Y.js +35 -0
- package/dist/{server-IWDHTES2.js → server-ZIGAFKOL.js} +10844 -7480
- package/dist/server-ZIGAFKOL.js.map +1 -0
- package/dist/site-extract-repository-PGQZNW6V.js +62 -0
- package/dist/site-extract-repository-PGQZNW6V.js.map +1 -0
- package/dist/{worker-645BZPEK.js → worker-EBB6CTGW.js} +7 -7
- package/docs/hosted-location-data.md +108 -0
- package/docs/mcp-tool-craft-lint.generated.md +6 -3
- package/docs/mcp-tool-manifest.generated.json +2073 -551
- package/docs/mcp-tool-quality-spec.md +1 -1
- package/docs/specs/connected-services-control-plane-decoupling-spec.md +1044 -0
- package/docs/specs/kernel-stealth-captcha-test-matrix.md +278 -0
- package/docs/specs/multimodal-image-memory-architecture-spec.md +1022 -0
- package/docs/specs/query-fanout-transport-contract-fix.md +9 -1
- package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +36 -27
- package/package.json +6 -5
- package/dist/chunk-5PZ6N2QM.js +0 -130
- package/dist/chunk-5PZ6N2QM.js.map +0 -1
- package/dist/chunk-62DQAWPF.js.map +0 -1
- package/dist/chunk-6OPHG76G.js.map +0 -1
- package/dist/chunk-7N2KYL4U.js +0 -7
- package/dist/chunk-7N2KYL4U.js.map +0 -1
- package/dist/chunk-BWXLTWF7.js.map +0 -1
- package/dist/chunk-JWIE5NCR.js.map +0 -1
- package/dist/chunk-M2S27J6Z.js.map +0 -1
- package/dist/chunk-R7EETU7Z.js +0 -419
- package/dist/chunk-R7EETU7Z.js.map +0 -1
- package/dist/chunk-XVVNKASZ.js.map +0 -1
- package/dist/chunk-ZID3WQID.js.map +0 -1
- package/dist/extract-bundle-GPZRLBVY.js +0 -331
- package/dist/extract-bundle-GPZRLBVY.js.map +0 -1
- package/dist/server-IWDHTES2.js.map +0 -1
- package/dist/site-extract-repository-OLVWMOU2.js +0 -30
- /package/dist/{chunk-3HBPKR5G.js.map → chunk-D7LM5QZN.js.map} +0 -0
- /package/dist/{db-YAI5AQOI.js.map → db-N6MPVMEF.js.map} +0 -0
- /package/dist/{site-extract-repository-OLVWMOU2.js.map → location-data-repository-Z4NQOU5Y.js.map} +0 -0
- /package/dist/{worker-645BZPEK.js.map → worker-EBB6CTGW.js.map} +0 -0
|
@@ -1,331 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
downloadAsset
|
|
3
|
-
} from "./chunk-JWIE5NCR.js";
|
|
4
|
-
import {
|
|
5
|
-
computeIssues,
|
|
6
|
-
getBlobStore,
|
|
7
|
-
renderImageSection,
|
|
8
|
-
renderIssueReport,
|
|
9
|
-
renderLinkReport
|
|
10
|
-
} from "./chunk-R7EETU7Z.js";
|
|
11
|
-
import {
|
|
12
|
-
getDb
|
|
13
|
-
} from "./chunk-62DQAWPF.js";
|
|
14
|
-
|
|
15
|
-
// src/api/extract-bundle.ts
|
|
16
|
-
import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
|
|
17
|
-
import { basename, join } from "path";
|
|
18
|
-
import { tmpdir } from "os";
|
|
19
|
-
import { createGzip } from "zlib";
|
|
20
|
-
import { once } from "events";
|
|
21
|
-
import { ZipFile } from "yazl";
|
|
22
|
-
import pLimit from "p-limit";
|
|
23
|
-
var CHUNK = 400;
|
|
24
|
-
var MAX_IMAGES_PER_PAGE = 20;
|
|
25
|
-
var MAX_IMAGES_PER_SITE = 500;
|
|
26
|
-
var IMAGE_DOWNLOAD_CONCURRENCY = 8;
|
|
27
|
-
function normalize(u) {
|
|
28
|
-
return u.split("#")[0].replace(/\/$/, "");
|
|
29
|
-
}
|
|
30
|
-
function registrableDomain(host) {
|
|
31
|
-
const h = host.replace(/^www\./, "").toLowerCase();
|
|
32
|
-
const parts = h.split(".");
|
|
33
|
-
return parts.length <= 2 ? h : parts.slice(-2).join(".");
|
|
34
|
-
}
|
|
35
|
-
function safeImageFilename(url, index) {
|
|
36
|
-
try {
|
|
37
|
-
const u = new URL(url);
|
|
38
|
-
const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
|
|
39
|
-
return base || `image-${index}`;
|
|
40
|
-
} catch {
|
|
41
|
-
return `image-${index}`;
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
function slugFactory() {
|
|
45
|
-
const counts = /* @__PURE__ */ new Map();
|
|
46
|
-
return (url) => {
|
|
47
|
-
const base = url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "page";
|
|
48
|
-
const n = counts.get(base) ?? 0;
|
|
49
|
-
counts.set(base, n + 1);
|
|
50
|
-
return n ? `${base}-${n}` : base;
|
|
51
|
-
};
|
|
52
|
-
}
|
|
53
|
-
async function* pageChunks(jobId) {
|
|
54
|
-
const db = getDb();
|
|
55
|
-
for (let offset = 0; ; offset += CHUNK) {
|
|
56
|
-
const res = await db.execute({
|
|
57
|
-
sql: `SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid LIMIT ${CHUNK} OFFSET ?`,
|
|
58
|
-
args: [jobId, offset]
|
|
59
|
-
});
|
|
60
|
-
if (!res.rows.length) return;
|
|
61
|
-
yield res.rows.map((r) => JSON.parse(String(r.page)));
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
async function assembleExtractArtifacts(job, extras = {}) {
|
|
65
|
-
const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`);
|
|
66
|
-
const downloadImages = job.options.downloadImages === true;
|
|
67
|
-
mkdirSync(join(dir, "pages"), { recursive: true });
|
|
68
|
-
if (downloadImages) mkdirSync(join(dir, "images"), { recursive: true });
|
|
69
|
-
try {
|
|
70
|
-
const metas = [];
|
|
71
|
-
const statusByUrl = /* @__PURE__ */ new Map();
|
|
72
|
-
for await (const chunk of pageChunks(job.id)) {
|
|
73
|
-
for (const p of chunk) {
|
|
74
|
-
statusByUrl.set(normalize(p.url), p.status);
|
|
75
|
-
const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...slim } = p;
|
|
76
|
-
metas.push(slim);
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
const gz = createGzip();
|
|
80
|
-
const linksOut = createWriteStream(join(dir, "links.jsonl.gz"));
|
|
81
|
-
gz.pipe(linksOut);
|
|
82
|
-
const gzWrite = async (line) => {
|
|
83
|
-
if (!gz.write(line)) await once(gz, "drain");
|
|
84
|
-
};
|
|
85
|
-
const slug = slugFactory();
|
|
86
|
-
const inboundFrom = /* @__PURE__ */ new Map();
|
|
87
|
-
const anchorCounts = /* @__PURE__ */ new Map();
|
|
88
|
-
const adjacency = /* @__PURE__ */ new Map();
|
|
89
|
-
const outCounts = /* @__PURE__ */ new Map();
|
|
90
|
-
const domMap = /* @__PURE__ */ new Map();
|
|
91
|
-
const brokenLinkPages = /* @__PURE__ */ new Set();
|
|
92
|
-
let internalTotal = 0;
|
|
93
|
-
let externalTotal = 0;
|
|
94
|
-
let brokenInternal = 0;
|
|
95
|
-
let pagesWithBody = 0;
|
|
96
|
-
let siteReg = "";
|
|
97
|
-
try {
|
|
98
|
-
siteReg = registrableDomain(new URL(job.startUrl).hostname);
|
|
99
|
-
} catch {
|
|
100
|
-
siteReg = "";
|
|
101
|
-
}
|
|
102
|
-
const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY);
|
|
103
|
-
const imageDownloads = [];
|
|
104
|
-
let imagesQueued = 0;
|
|
105
|
-
let imagesFailed = 0;
|
|
106
|
-
for await (const chunk of pageChunks(job.id)) {
|
|
107
|
-
for (const p of chunk) {
|
|
108
|
-
if (p.bodyMarkdown) {
|
|
109
|
-
pagesWithBody++;
|
|
110
|
-
writeFileSync(
|
|
111
|
-
join(dir, "pages", slug(p.url) + ".md"),
|
|
112
|
-
`# ${p.title ?? p.url}
|
|
113
|
-
|
|
114
|
-
URL: ${p.url}
|
|
115
|
-
Status: ${p.status}
|
|
116
|
-
|
|
117
|
-
---
|
|
118
|
-
|
|
119
|
-
${p.bodyMarkdown}
|
|
120
|
-
`
|
|
121
|
-
);
|
|
122
|
-
}
|
|
123
|
-
if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {
|
|
124
|
-
const pageDir = join(dir, "images", slug(p.url));
|
|
125
|
-
mkdirSync(pageDir, { recursive: true });
|
|
126
|
-
for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {
|
|
127
|
-
if (imagesQueued >= MAX_IMAGES_PER_SITE) break;
|
|
128
|
-
imagesQueued++;
|
|
129
|
-
const filename = safeImageFilename(imgUrl, idx);
|
|
130
|
-
imageDownloads.push(
|
|
131
|
-
imageLimit(() => downloadAsset(imgUrl, pageDir, filename)).then(() => void 0, () => {
|
|
132
|
-
imagesFailed++;
|
|
133
|
-
})
|
|
134
|
-
);
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
const from = normalize(p.url);
|
|
138
|
-
const oc = { int: 0, ext: 0 };
|
|
139
|
-
for (const l of p.outlinks ?? []) {
|
|
140
|
-
const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
|
|
141
|
-
const to = normalize(l.href);
|
|
142
|
-
const targetStatus = statusByUrl.get(to) ?? null;
|
|
143
|
-
await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + "\n");
|
|
144
|
-
if (l.internal) {
|
|
145
|
-
oc.int++;
|
|
146
|
-
internalTotal++;
|
|
147
|
-
if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
|
|
148
|
-
inboundFrom.get(to).add(from);
|
|
149
|
-
if (l.anchor) {
|
|
150
|
-
if (!anchorCounts.has(to)) anchorCounts.set(to, /* @__PURE__ */ new Map());
|
|
151
|
-
const ac = anchorCounts.get(to);
|
|
152
|
-
ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1);
|
|
153
|
-
}
|
|
154
|
-
if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
|
|
155
|
-
if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
|
|
156
|
-
adjacency.get(from).add(to);
|
|
157
|
-
}
|
|
158
|
-
if (targetStatus != null && targetStatus >= 400) {
|
|
159
|
-
brokenInternal++;
|
|
160
|
-
brokenLinkPages.add(p.url);
|
|
161
|
-
}
|
|
162
|
-
} else {
|
|
163
|
-
oc.ext++;
|
|
164
|
-
let domain = "";
|
|
165
|
-
try {
|
|
166
|
-
domain = registrableDomain(new URL(l.href).hostname);
|
|
167
|
-
} catch {
|
|
168
|
-
continue;
|
|
169
|
-
}
|
|
170
|
-
if (!domain || domain === siteReg) continue;
|
|
171
|
-
externalTotal++;
|
|
172
|
-
const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
|
|
173
|
-
d.links++;
|
|
174
|
-
if (nofollow) d.nofollow++;
|
|
175
|
-
d.pages.add(p.url);
|
|
176
|
-
domMap.set(domain, d);
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
outCounts.set(p.url, oc);
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
gz.end();
|
|
183
|
-
await once(linksOut, "finish");
|
|
184
|
-
await Promise.all(imageDownloads);
|
|
185
|
-
const depth = /* @__PURE__ */ new Map();
|
|
186
|
-
const start = normalize(job.startUrl);
|
|
187
|
-
depth.set(start, 0);
|
|
188
|
-
const queue = [start];
|
|
189
|
-
while (queue.length > 0) {
|
|
190
|
-
const cur = queue.shift();
|
|
191
|
-
const d = depth.get(cur);
|
|
192
|
-
for (const next of adjacency.get(cur) ?? []) {
|
|
193
|
-
if (!depth.has(next)) {
|
|
194
|
-
depth.set(next, d + 1);
|
|
195
|
-
queue.push(next);
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
const metrics = /* @__PURE__ */ new Map();
|
|
200
|
-
for (const p of metas) {
|
|
201
|
-
const key = normalize(p.url);
|
|
202
|
-
const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
|
|
203
|
-
const ac = anchorCounts.get(key);
|
|
204
|
-
const topAnchors = ac ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]) : [];
|
|
205
|
-
const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 };
|
|
206
|
-
metrics.set(p.url, {
|
|
207
|
-
url: p.url,
|
|
208
|
-
inlinks: inbound.size,
|
|
209
|
-
uniqueInlinks: inbound.size,
|
|
210
|
-
outlinksInternal: oc.int,
|
|
211
|
-
outlinksExternal: oc.ext,
|
|
212
|
-
crawlDepth: depth.has(key) ? depth.get(key) : null,
|
|
213
|
-
orphan: inbound.size === 0 && key !== start,
|
|
214
|
-
topAnchors
|
|
215
|
-
});
|
|
216
|
-
}
|
|
217
|
-
const issues = computeIssues(metas, metrics, brokenLinkPages);
|
|
218
|
-
let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics);
|
|
219
|
-
if (extras.imageAudit) reportMd += `
|
|
220
|
-
|
|
221
|
-
${renderImageSection(extras.imageAudit)}`;
|
|
222
|
-
const metricValues = [...metrics.values()];
|
|
223
|
-
const round = (n) => Math.round(n * 10) / 10;
|
|
224
|
-
const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
|
|
225
|
-
let sumInlinks = 0;
|
|
226
|
-
let sumOutlinks = 0;
|
|
227
|
-
for (const m of metricValues) {
|
|
228
|
-
sumInlinks += m.inlinks;
|
|
229
|
-
sumOutlinks += m.outlinksInternal + m.outlinksExternal;
|
|
230
|
-
if (m.inlinks === 0) distribution.zero++;
|
|
231
|
-
else if (m.inlinks <= 2) distribution.oneToTwo++;
|
|
232
|
-
else if (m.inlinks <= 10) distribution.threeToTen++;
|
|
233
|
-
else distribution.elevenPlus++;
|
|
234
|
-
}
|
|
235
|
-
const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
|
|
236
|
-
const linkReport = {
|
|
237
|
-
summary: {
|
|
238
|
-
internal: {
|
|
239
|
-
totalLinks: internalTotal,
|
|
240
|
-
pages: metas.length,
|
|
241
|
-
orphans: metricValues.filter((m) => m.orphan).length,
|
|
242
|
-
brokenInternal,
|
|
243
|
-
avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,
|
|
244
|
-
avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,
|
|
245
|
-
distribution,
|
|
246
|
-
topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }))
|
|
247
|
-
},
|
|
248
|
-
external: {
|
|
249
|
-
totalLinks: externalTotal,
|
|
250
|
-
uniqueDomains: externalDomains.length,
|
|
251
|
-
topDomains: externalDomains.slice(0, 20)
|
|
252
|
-
}
|
|
253
|
-
},
|
|
254
|
-
externalDomains
|
|
255
|
-
};
|
|
256
|
-
const pagesOut = createWriteStream(join(dir, "pages.jsonl"));
|
|
257
|
-
for (const p of metas) {
|
|
258
|
-
const m = metrics.get(p.url);
|
|
259
|
-
const row = { ...p, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
|
|
260
|
-
if (!pagesOut.write(JSON.stringify(row) + "\n")) await once(pagesOut, "drain");
|
|
261
|
-
}
|
|
262
|
-
pagesOut.end();
|
|
263
|
-
await once(pagesOut, "finish");
|
|
264
|
-
writeFileSync(join(dir, "report.md"), reportMd);
|
|
265
|
-
writeFileSync(join(dir, "issues.json"), JSON.stringify(issues, null, 2));
|
|
266
|
-
writeFileSync(join(dir, "link-metrics.jsonl"), metricValues.map((m) => JSON.stringify(m)).join("\n"));
|
|
267
|
-
writeFileSync(join(dir, "link-report.md"), renderLinkReport(linkReport));
|
|
268
|
-
writeFileSync(join(dir, "links-summary.json"), JSON.stringify(linkReport.summary, null, 2));
|
|
269
|
-
writeFileSync(join(dir, "external-domains.json"), JSON.stringify(linkReport.externalDomains, null, 2));
|
|
270
|
-
if (extras.imageAudit) {
|
|
271
|
-
writeFileSync(join(dir, "images.jsonl"), extras.imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"));
|
|
272
|
-
writeFileSync(join(dir, "images-summary.json"), JSON.stringify(extras.imageAudit.summary, null, 2));
|
|
273
|
-
}
|
|
274
|
-
if (extras.seoAudit) writeFileSync(join(dir, "seo-audit.json"), JSON.stringify(extras.seoAudit, null, 2));
|
|
275
|
-
if (extras.branding) writeFileSync(join(dir, "branding.json"), JSON.stringify(extras.branding, null, 2));
|
|
276
|
-
if (downloadImages) {
|
|
277
|
-
writeFileSync(join(dir, "images-download-summary.json"), JSON.stringify({
|
|
278
|
-
queued: imagesQueued,
|
|
279
|
-
downloaded: imagesQueued - imagesFailed,
|
|
280
|
-
failed: imagesFailed,
|
|
281
|
-
perPageCap: MAX_IMAGES_PER_PAGE,
|
|
282
|
-
perSiteCap: MAX_IMAGES_PER_SITE
|
|
283
|
-
}, null, 2));
|
|
284
|
-
}
|
|
285
|
-
const artifactFiles = [
|
|
286
|
-
{ name: "report.md", type: "text/markdown" },
|
|
287
|
-
{ name: "issues.json", type: "application/json" },
|
|
288
|
-
{ name: "pages.jsonl", type: "application/x-ndjson" },
|
|
289
|
-
{ name: "links.jsonl.gz", type: "application/gzip" },
|
|
290
|
-
{ name: "link-metrics.jsonl", type: "application/x-ndjson" },
|
|
291
|
-
{ name: "link-report.md", type: "text/markdown" },
|
|
292
|
-
{ name: "links-summary.json", type: "application/json" },
|
|
293
|
-
{ name: "external-domains.json", type: "application/json" }
|
|
294
|
-
];
|
|
295
|
-
if (extras.imageAudit) {
|
|
296
|
-
artifactFiles.push({ name: "images.jsonl", type: "application/x-ndjson" });
|
|
297
|
-
artifactFiles.push({ name: "images-summary.json", type: "application/json" });
|
|
298
|
-
}
|
|
299
|
-
if (extras.seoAudit) artifactFiles.push({ name: "seo-audit.json", type: "application/json" });
|
|
300
|
-
if (extras.branding) artifactFiles.push({ name: "branding.json", type: "application/json" });
|
|
301
|
-
if (downloadImages) artifactFiles.push({ name: "images-download-summary.json", type: "application/json" });
|
|
302
|
-
const zip = new ZipFile();
|
|
303
|
-
const zipOut = createWriteStream(join(dir, "bundle.zip"));
|
|
304
|
-
zip.outputStream.pipe(zipOut);
|
|
305
|
-
for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name);
|
|
306
|
-
if (pagesWithBody > 0) {
|
|
307
|
-
for (const f of readdirSync(join(dir, "pages"))) zip.addFile(join(dir, "pages", f), `pages/${f}`);
|
|
308
|
-
}
|
|
309
|
-
if (downloadImages && imagesQueued > imagesFailed) {
|
|
310
|
-
const imagesDir = join(dir, "images");
|
|
311
|
-
for (const rel of readdirSync(imagesDir, { recursive: true })) {
|
|
312
|
-
const full = join(imagesDir, rel);
|
|
313
|
-
if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split("\\").join("/")}`);
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
zip.end();
|
|
317
|
-
await once(zipOut, "finish");
|
|
318
|
-
const store = getBlobStore();
|
|
319
|
-
const stored = [];
|
|
320
|
-
for (const f of [...artifactFiles, { name: "bundle.zip", type: "application/zip" }]) {
|
|
321
|
-
stored.push(await store.put(`extract/${job.id}/${f.name}`, readFileSync(join(dir, f.name)), f.type));
|
|
322
|
-
}
|
|
323
|
-
return stored;
|
|
324
|
-
} finally {
|
|
325
|
-
rmSync(dir, { recursive: true, force: true });
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
export {
|
|
329
|
-
assembleExtractArtifacts
|
|
330
|
-
};
|
|
331
|
-
//# sourceMappingURL=extract-bundle-GPZRLBVY.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/api/extract-bundle.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'\nimport { basename, join } from 'node:path'\nimport { tmpdir } from 'node:os'\nimport { createGzip } from 'node:zlib'\nimport { once } from 'node:events'\nimport { ZipFile } from 'yazl'\nimport pLimit from 'p-limit'\nimport { getDb } from './db.js'\nimport { getBlobStore, type StoredArtifact } from './blob-store.js'\nimport { computeIssues, renderIssueReport } from './seo-issues.js'\nimport { renderLinkReport, type ExtDomainRow, type LinkReport } from './seo-link-report.js'\nimport type { PageLinkMetrics } from './seo-link-graph.js'\nimport type { SeoAudit } from './seo-audit.js'\nimport type { ExtractJobRow } from './site-extract-repository.js'\nimport type { PageData } from './site-extractor.js'\nimport { renderImageSection, type ImageAudit } from './image-audit.js'\nimport { downloadAsset } from '../lib/media-extractor.js'\n\nconst CHUNK = 400\nconst MAX_IMAGES_PER_PAGE = 20\nconst MAX_IMAGES_PER_SITE = 500\nconst IMAGE_DOWNLOAD_CONCURRENCY = 8\n\nfunction normalize(u: string): string {\n return u.split('#')[0].replace(/\\/$/, '')\n}\n\nfunction registrableDomain(host: string): string {\n const h = host.replace(/^www\\./, '').toLowerCase()\n const parts = h.split('.')\n return parts.length <= 2 ? h : parts.slice(-2).join('.')\n}\n\nfunction safeImageFilename(url: string, index: number): string {\n try {\n const u = new URL(url)\n const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n return base || `image-${index}`\n } catch {\n return `image-${index}`\n }\n}\n\nfunction slugFactory() {\n const counts = new Map<string, number>()\n return (url: string): string => {\n const base = url.replace(/^https?:\\/\\//, '').replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80) || 'page'\n const n = counts.get(base) ?? 0\n counts.set(base, n + 1)\n return n ? `${base}-${n}` : base\n }\n}\n\nasync function* pageChunks(jobId: string): AsyncGenerator<PageData[]> {\n const db = getDb()\n for (let offset = 0; ; offset += CHUNK) {\n const res = await db.execute({\n sql: `SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid LIMIT ${CHUNK} OFFSET ?`,\n args: [jobId, offset],\n })\n if (!res.rows.length) return\n yield res.rows.map(r => JSON.parse(String((r as unknown as Record<string, unknown>).page)) as PageData)\n }\n}\n\nexport interface AssembleExtras {\n branding?: unknown\n imageAudit?: ImageAudit | null\n seoAudit?: SeoAudit | null\n}\n\nexport async function assembleExtractArtifacts(job: ExtractJobRow, extras: AssembleExtras = {}): Promise<StoredArtifact[]> {\n const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`)\n const downloadImages = job.options.downloadImages === true\n mkdirSync(join(dir, 'pages'), { recursive: true })\n if (downloadImages) mkdirSync(join(dir, 'images'), { recursive: true })\n try {\n const metas: PageData[] = []\n const statusByUrl = new Map<string, number | null>()\n for await (const chunk of pageChunks(job.id)) {\n for (const p of chunk) {\n statusByUrl.set(normalize(p.url), p.status)\n const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...slim } = p\n metas.push(slim as unknown as PageData)\n }\n }\n\n const gz = createGzip()\n const linksOut = createWriteStream(join(dir, 'links.jsonl.gz'))\n gz.pipe(linksOut)\n const gzWrite = async (line: string) => { if (!gz.write(line)) await once(gz, 'drain') }\n\n const slug = slugFactory()\n const inboundFrom = new Map<string, Set<string>>()\n const anchorCounts = new Map<string, Map<string, number>>()\n const adjacency = new Map<string, Set<string>>()\n const outCounts = new Map<string, { int: number; ext: number }>()\n const domMap = new Map<string, { links: number; nofollow: number; pages: Set<string> }>()\n const brokenLinkPages = new Set<string>()\n let internalTotal = 0\n let externalTotal = 0\n let brokenInternal = 0\n let pagesWithBody = 0\n let siteReg = ''\n try { siteReg = registrableDomain(new URL(job.startUrl).hostname) } catch { siteReg = '' }\n\n const imageLimit = pLimit(IMAGE_DOWNLOAD_CONCURRENCY)\n const imageDownloads: Promise<void>[] = []\n let imagesQueued = 0\n let imagesFailed = 0\n\n for await (const chunk of pageChunks(job.id)) {\n for (const p of chunk) {\n if (p.bodyMarkdown) {\n pagesWithBody++\n writeFileSync(join(dir, 'pages', slug(p.url) + '.md'),\n `# ${p.title ?? p.url}\\n\\nURL: ${p.url}\\nStatus: ${p.status}\\n\\n---\\n\\n${p.bodyMarkdown}\\n`)\n }\n if (downloadImages && p.imageLinks?.length && imagesQueued < MAX_IMAGES_PER_SITE) {\n const pageDir = join(dir, 'images', slug(p.url))\n mkdirSync(pageDir, { recursive: true })\n for (const [idx, imgUrl] of p.imageLinks.slice(0, MAX_IMAGES_PER_PAGE).entries()) {\n if (imagesQueued >= MAX_IMAGES_PER_SITE) break\n imagesQueued++\n const filename = safeImageFilename(imgUrl, idx)\n imageDownloads.push(\n imageLimit(() => downloadAsset(imgUrl, pageDir, filename)).then(() => undefined, () => { imagesFailed++ }),\n )\n }\n }\n const from = normalize(p.url)\n const oc = { int: 0, ext: 0 }\n for (const l of p.outlinks ?? []) {\n const nofollow = (l.rel ?? '').split(/\\s+/).includes('nofollow')\n const to = normalize(l.href)\n const targetStatus = statusByUrl.get(to) ?? null\n await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + '\\n')\n if (l.internal) {\n oc.int++\n internalTotal++\n if (!inboundFrom.has(to)) inboundFrom.set(to, new Set())\n inboundFrom.get(to)!.add(from)\n if (l.anchor) {\n if (!anchorCounts.has(to)) anchorCounts.set(to, new Map())\n const ac = anchorCounts.get(to)!\n ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1)\n }\n if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {\n if (!adjacency.has(from)) adjacency.set(from, new Set())\n adjacency.get(from)!.add(to)\n }\n if (targetStatus != null && targetStatus >= 400) {\n brokenInternal++\n brokenLinkPages.add(p.url)\n }\n } else {\n oc.ext++\n let domain = ''\n try { domain = registrableDomain(new URL(l.href).hostname) } catch { continue }\n if (!domain || domain === siteReg) continue\n externalTotal++\n const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: new Set<string>() }\n d.links++\n if (nofollow) d.nofollow++\n d.pages.add(p.url)\n domMap.set(domain, d)\n }\n }\n outCounts.set(p.url, oc)\n }\n }\n gz.end()\n await once(linksOut, 'finish')\n await Promise.all(imageDownloads)\n\n const depth = new Map<string, number>()\n const start = normalize(job.startUrl)\n depth.set(start, 0)\n const queue: string[] = [start]\n while (queue.length > 0) {\n const cur = queue.shift()!\n const d = depth.get(cur)!\n for (const next of adjacency.get(cur) ?? []) {\n if (!depth.has(next)) { depth.set(next, d + 1); queue.push(next) }\n }\n }\n\n const metrics = new Map<string, PageLinkMetrics>()\n for (const p of metas) {\n const key = normalize(p.url)\n const inbound = inboundFrom.get(key) ?? new Set()\n const ac = anchorCounts.get(key)\n const topAnchors = ac\n ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(e => e[0])\n : []\n const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 }\n metrics.set(p.url, {\n url: p.url,\n inlinks: inbound.size,\n uniqueInlinks: inbound.size,\n outlinksInternal: oc.int,\n outlinksExternal: oc.ext,\n crawlDepth: depth.has(key) ? depth.get(key)! : null,\n orphan: inbound.size === 0 && key !== start,\n topAnchors,\n })\n }\n\n const issues = computeIssues(metas, metrics, brokenLinkPages)\n let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics)\n if (extras.imageAudit) reportMd += `\\n\\n${renderImageSection(extras.imageAudit)}`\n\n const metricValues = [...metrics.values()]\n const round = (n: number) => Math.round(n * 10) / 10\n const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 }\n let sumInlinks = 0\n let sumOutlinks = 0\n for (const m of metricValues) {\n sumInlinks += m.inlinks\n sumOutlinks += m.outlinksInternal + m.outlinksExternal\n if (m.inlinks === 0) distribution.zero++\n else if (m.inlinks <= 2) distribution.oneToTwo++\n else if (m.inlinks <= 10) distribution.threeToTen++\n else distribution.elevenPlus++\n }\n const externalDomains: ExtDomainRow[] = [...domMap.entries()]\n .map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size }))\n .sort((a, b) => b.links - a.links)\n const linkReport: LinkReport = {\n summary: {\n internal: {\n totalLinks: internalTotal,\n pages: metas.length,\n orphans: metricValues.filter(m => m.orphan).length,\n brokenInternal,\n avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,\n avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,\n distribution,\n topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20)\n .map(m => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal })),\n },\n external: {\n totalLinks: externalTotal,\n uniqueDomains: externalDomains.length,\n topDomains: externalDomains.slice(0, 20),\n },\n },\n externalDomains,\n }\n\n const pagesOut = createWriteStream(join(dir, 'pages.jsonl'))\n for (const p of metas) {\n const m = metrics.get(p.url)\n const row = { ...(p as unknown as Record<string, unknown>), inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false }\n if (!pagesOut.write(JSON.stringify(row) + '\\n')) await once(pagesOut, 'drain')\n }\n pagesOut.end()\n await once(pagesOut, 'finish')\n\n writeFileSync(join(dir, 'report.md'), reportMd)\n writeFileSync(join(dir, 'issues.json'), JSON.stringify(issues, null, 2))\n writeFileSync(join(dir, 'link-metrics.jsonl'), metricValues.map(m => JSON.stringify(m)).join('\\n'))\n writeFileSync(join(dir, 'link-report.md'), renderLinkReport(linkReport))\n writeFileSync(join(dir, 'links-summary.json'), JSON.stringify(linkReport.summary, null, 2))\n writeFileSync(join(dir, 'external-domains.json'), JSON.stringify(linkReport.externalDomains, null, 2))\n if (extras.imageAudit) {\n writeFileSync(join(dir, 'images.jsonl'), extras.imageAudit.rows.map(r => JSON.stringify(r)).join('\\n'))\n writeFileSync(join(dir, 'images-summary.json'), JSON.stringify(extras.imageAudit.summary, null, 2))\n }\n if (extras.seoAudit) writeFileSync(join(dir, 'seo-audit.json'), JSON.stringify(extras.seoAudit, null, 2))\n if (extras.branding) writeFileSync(join(dir, 'branding.json'), JSON.stringify(extras.branding, null, 2))\n if (downloadImages) {\n writeFileSync(join(dir, 'images-download-summary.json'), JSON.stringify({\n queued: imagesQueued,\n downloaded: imagesQueued - imagesFailed,\n failed: imagesFailed,\n perPageCap: MAX_IMAGES_PER_PAGE,\n perSiteCap: MAX_IMAGES_PER_SITE,\n }, null, 2))\n }\n\n const artifactFiles: Array<{ name: string; type: string }> = [\n { name: 'report.md', type: 'text/markdown' },\n { name: 'issues.json', type: 'application/json' },\n { name: 'pages.jsonl', type: 'application/x-ndjson' },\n { name: 'links.jsonl.gz', type: 'application/gzip' },\n { name: 'link-metrics.jsonl', type: 'application/x-ndjson' },\n { name: 'link-report.md', type: 'text/markdown' },\n { name: 'links-summary.json', type: 'application/json' },\n { name: 'external-domains.json', type: 'application/json' },\n ]\n if (extras.imageAudit) {\n artifactFiles.push({ name: 'images.jsonl', type: 'application/x-ndjson' })\n artifactFiles.push({ name: 'images-summary.json', type: 'application/json' })\n }\n if (extras.seoAudit) artifactFiles.push({ name: 'seo-audit.json', type: 'application/json' })\n if (extras.branding) artifactFiles.push({ name: 'branding.json', type: 'application/json' })\n if (downloadImages) artifactFiles.push({ name: 'images-download-summary.json', type: 'application/json' })\n\n const zip = new ZipFile()\n const zipOut = createWriteStream(join(dir, 'bundle.zip'))\n zip.outputStream.pipe(zipOut)\n for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name)\n if (pagesWithBody > 0) {\n for (const f of readdirSync(join(dir, 'pages'))) zip.addFile(join(dir, 'pages', f), `pages/${f}`)\n }\n if (downloadImages && imagesQueued > imagesFailed) {\n const imagesDir = join(dir, 'images')\n for (const rel of readdirSync(imagesDir, { recursive: true }) as string[]) {\n const full = join(imagesDir, rel)\n if (statSync(full).isFile()) zip.addFile(full, `images/${rel.split('\\\\').join('/')}`)\n }\n }\n zip.end()\n await once(zipOut, 'finish')\n\n const store = getBlobStore()\n const stored: StoredArtifact[] = []\n for (const f of [...artifactFiles, { name: 'bundle.zip', type: 'application/zip' }]) {\n stored.push(await store.put(`extract/${job.id}/${f.name}`, readFileSync(join(dir, f.name)), f.type))\n }\n return stored\n } finally {\n rmSync(dir, { recursive: true, force: true })\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc,aAAa,QAAQ,UAAU,qBAAqB;AACzG,SAAS,UAAU,YAAY;AAC/B,SAAS,cAAc;AACvB,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AACrB,SAAS,eAAe;AACxB,OAAO,YAAY;AAYnB,IAAM,QAAQ;AACd,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAC5B,IAAM,6BAA6B;AAEnC,SAAS,UAAU,GAAmB;AACpC,SAAO,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,OAAO,EAAE;AAC1C;AAEA,SAAS,kBAAkB,MAAsB;AAC/C,QAAM,IAAI,KAAK,QAAQ,UAAU,EAAE,EAAE,YAAY;AACjD,QAAM,QAAQ,EAAE,MAAM,GAAG;AACzB,SAAO,MAAM,UAAU,IAAI,IAAI,MAAM,MAAM,EAAE,EAAE,KAAK,GAAG;AACzD;AAEA,SAAS,kBAAkB,KAAa,OAAuB;AAC7D,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEA,SAAS,cAAc;AACrB,QAAM,SAAS,oBAAI,IAAoB;AACvC,SAAO,CAAC,QAAwB;AAC9B,UAAM,OAAO,IAAI,QAAQ,gBAAgB,EAAE,EAAE,QAAQ,kBAAkB,GAAG,EAAE,QAAQ,YAAY,EAAE,EAAE,MAAM,GAAG,EAAE,KAAK;AACpH,UAAM,IAAI,OAAO,IAAI,IAAI,KAAK;AAC9B,WAAO,IAAI,MAAM,IAAI,CAAC;AACtB,WAAO,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK;AAAA,EAC9B;AACF;AAEA,gBAAgB,WAAW,OAA2C;AACpE,QAAM,KAAK,MAAM;AACjB,WAAS,SAAS,KAAK,UAAU,OAAO;AACtC,UAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,MAC3B,KAAK,6EAA6E,KAAK;AAAA,MACvF,MAAM,CAAC,OAAO,MAAM;AAAA,IACtB,CAAC;AACD,QAAI,CAAC,IAAI,KAAK,OAAQ;AACtB,UAAM,IAAI,KAAK,IAAI,OAAK,KAAK,MAAM,OAAQ,EAAyC,IAAI,CAAC,CAAa;AAAA,EACxG;AACF;AAQA,eAAsB,yBAAyB,KAAoB,SAAyB,CAAC,GAA8B;AACzH,QAAM,MAAM,KAAK,OAAO,GAAG,kBAAkB,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AACnE,QAAM,iBAAiB,IAAI,QAAQ,mBAAmB;AACtD,YAAU,KAAK,KAAK,OAAO,GAAG,EAAE,WAAW,KAAK,CAAC;AACjD,MAAI,eAAgB,WAAU,KAAK,KAAK,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;AACtE,MAAI;AACF,UAAM,QAAoB,CAAC;AAC3B,UAAM,cAAc,oBAAI,IAA2B;AACnD,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,oBAAY,IAAI,UAAU,EAAE,GAAG,GAAG,EAAE,MAAM;AAC1C,cAAM,EAAE,cAAc,IAAI,QAAQ,IAAI,UAAU,IAAI,GAAG,KAAK,IAAI;AAChE,cAAM,KAAK,IAA2B;AAAA,MACxC;AAAA,IACF;AAEA,UAAM,KAAK,WAAW;AACtB,UAAM,WAAW,kBAAkB,KAAK,KAAK,gBAAgB,CAAC;AAC9D,OAAG,KAAK,QAAQ;AAChB,UAAM,UAAU,OAAO,SAAiB;AAAE,UAAI,CAAC,GAAG,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI,OAAO;AAAA,IAAE;AAEvF,UAAM,OAAO,YAAY;AACzB,UAAM,cAAc,oBAAI,IAAyB;AACjD,UAAM,eAAe,oBAAI,IAAiC;AAC1D,UAAM,YAAY,oBAAI,IAAyB;AAC/C,UAAM,YAAY,oBAAI,IAA0C;AAChE,UAAM,SAAS,oBAAI,IAAqE;AACxF,UAAM,kBAAkB,oBAAI,IAAY;AACxC,QAAI,gBAAgB;AACpB,QAAI,gBAAgB;AACpB,QAAI,iBAAiB;AACrB,QAAI,gBAAgB;AACpB,QAAI,UAAU;AACd,QAAI;AAAE,gBAAU,kBAAkB,IAAI,IAAI,IAAI,QAAQ,EAAE,QAAQ;AAAA,IAAE,QAAQ;AAAE,gBAAU;AAAA,IAAG;AAEzF,UAAM,aAAa,OAAO,0BAA0B;AACpD,UAAM,iBAAkC,CAAC;AACzC,QAAI,eAAe;AACnB,QAAI,eAAe;AAEnB,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,YAAI,EAAE,cAAc;AAClB;AACA;AAAA,YAAc,KAAK,KAAK,SAAS,KAAK,EAAE,GAAG,IAAI,KAAK;AAAA,YAClD,KAAK,EAAE,SAAS,EAAE,GAAG;AAAA;AAAA,OAAY,EAAE,GAAG;AAAA,UAAa,EAAE,MAAM;AAAA;AAAA;AAAA;AAAA,EAAc,EAAE,YAAY;AAAA;AAAA,UAAI;AAAA,QAC/F;AACA,YAAI,kBAAkB,EAAE,YAAY,UAAU,eAAe,qBAAqB;AAChF,gBAAM,UAAU,KAAK,KAAK,UAAU,KAAK,EAAE,GAAG,CAAC;AAC/C,oBAAU,SAAS,EAAE,WAAW,KAAK,CAAC;AACtC,qBAAW,CAAC,KAAK,MAAM,KAAK,EAAE,WAAW,MAAM,GAAG,mBAAmB,EAAE,QAAQ,GAAG;AAChF,gBAAI,gBAAgB,oBAAqB;AACzC;AACA,kBAAM,WAAW,kBAAkB,QAAQ,GAAG;AAC9C,2BAAe;AAAA,cACb,WAAW,MAAM,cAAc,QAAQ,SAAS,QAAQ,CAAC,EAAE,KAAK,MAAM,QAAW,MAAM;AAAE;AAAA,cAAe,CAAC;AAAA,YAC3G;AAAA,UACF;AAAA,QACF;AACA,cAAM,OAAO,UAAU,EAAE,GAAG;AAC5B,cAAM,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AAC5B,mBAAW,KAAK,EAAE,YAAY,CAAC,GAAG;AAChC,gBAAM,YAAY,EAAE,OAAO,IAAI,MAAM,KAAK,EAAE,SAAS,UAAU;AAC/D,gBAAM,KAAK,UAAU,EAAE,IAAI;AAC3B,gBAAM,eAAe,YAAY,IAAI,EAAE,KAAK;AAC5C,gBAAM,QAAQ,KAAK,UAAU,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,UAAU,EAAE,UAAU,UAAU,aAAa,CAAC,IAAI,IAAI;AAC5I,cAAI,EAAE,UAAU;AACd,eAAG;AACH;AACA,gBAAI,CAAC,YAAY,IAAI,EAAE,EAAG,aAAY,IAAI,IAAI,oBAAI,IAAI,CAAC;AACvD,wBAAY,IAAI,EAAE,EAAG,IAAI,IAAI;AAC7B,gBAAI,EAAE,QAAQ;AACZ,kBAAI,CAAC,aAAa,IAAI,EAAE,EAAG,cAAa,IAAI,IAAI,oBAAI,IAAI,CAAC;AACzD,oBAAM,KAAK,aAAa,IAAI,EAAE;AAC9B,iBAAG,IAAI,EAAE,SAAS,GAAG,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC;AAAA,YAC9C;AACA,gBAAI,CAAC,aAAa,YAAY,IAAI,EAAE,MAAM,OAAO,CAAC,YAAY,IAAI,EAAE,IAAI;AACtE,kBAAI,CAAC,UAAU,IAAI,IAAI,EAAG,WAAU,IAAI,MAAM,oBAAI,IAAI,CAAC;AACvD,wBAAU,IAAI,IAAI,EAAG,IAAI,EAAE;AAAA,YAC7B;AACA,gBAAI,gBAAgB,QAAQ,gBAAgB,KAAK;AAC/C;AACA,8BAAgB,IAAI,EAAE,GAAG;AAAA,YAC3B;AAAA,UACF,OAAO;AACL,eAAG;AACH,gBAAI,SAAS;AACb,gBAAI;AAAE,uBAAS,kBAAkB,IAAI,IAAI,EAAE,IAAI,EAAE,QAAQ;AAAA,YAAE,QAAQ;AAAE;AAAA,YAAS;AAC9E,gBAAI,CAAC,UAAU,WAAW,QAAS;AACnC;AACA,kBAAM,IAAI,OAAO,IAAI,MAAM,KAAK,EAAE,OAAO,GAAG,UAAU,GAAG,OAAO,oBAAI,IAAY,EAAE;AAClF,cAAE;AACF,gBAAI,SAAU,GAAE;AAChB,cAAE,MAAM,IAAI,EAAE,GAAG;AACjB,mBAAO,IAAI,QAAQ,CAAC;AAAA,UACtB;AAAA,QACF;AACA,kBAAU,IAAI,EAAE,KAAK,EAAE;AAAA,MACzB;AAAA,IACF;AACA,OAAG,IAAI;AACP,UAAM,KAAK,UAAU,QAAQ;AAC7B,UAAM,QAAQ,IAAI,cAAc;AAEhC,UAAM,QAAQ,oBAAI,IAAoB;AACtC,UAAM,QAAQ,UAAU,IAAI,QAAQ;AACpC,UAAM,IAAI,OAAO,CAAC;AAClB,UAAM,QAAkB,CAAC,KAAK;AAC9B,WAAO,MAAM,SAAS,GAAG;AACvB,YAAM,MAAM,MAAM,MAAM;AACxB,YAAM,IAAI,MAAM,IAAI,GAAG;AACvB,iBAAW,QAAQ,UAAU,IAAI,GAAG,KAAK,CAAC,GAAG;AAC3C,YAAI,CAAC,MAAM,IAAI,IAAI,GAAG;AAAE,gBAAM,IAAI,MAAM,IAAI,CAAC;AAAG,gBAAM,KAAK,IAAI;AAAA,QAAE;AAAA,MACnE;AAAA,IACF;AAEA,UAAM,UAAU,oBAAI,IAA6B;AACjD,eAAW,KAAK,OAAO;AACrB,YAAM,MAAM,UAAU,EAAE,GAAG;AAC3B,YAAM,UAAU,YAAY,IAAI,GAAG,KAAK,oBAAI,IAAI;AAChD,YAAM,KAAK,aAAa,IAAI,GAAG;AAC/B,YAAM,aAAa,KACf,CAAC,GAAG,GAAG,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,IAAI,OAAK,EAAE,CAAC,CAAC,IACvE,CAAC;AACL,YAAM,KAAK,UAAU,IAAI,EAAE,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AACpD,cAAQ,IAAI,EAAE,KAAK;AAAA,QACjB,KAAK,EAAE;AAAA,QACP,SAAS,QAAQ;AAAA,QACjB,eAAe,QAAQ;AAAA,QACvB,kBAAkB,GAAG;AAAA,QACrB,kBAAkB,GAAG;AAAA,QACrB,YAAY,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,IAAK;AAAA,QAC/C,QAAQ,QAAQ,SAAS,KAAK,QAAQ;AAAA,QACtC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,cAAc,OAAO,SAAS,eAAe;AAC5D,QAAI,WAAW,kBAAkB,IAAI,UAAU,OAAO,QAAQ,OAAO;AACrE,QAAI,OAAO,WAAY,aAAY;AAAA;AAAA,EAAO,mBAAmB,OAAO,UAAU,CAAC;AAE/E,UAAM,eAAe,CAAC,GAAG,QAAQ,OAAO,CAAC;AACzC,UAAM,QAAQ,CAAC,MAAc,KAAK,MAAM,IAAI,EAAE,IAAI;AAClD,UAAM,eAAe,EAAE,MAAM,GAAG,UAAU,GAAG,YAAY,GAAG,YAAY,EAAE;AAC1E,QAAI,aAAa;AACjB,QAAI,cAAc;AAClB,eAAW,KAAK,cAAc;AAC5B,oBAAc,EAAE;AAChB,qBAAe,EAAE,mBAAmB,EAAE;AACtC,UAAI,EAAE,YAAY,EAAG,cAAa;AAAA,eACzB,EAAE,WAAW,EAAG,cAAa;AAAA,eAC7B,EAAE,WAAW,GAAI,cAAa;AAAA,UAClC,cAAa;AAAA,IACpB;AACA,UAAM,kBAAkC,CAAC,GAAG,OAAO,QAAQ,CAAC,EACzD,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,OAAO,EAAE,MAAM,KAAK,EAAE,EAC5F,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACnC,UAAM,aAAyB;AAAA,MAC7B,SAAS;AAAA,QACP,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,OAAO,MAAM;AAAA,UACb,SAAS,aAAa,OAAO,OAAK,EAAE,MAAM,EAAE;AAAA,UAC5C;AAAA,UACA,YAAY,MAAM,SAAS,MAAM,aAAa,MAAM,MAAM,IAAI;AAAA,UAC9D,aAAa,MAAM,SAAS,MAAM,cAAc,MAAM,MAAM,IAAI;AAAA,UAChE;AAAA,UACA,cAAc,CAAC,GAAG,YAAY,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,EAAE,EAC9E,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,SAAS,EAAE,SAAS,kBAAkB,EAAE,kBAAkB,kBAAkB,EAAE,iBAAiB,EAAE;AAAA,QAC9H;AAAA,QACA,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,eAAe,gBAAgB;AAAA,UAC/B,YAAY,gBAAgB,MAAM,GAAG,EAAE;AAAA,QACzC;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,UAAM,WAAW,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAC3D,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,IAAI,EAAE,GAAG;AAC3B,YAAM,MAAM,EAAE,GAAI,GAA0C,SAAS,GAAG,WAAW,GAAG,YAAY,GAAG,cAAc,MAAM,QAAQ,GAAG,UAAU,MAAM;AACpJ,UAAI,CAAC,SAAS,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,EAAG,OAAM,KAAK,UAAU,OAAO;AAAA,IAC/E;AACA,aAAS,IAAI;AACb,UAAM,KAAK,UAAU,QAAQ;AAE7B,kBAAc,KAAK,KAAK,WAAW,GAAG,QAAQ;AAC9C,kBAAc,KAAK,KAAK,aAAa,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,aAAa,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAClG,kBAAc,KAAK,KAAK,gBAAgB,GAAG,iBAAiB,UAAU,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,KAAK,UAAU,WAAW,SAAS,MAAM,CAAC,CAAC;AAC1F,kBAAc,KAAK,KAAK,uBAAuB,GAAG,KAAK,UAAU,WAAW,iBAAiB,MAAM,CAAC,CAAC;AACrG,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,KAAK,cAAc,GAAG,OAAO,WAAW,KAAK,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACtG,oBAAc,KAAK,KAAK,qBAAqB,GAAG,KAAK,UAAU,OAAO,WAAW,SAAS,MAAM,CAAC,CAAC;AAAA,IACpG;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,gBAAgB,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACxG,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,eAAe,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AACvG,QAAI,gBAAgB;AAClB,oBAAc,KAAK,KAAK,8BAA8B,GAAG,KAAK,UAAU;AAAA,QACtE,QAAQ;AAAA,QACR,YAAY,eAAe;AAAA,QAC3B,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,YAAY;AAAA,MACd,GAAG,MAAM,CAAC,CAAC;AAAA,IACb;AAEA,UAAM,gBAAuD;AAAA,MAC3D,EAAE,MAAM,aAAa,MAAM,gBAAgB;AAAA,MAC3C,EAAE,MAAM,eAAe,MAAM,mBAAmB;AAAA,MAChD,EAAE,MAAM,eAAe,MAAM,uBAAuB;AAAA,MACpD,EAAE,MAAM,kBAAkB,MAAM,mBAAmB;AAAA,MACnD,EAAE,MAAM,sBAAsB,MAAM,uBAAuB;AAAA,MAC3D,EAAE,MAAM,kBAAkB,MAAM,gBAAgB;AAAA,MAChD,EAAE,MAAM,sBAAsB,MAAM,mBAAmB;AAAA,MACvD,EAAE,MAAM,yBAAyB,MAAM,mBAAmB;AAAA,IAC5D;AACA,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,EAAE,MAAM,gBAAgB,MAAM,uBAAuB,CAAC;AACzE,oBAAc,KAAK,EAAE,MAAM,uBAAuB,MAAM,mBAAmB,CAAC;AAAA,IAC9E;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,kBAAkB,MAAM,mBAAmB,CAAC;AAC5F,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,iBAAiB,MAAM,mBAAmB,CAAC;AAC3F,QAAI,eAAgB,eAAc,KAAK,EAAE,MAAM,gCAAgC,MAAM,mBAAmB,CAAC;AAEzG,UAAM,MAAM,IAAI,QAAQ;AACxB,UAAM,SAAS,kBAAkB,KAAK,KAAK,YAAY,CAAC;AACxD,QAAI,aAAa,KAAK,MAAM;AAC5B,eAAW,KAAK,cAAe,KAAI,QAAQ,KAAK,KAAK,EAAE,IAAI,GAAG,EAAE,IAAI;AACpE,QAAI,gBAAgB,GAAG;AACrB,iBAAW,KAAK,YAAY,KAAK,KAAK,OAAO,CAAC,EAAG,KAAI,QAAQ,KAAK,KAAK,SAAS,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAClG;AACA,QAAI,kBAAkB,eAAe,cAAc;AACjD,YAAM,YAAY,KAAK,KAAK,QAAQ;AACpC,iBAAW,OAAO,YAAY,WAAW,EAAE,WAAW,KAAK,CAAC,GAAe;AACzE,cAAM,OAAO,KAAK,WAAW,GAAG;AAChC,YAAI,SAAS,IAAI,EAAE,OAAO,EAAG,KAAI,QAAQ,MAAM,UAAU,IAAI,MAAM,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE;AAAA,MACtF;AAAA,IACF;AACA,QAAI,IAAI;AACR,UAAM,KAAK,QAAQ,QAAQ;AAE3B,UAAM,QAAQ,aAAa;AAC3B,UAAM,SAA2B,CAAC;AAClC,eAAW,KAAK,CAAC,GAAG,eAAe,EAAE,MAAM,cAAc,MAAM,kBAAkB,CAAC,GAAG;AACnF,aAAO,KAAK,MAAM,MAAM,IAAI,WAAW,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,aAAa,KAAK,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC;AAAA,IACrG;AACA,WAAO;AAAA,EACT,UAAE;AACA,WAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAAA,EAC9C;AACF;","names":[]}
|