mdld-parse 1.0.8 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +250 -82
- package/dist/crawl.js +660 -0
- package/dist/{index.js → mdld-parse.js} +21 -16
- package/dist/quad-graph.js +1219 -0
- package/package.json +8 -4
package/dist/crawl.js
ADDED
|
@@ -0,0 +1,660 @@
|
|
|
1
|
+
//#region src/crawl.js
|
|
2
|
+
/**
|
|
3
|
+
* crawl — a platform-agnostic linked-text-document crawler.
|
|
4
|
+
*
|
|
5
|
+
* Fetches raw text starting from one URL, follows markdown `[label](url)`
|
|
6
|
+
* and HTML `<a href="url">` links (both valid in CommonMark inline HTML, so
|
|
7
|
+
* both extractors run on every document), and returns pages in deterministic
|
|
8
|
+
* BFS discovery order.
|
|
9
|
+
*
|
|
10
|
+
* ZERO dependencies and ZERO internal imports: this module is fully
|
|
11
|
+
* self-contained and can be published standalone or vendored verbatim.
|
|
12
|
+
* Nothing here knows about MD-LD — the consumer decides how to interpret
|
|
13
|
+
* the text (see "Consuming with mdld-parse" below).
|
|
14
|
+
*
|
|
15
|
+
* ── Cache ────────────────────────────────────────────────────────────────
|
|
16
|
+
* Inject any adapter implementing:
|
|
17
|
+
* get(url) → Promise<{text, contentType, finalUrl, etag, lastModified, timestamp} | null>
|
|
18
|
+
* set(url, e) → Promise<void>
|
|
19
|
+
* delete?(url)→ Promise<void>
|
|
20
|
+
* Cache failures are swallowed: a broken cache degrades to "no cache",
|
|
21
|
+
* never to a failed crawl. Default: in-process LRU (`memoryCache`).
|
|
22
|
+
* Conditional requests reuse stored ETag/Last-Modified. fetch runs with
|
|
23
|
+
* `cache: 'no-store'` so this module's cache is the single authority.
|
|
24
|
+
*
|
|
25
|
+
* ── Guarantees ───────────────────────────────────────────────────────────
|
|
26
|
+
* - Deterministic: pages return in BFS discovery order, never completion order.
|
|
27
|
+
* - Partial-result-safe: HTTP/network failures land in `errors`; the crawl
|
|
28
|
+
* continues. 5xx/network failure with a cached copy → stale-if-error.
|
|
29
|
+
* - Cancellable: pass `signal`; resolves with `aborted: true` plus partial results.
|
|
30
|
+
* - Bounded: `maxDepth`, `maxPages` (default 100), optional `sameOrigin`,
|
|
31
|
+
* http(s)-only protocol allow-list, binary content-types rejected.
|
|
32
|
+
*
|
|
33
|
+
* @example
|
|
34
|
+
* // Generic use: crawl a docs site, get text + edges
|
|
35
|
+
* const { pages, errors, aborted } = await crawl("/docs/index.md", { sameOrigin: true });
|
|
36
|
+
* for (const p of pages) console.log(p.url, "→", p.links.length, "outgoing links");
|
|
37
|
+
*/
|
|
38
|
+
var HTTP_PROTOCOLS = /* @__PURE__ */ new Set(["http:", "https:"]);
|
|
39
|
+
var ACCEPT = "text/markdown, text/x-mdld, text/plain;q=0.9, text/html;q=0.8, application/xhtml+xml;q=0.8, */*;q=0.5";
|
|
40
|
+
var DEFAULT_MAX_DEPTH = 5;
|
|
41
|
+
var DEFAULT_CONCURRENCY = 5;
|
|
42
|
+
var DEFAULT_MAX_PAGES = 100;
|
|
43
|
+
var DEFAULT_ACCEPT_RE = /\.(md|mdld|markdown|html?|xhtml)$/i;
|
|
44
|
+
var BINARY_CT = /image\/|audio\/|video\/|font\/|application\/(octet-stream|zip|gzip|pdf|wasm|protobuf)/i;
|
|
45
|
+
/**
|
|
46
|
+
* In-process LRU cache. O(1) get/set, recency refreshed on read.
|
|
47
|
+
* @param {number} [maxEntries=500]
|
|
48
|
+
*/
|
|
49
|
+
function memoryCache(maxEntries = 500) {
|
|
50
|
+
const map = /* @__PURE__ */ new Map();
|
|
51
|
+
return {
|
|
52
|
+
async get(url) {
|
|
53
|
+
const entry = map.get(url);
|
|
54
|
+
if (entry === void 0) return null;
|
|
55
|
+
map.delete(url);
|
|
56
|
+
map.set(url, entry);
|
|
57
|
+
return entry;
|
|
58
|
+
},
|
|
59
|
+
async set(url, entry) {
|
|
60
|
+
if (map.has(url)) map.delete(url);
|
|
61
|
+
map.set(url, entry);
|
|
62
|
+
if (map.size > maxEntries) map.delete(map.keys().next().value);
|
|
63
|
+
},
|
|
64
|
+
async delete(url) {
|
|
65
|
+
map.delete(url);
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/** Resolve + validate. Absolute http(s) URL without fragment, or null. */
|
|
70
|
+
function normalizeUrl(raw, base) {
|
|
71
|
+
if (raw == null) return null;
|
|
72
|
+
const s = String(raw).trim();
|
|
73
|
+
if (!s) return null;
|
|
74
|
+
try {
|
|
75
|
+
const url = new URL(s, base || void 0);
|
|
76
|
+
if (!HTTP_PROTOCOLS.has(url.protocol)) return null;
|
|
77
|
+
url.hash = "";
|
|
78
|
+
return url.href;
|
|
79
|
+
} catch {
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** Default target filter: docs extensions + extensionless clean URLs. */
|
|
84
|
+
function defaultAccept(url) {
|
|
85
|
+
let file = url.slice(url.lastIndexOf("/") + 1);
|
|
86
|
+
const q = file.indexOf("?");
|
|
87
|
+
if (q !== -1) file = file.slice(0, q);
|
|
88
|
+
if (!file.includes(".")) return true;
|
|
89
|
+
return DEFAULT_ACCEPT_RE.test(file);
|
|
90
|
+
}
|
|
91
|
+
/** Cheap hint for consumers (they decide how to interpret the text). */
|
|
92
|
+
function looksLikeHTML(contentType, text) {
|
|
93
|
+
if (contentType) {
|
|
94
|
+
const ct = contentType.toLowerCase();
|
|
95
|
+
if (ct.includes("html")) return true;
|
|
96
|
+
if (/(markdown|mdld|json|javascript|css)/.test(ct)) return false;
|
|
97
|
+
}
|
|
98
|
+
const head = text.slice(0, 256).trimStart().slice(0, 32).toLowerCase();
|
|
99
|
+
return head.startsWith("<!doctype html") || head.startsWith("<html");
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Extract all http(s) links from a text document (markdown + HTML syntaxes).
|
|
103
|
+
* Character-scanned: fence-aware, code-span-aware, comment-aware, and
|
|
104
|
+
* `<script>`/`<style>`-skipping. Handles inline links, images, autolinks,
|
|
105
|
+
* reference definitions, titles, balanced parens, backslash escapes,
|
|
106
|
+
* HTML entities in hrefs.
|
|
107
|
+
*
|
|
108
|
+
* @param {string} text
|
|
109
|
+
* @param {string} [base] Base URL for relative resolution.
|
|
110
|
+
* @returns {string[]} Absolute http(s) URLs (fragments stripped), deduped, discovery order.
|
|
111
|
+
*/
|
|
112
|
+
function extractLinks(text, base) {
|
|
113
|
+
const out = /* @__PURE__ */ new Set();
|
|
114
|
+
extractMarkdownLinks(text, base, out);
|
|
115
|
+
extractHTMLLinks(text, base, out);
|
|
116
|
+
return [...out];
|
|
117
|
+
}
|
|
118
|
+
function extractMarkdownLinks(text, base, out) {
|
|
119
|
+
let fenceChar = 0;
|
|
120
|
+
let fenceLen = 0;
|
|
121
|
+
let lineStart = 0;
|
|
122
|
+
for (let i = 0; i <= text.length; i++) {
|
|
123
|
+
if (i < text.length && text.charCodeAt(i) !== 10) continue;
|
|
124
|
+
let line = text.slice(lineStart, i);
|
|
125
|
+
lineStart = i + 1;
|
|
126
|
+
if (line.endsWith("\r")) line = line.slice(0, -1);
|
|
127
|
+
const trimmed = line.trimStart();
|
|
128
|
+
const indent = line.length - trimmed.length;
|
|
129
|
+
const c0 = trimmed.charCodeAt(0);
|
|
130
|
+
if (fenceLen > 0) {
|
|
131
|
+
if (c0 === fenceChar) {
|
|
132
|
+
let n = 0;
|
|
133
|
+
while (trimmed.charCodeAt(n) === fenceChar) n++;
|
|
134
|
+
if (n >= fenceLen && trimmed.slice(n).trim() === "") fenceLen = 0;
|
|
135
|
+
}
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (indent <= 3 && (c0 === 96 || c0 === 126)) {
|
|
139
|
+
let n = 0;
|
|
140
|
+
while (trimmed.charCodeAt(n) === c0) n++;
|
|
141
|
+
if (n >= 3) {
|
|
142
|
+
fenceChar = c0;
|
|
143
|
+
fenceLen = n;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
if (indent <= 3 && c0 === 91 && extractRefDefinition(trimmed, base, out)) continue;
|
|
148
|
+
scanInlineLinks(line, base, out);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/** `[label]: url "title"` — CommonMark reference definition. */
|
|
152
|
+
function extractRefDefinition(line, base, out) {
|
|
153
|
+
let close = -1;
|
|
154
|
+
let depth = 0;
|
|
155
|
+
for (let i = 1; i < line.length; i++) {
|
|
156
|
+
const c = line.charCodeAt(i);
|
|
157
|
+
if (c === 92) i++;
|
|
158
|
+
else if (c === 91) depth++;
|
|
159
|
+
else if (c === 93) {
|
|
160
|
+
if (depth === 0) {
|
|
161
|
+
close = i;
|
|
162
|
+
break;
|
|
163
|
+
}
|
|
164
|
+
depth--;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
if (close === -1 || line.charCodeAt(close + 1) !== 58) return false;
|
|
168
|
+
let j = close + 2;
|
|
169
|
+
while (line[j] === " " || line[j] === " ") j++;
|
|
170
|
+
let raw;
|
|
171
|
+
if (line[j] === "<") {
|
|
172
|
+
const gt = line.indexOf(">", j + 1);
|
|
173
|
+
if (gt === -1) return true;
|
|
174
|
+
raw = line.slice(j + 1, gt);
|
|
175
|
+
} else {
|
|
176
|
+
let k = j;
|
|
177
|
+
while (k < line.length && line[k] !== " " && line[k] !== " ") k++;
|
|
178
|
+
raw = line.slice(j, k);
|
|
179
|
+
}
|
|
180
|
+
addLink(raw, base, out);
|
|
181
|
+
return true;
|
|
182
|
+
}
|
|
183
|
+
function scanInlineLinks(line, base, out) {
|
|
184
|
+
let i = 0;
|
|
185
|
+
while (i < line.length) {
|
|
186
|
+
const c = line.charCodeAt(i);
|
|
187
|
+
if (c === 92) {
|
|
188
|
+
i += 2;
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
if (c === 96) {
|
|
192
|
+
let n = 1;
|
|
193
|
+
while (line.charCodeAt(i + n) === 96) n++;
|
|
194
|
+
const close = findBacktickRun(line, i + n, n);
|
|
195
|
+
i = close === -1 ? i + n : close;
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
if (c === 60) {
|
|
199
|
+
if (line.startsWith("<!--", i)) {
|
|
200
|
+
const end = line.indexOf("-->", i + 4);
|
|
201
|
+
if (end === -1) return;
|
|
202
|
+
i = end + 3;
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
const gt = line.indexOf(">", i + 1);
|
|
206
|
+
if (gt !== -1 && isAutolink(line, i + 1, gt)) {
|
|
207
|
+
addLink(line.slice(i + 1, gt), base, out);
|
|
208
|
+
i = gt + 1;
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
i++;
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
if (c === 93 && line.charCodeAt(i + 1) === 40) {
|
|
215
|
+
const m = findLinkEnd(line, i + 2);
|
|
216
|
+
if (m) {
|
|
217
|
+
addLink(line.slice(m.urlStart, m.urlEnd), base, out);
|
|
218
|
+
i = m.end + 1;
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
i++;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
function findBacktickRun(line, from, n) {
|
|
226
|
+
let i = from;
|
|
227
|
+
while (i < line.length) if (line.charCodeAt(i) === 96) {
|
|
228
|
+
let m = 1;
|
|
229
|
+
while (line.charCodeAt(i + m) === 96) m++;
|
|
230
|
+
if (m === n) return i + m;
|
|
231
|
+
i += m;
|
|
232
|
+
} else i++;
|
|
233
|
+
return -1;
|
|
234
|
+
}
|
|
235
|
+
/** scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) ":" */
|
|
236
|
+
function isAutolink(line, s, e) {
|
|
237
|
+
if (e - s < 2) return false;
|
|
238
|
+
let c = line.charCodeAt(s);
|
|
239
|
+
if (!(c >= 65 && c <= 90 || c >= 97 && c <= 122)) return false;
|
|
240
|
+
for (let i = s + 1; i < e; i++) {
|
|
241
|
+
c = line.charCodeAt(i);
|
|
242
|
+
if (c === 58) return true;
|
|
243
|
+
if (!(c >= 65 && c <= 90 || c >= 97 && c <= 122 || c >= 48 && c <= 57 || c === 43 || c === 45 || c === 46)) return false;
|
|
244
|
+
}
|
|
245
|
+
return false;
|
|
246
|
+
}
|
|
247
|
+
/** CommonMark inline destination: `<url>`, bare url, optional title. */
|
|
248
|
+
function findLinkEnd(line, start) {
|
|
249
|
+
if (line[start] === "<") {
|
|
250
|
+
const gt = line.indexOf(">", start + 1);
|
|
251
|
+
if (gt === -1) return null;
|
|
252
|
+
const end = findTitle(line, gt + 1);
|
|
253
|
+
return end === -1 ? null : {
|
|
254
|
+
urlStart: start + 1,
|
|
255
|
+
urlEnd: gt,
|
|
256
|
+
end
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
let depth = 0;
|
|
260
|
+
let i = start;
|
|
261
|
+
while (i < line.length) {
|
|
262
|
+
const c = line[i];
|
|
263
|
+
if (c === "\\") {
|
|
264
|
+
i += 2;
|
|
265
|
+
continue;
|
|
266
|
+
}
|
|
267
|
+
if (c === "(") depth++;
|
|
268
|
+
else if (c === ")") {
|
|
269
|
+
if (depth === 0) return {
|
|
270
|
+
urlStart: start,
|
|
271
|
+
urlEnd: i,
|
|
272
|
+
end: i
|
|
273
|
+
};
|
|
274
|
+
depth--;
|
|
275
|
+
} else if (c === " " || c === " ") {
|
|
276
|
+
const end = findTitle(line, i);
|
|
277
|
+
if (end === -1) return null;
|
|
278
|
+
return {
|
|
279
|
+
urlStart: start,
|
|
280
|
+
urlEnd: i,
|
|
281
|
+
end
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
i++;
|
|
285
|
+
}
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
function findTitle(line, from) {
|
|
289
|
+
let i = from;
|
|
290
|
+
while (line[i] === " " || line[i] === " ") i++;
|
|
291
|
+
const open = line[i];
|
|
292
|
+
if (open !== "\"" && open !== "'" && open !== "(") return -1;
|
|
293
|
+
const close = open === "(" ? ")" : open;
|
|
294
|
+
const end = line.indexOf(close, i + 1);
|
|
295
|
+
if (end === -1) return -1;
|
|
296
|
+
let j = end + 1;
|
|
297
|
+
while (line[j] === " " || line[j] === " ") j++;
|
|
298
|
+
return line[j] === ")" ? j : -1;
|
|
299
|
+
}
|
|
300
|
+
function extractHTMLLinks(text, base, out) {
|
|
301
|
+
let i = 0;
|
|
302
|
+
while (i < text.length) {
|
|
303
|
+
const lt = text.indexOf("<", i);
|
|
304
|
+
if (lt === -1) break;
|
|
305
|
+
i = lt;
|
|
306
|
+
if (text.startsWith("<!--", i)) {
|
|
307
|
+
const end = text.indexOf("-->", i + 4);
|
|
308
|
+
if (end === -1) return;
|
|
309
|
+
i = end + 3;
|
|
310
|
+
continue;
|
|
311
|
+
}
|
|
312
|
+
const t1 = text[i + 1];
|
|
313
|
+
if (t1 === "s" || t1 === "S") {
|
|
314
|
+
const name = rawTagName(text, i);
|
|
315
|
+
if (name) {
|
|
316
|
+
i = skipRawElement(text, i, name);
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
if ((t1 === "a" || t1 === "A") && isNameBoundary(text[i + 2])) {
|
|
321
|
+
const tagEnd = scanTagEnd(text, i);
|
|
322
|
+
if (tagEnd === -1) return;
|
|
323
|
+
const href = readAttr(text.slice(i, tagEnd + 1), "href");
|
|
324
|
+
if (href) addLink(href, base, out);
|
|
325
|
+
i = tagEnd + 1;
|
|
326
|
+
continue;
|
|
327
|
+
}
|
|
328
|
+
i = lt + 1;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
/** Returns "script"|"style" if text[i..] opens one (case-insensitive), else null. */
|
|
332
|
+
function rawTagName(text, i) {
|
|
333
|
+
if (!(text[i + 1] === "s" || text[i + 1] === "S" ? text[i + 1] : null)) return null;
|
|
334
|
+
const rest4 = text.slice(i + 2, i + 7).toLowerCase();
|
|
335
|
+
if (rest4.startsWith("cript")) {
|
|
336
|
+
const after = text[i + 7];
|
|
337
|
+
if (isNameBoundary(after) || after === void 0) return "script";
|
|
338
|
+
}
|
|
339
|
+
if (rest4.startsWith("tyle")) {
|
|
340
|
+
const after = text[i + 6];
|
|
341
|
+
if (isNameBoundary(after) || after === void 0) return "style";
|
|
342
|
+
}
|
|
343
|
+
return null;
|
|
344
|
+
}
|
|
345
|
+
function skipRawElement(text, start, name) {
|
|
346
|
+
let i = start + 1;
|
|
347
|
+
while (i < text.length) {
|
|
348
|
+
const c = text.indexOf("</", i);
|
|
349
|
+
if (c === -1) return text.length;
|
|
350
|
+
if (text.slice(c + 2, c + 2 + name.length).toLowerCase() === name) {
|
|
351
|
+
const after = text[c + 2 + name.length];
|
|
352
|
+
if (isNameBoundary(after) || after === void 0) {
|
|
353
|
+
const gt = text.indexOf(">", c);
|
|
354
|
+
return gt === -1 ? text.length : gt + 1;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
i = c + 2;
|
|
358
|
+
}
|
|
359
|
+
return text.length;
|
|
360
|
+
}
|
|
361
|
+
function isNameBoundary(c) {
|
|
362
|
+
return c === void 0 || c === ">" || c === "/" || c === " " || c === " " || c === "\n" || c === "\r" || c === "\f";
|
|
363
|
+
}
|
|
364
|
+
function scanTagEnd(text, start) {
|
|
365
|
+
let quote = 0;
|
|
366
|
+
for (let j = start + 1; j < text.length; j++) {
|
|
367
|
+
const c = text[j];
|
|
368
|
+
if (quote) {
|
|
369
|
+
if (c === quote) quote = 0;
|
|
370
|
+
} else if (c === "\"" || c === "'") quote = c;
|
|
371
|
+
else if (c === ">") return j;
|
|
372
|
+
}
|
|
373
|
+
return -1;
|
|
374
|
+
}
|
|
375
|
+
function isWs(c) {
|
|
376
|
+
return c === " " || c === " " || c === "\n" || c === "\r" || c === "\f";
|
|
377
|
+
}
|
|
378
|
+
/** Read an attribute value from a tag substring (case-insensitive name, entities decoded). */
|
|
379
|
+
function readAttr(tag, name) {
|
|
380
|
+
let i = 0;
|
|
381
|
+
while (i < tag.length) {
|
|
382
|
+
const eq = tag.indexOf("=", i);
|
|
383
|
+
if (eq === -1) return null;
|
|
384
|
+
let e2 = eq - 1;
|
|
385
|
+
while (e2 >= 0 && isWs(tag[e2])) e2--;
|
|
386
|
+
let s = e2;
|
|
387
|
+
while (s >= 0 && !isWs(tag[s])) s--;
|
|
388
|
+
let v = eq + 1;
|
|
389
|
+
while (isWs(tag[v])) v++;
|
|
390
|
+
let value, next;
|
|
391
|
+
const q = tag[v];
|
|
392
|
+
if (q === "\"" || q === "'") {
|
|
393
|
+
const end = tag.indexOf(q, v + 1);
|
|
394
|
+
if (end === -1) return null;
|
|
395
|
+
value = tag.slice(v + 1, end);
|
|
396
|
+
next = end + 1;
|
|
397
|
+
} else {
|
|
398
|
+
let e = v;
|
|
399
|
+
while (e < tag.length && !isWs(tag[e]) && tag[e] !== ">") e++;
|
|
400
|
+
value = tag.slice(v, e);
|
|
401
|
+
next = e;
|
|
402
|
+
}
|
|
403
|
+
if (tag.slice(s + 1, e2 + 1).toLowerCase() === name) return decodeEntities(value);
|
|
404
|
+
i = next;
|
|
405
|
+
}
|
|
406
|
+
return null;
|
|
407
|
+
}
|
|
408
|
+
function decodeEntities(s) {
|
|
409
|
+
if (!s.includes("&")) return s;
|
|
410
|
+
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/g, (m, body) => {
|
|
411
|
+
if (body[0] === "#") {
|
|
412
|
+
const code = body[1] === "x" || body[1] === "X" ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
|
413
|
+
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
|
|
414
|
+
}
|
|
415
|
+
switch (body) {
|
|
416
|
+
case "amp": return "&";
|
|
417
|
+
case "lt": return "<";
|
|
418
|
+
case "gt": return ">";
|
|
419
|
+
case "quot": return "\"";
|
|
420
|
+
case "apos": return "'";
|
|
421
|
+
default: return m;
|
|
422
|
+
}
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
function addLink(raw, base, out) {
|
|
426
|
+
const url = normalizeUrl(raw, base);
|
|
427
|
+
if (url) out.add(url);
|
|
428
|
+
}
|
|
429
|
+
/**
|
|
430
|
+
* Crawl a graph of linked text documents.
|
|
431
|
+
*
|
|
432
|
+
* @param {string} startUrl Absolute URL, or relative path resolved against `baseHref`.
|
|
433
|
+
* @param {Object} [options]
|
|
434
|
+
* @param {number} [options.maxDepth=5] Maximum link distance from the start document.
|
|
435
|
+
* @param {number} [options.concurrency=5] Max in-flight fetches.
|
|
436
|
+
* @param {number} [options.maxPages=100] Hard cap on total documents considered.
|
|
437
|
+
* @param {boolean} [options.sameOrigin=false] Only follow links on the start document's origin.
|
|
438
|
+
* @param {Function} [options.fetchFn] Defaults to `globalThis.fetch`.
|
|
439
|
+
* @param {string} [options.baseHref] Base for relative URLs. Defaults to
|
|
440
|
+
* `location.href` when available.
|
|
441
|
+
* @param {Object} [options.cache] Cache adapter. Default: `memoryCache()`.
|
|
442
|
+
* @param {(url: string) => boolean} [options.accept] Target filter (sync).
|
|
443
|
+
* @param {(page: CrawlPage) => (void|Promise<void>)} [options.onPage]
|
|
444
|
+
* Called per page as it completes —
|
|
445
|
+
* parse incrementally instead of waiting
|
|
446
|
+
* for the whole crawl.
|
|
447
|
+
* @param {AbortSignal} [options.signal] Cancels the crawl; partial results returned.
|
|
448
|
+
* @returns {Promise<CrawlResult>}
|
|
449
|
+
*/
|
|
450
|
+
async function crawl(startUrl, options = {}) {
|
|
451
|
+
const maxDepth = Math.max(0, options.maxDepth ?? DEFAULT_MAX_DEPTH) | 0;
|
|
452
|
+
const concurrency = Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY) | 0;
|
|
453
|
+
const maxPages = Math.max(1, options.maxPages ?? DEFAULT_MAX_PAGES) | 0;
|
|
454
|
+
const sameOrigin = options.sameOrigin ?? false;
|
|
455
|
+
const fetchFn = options.fetchFn ?? globalThis.fetch;
|
|
456
|
+
const baseHref = options.baseHref ?? globalThis.location?.href;
|
|
457
|
+
const cache = options.cache ?? memoryCache();
|
|
458
|
+
const accept = options.accept ?? defaultAccept;
|
|
459
|
+
const signal = options.signal ?? null;
|
|
460
|
+
if (typeof fetchFn !== "function") throw new TypeError("crawl: no fetch implementation available — pass options.fetchFn");
|
|
461
|
+
const start = normalizeUrl(startUrl, baseHref);
|
|
462
|
+
if (!start) throw new TypeError(`crawl: cannot resolve "${startUrl}" to an absolute http(s) URL (baseHref: ${baseHref ?? "none"})`);
|
|
463
|
+
if (signal?.aborted) return {
|
|
464
|
+
urls: [],
|
|
465
|
+
pages: [],
|
|
466
|
+
errors: [],
|
|
467
|
+
aborted: true
|
|
468
|
+
};
|
|
469
|
+
const startOrigin = new URL(start).origin;
|
|
470
|
+
const store = {
|
|
471
|
+
get: async (url) => {
|
|
472
|
+
try {
|
|
473
|
+
return await cache.get(url) ?? null;
|
|
474
|
+
} catch {
|
|
475
|
+
return null;
|
|
476
|
+
}
|
|
477
|
+
},
|
|
478
|
+
set: async (url, entry) => {
|
|
479
|
+
try {
|
|
480
|
+
await cache.set(url, entry);
|
|
481
|
+
} catch {}
|
|
482
|
+
}
|
|
483
|
+
};
|
|
484
|
+
const queue = [{
|
|
485
|
+
url: start,
|
|
486
|
+
depth: 0
|
|
487
|
+
}];
|
|
488
|
+
const visited = /* @__PURE__ */ new Set([start]);
|
|
489
|
+
const pages = /* @__PURE__ */ new Map();
|
|
490
|
+
const errors = [];
|
|
491
|
+
let cursor = 0;
|
|
492
|
+
let aborted = false;
|
|
493
|
+
const isAllowed = (url) => (!sameOrigin || new URL(url).origin === startOrigin) && accept(url);
|
|
494
|
+
async function process(job) {
|
|
495
|
+
if (signal?.aborted) {
|
|
496
|
+
aborted = true;
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
const { url, depth } = job;
|
|
500
|
+
const entry = await store.get(url);
|
|
501
|
+
const headers = { Accept: ACCEPT };
|
|
502
|
+
if (entry?.etag) headers["If-None-Match"] = entry.etag;
|
|
503
|
+
if (entry?.lastModified) headers["If-Modified-Since"] = entry.lastModified;
|
|
504
|
+
let text, contentType, finalUrl = url, status = null, stale = false;
|
|
505
|
+
try {
|
|
506
|
+
const res = await fetchFn(url, {
|
|
507
|
+
headers,
|
|
508
|
+
signal: signal || void 0,
|
|
509
|
+
redirect: "follow",
|
|
510
|
+
cache: "no-store"
|
|
511
|
+
});
|
|
512
|
+
status = res.status;
|
|
513
|
+
finalUrl = normalizeUrl(res.url, url) || url;
|
|
514
|
+
if (res.status === 304 && entry) {
|
|
515
|
+
text = entry.text;
|
|
516
|
+
contentType = entry.contentType || "";
|
|
517
|
+
finalUrl = entry.finalUrl || finalUrl;
|
|
518
|
+
await store.set(url, entry);
|
|
519
|
+
} else if (res.ok) {
|
|
520
|
+
contentType = res.headers.get("content-type") || "";
|
|
521
|
+
if (BINARY_CT.test(contentType)) {
|
|
522
|
+
try {
|
|
523
|
+
await res.body?.cancel();
|
|
524
|
+
} catch {}
|
|
525
|
+
errors.push({
|
|
526
|
+
url,
|
|
527
|
+
kind: "binary",
|
|
528
|
+
status,
|
|
529
|
+
message: `unsupported content-type: ${contentType}`
|
|
530
|
+
});
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
text = await res.text();
|
|
534
|
+
await store.set(url, {
|
|
535
|
+
text,
|
|
536
|
+
contentType,
|
|
537
|
+
finalUrl,
|
|
538
|
+
etag: res.headers.get("etag"),
|
|
539
|
+
lastModified: res.headers.get("last-modified"),
|
|
540
|
+
timestamp: Date.now()
|
|
541
|
+
});
|
|
542
|
+
} else if (res.status >= 500 && entry) {
|
|
543
|
+
text = entry.text;
|
|
544
|
+
contentType = entry.contentType || "";
|
|
545
|
+
finalUrl = entry.finalUrl || finalUrl;
|
|
546
|
+
stale = true;
|
|
547
|
+
} else {
|
|
548
|
+
errors.push({
|
|
549
|
+
url,
|
|
550
|
+
kind: "http",
|
|
551
|
+
status: res.status,
|
|
552
|
+
message: `HTTP ${res.status} for ${url}`
|
|
553
|
+
});
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
} catch (err) {
|
|
557
|
+
if (signal?.aborted || err?.name === "AbortError") {
|
|
558
|
+
aborted = true;
|
|
559
|
+
return;
|
|
560
|
+
}
|
|
561
|
+
if (entry) {
|
|
562
|
+
text = entry.text;
|
|
563
|
+
contentType = entry.contentType || "";
|
|
564
|
+
finalUrl = entry.finalUrl || url;
|
|
565
|
+
stale = true;
|
|
566
|
+
} else {
|
|
567
|
+
errors.push({
|
|
568
|
+
url,
|
|
569
|
+
kind: "network",
|
|
570
|
+
message: String(err?.message || err)
|
|
571
|
+
});
|
|
572
|
+
return;
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
const links = /* @__PURE__ */ new Set();
|
|
576
|
+
extractMarkdownLinks(text, finalUrl, links);
|
|
577
|
+
extractHTMLLinks(text, finalUrl, links);
|
|
578
|
+
const page = {
|
|
579
|
+
url,
|
|
580
|
+
finalUrl,
|
|
581
|
+
depth,
|
|
582
|
+
status,
|
|
583
|
+
contentType,
|
|
584
|
+
kind: looksLikeHTML(contentType, text) ? "html" : "text",
|
|
585
|
+
stale,
|
|
586
|
+
text,
|
|
587
|
+
links: [...links]
|
|
588
|
+
};
|
|
589
|
+
pages.set(url, page);
|
|
590
|
+
if (depth < maxDepth) for (const link of links) {
|
|
591
|
+
if (visited.size >= maxPages) return;
|
|
592
|
+
if (visited.has(link) || !isAllowed(link)) continue;
|
|
593
|
+
visited.add(link);
|
|
594
|
+
queue.push({
|
|
595
|
+
url: link,
|
|
596
|
+
depth: depth + 1
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
try {
|
|
600
|
+
await options.onPage?.(page);
|
|
601
|
+
} catch (err) {
|
|
602
|
+
errors.push({
|
|
603
|
+
url,
|
|
604
|
+
kind: "callback",
|
|
605
|
+
message: String(err?.message || err)
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
const inflight = /* @__PURE__ */ new Set();
|
|
610
|
+
for (;;) {
|
|
611
|
+
while (cursor < queue.length && inflight.size < concurrency && !aborted) {
|
|
612
|
+
const job = queue[cursor++];
|
|
613
|
+
const p = process(job).catch((err) => {
|
|
614
|
+
errors.push({
|
|
615
|
+
url: job.url,
|
|
616
|
+
kind: "internal",
|
|
617
|
+
message: String(err?.message || err)
|
|
618
|
+
});
|
|
619
|
+
}).finally(() => {
|
|
620
|
+
inflight.delete(p);
|
|
621
|
+
});
|
|
622
|
+
inflight.add(p);
|
|
623
|
+
}
|
|
624
|
+
if (inflight.size === 0) break;
|
|
625
|
+
await Promise.race(inflight);
|
|
626
|
+
}
|
|
627
|
+
const ordered = [];
|
|
628
|
+
for (const { url } of queue) {
|
|
629
|
+
const page = pages.get(url);
|
|
630
|
+
if (page) ordered.push(page);
|
|
631
|
+
}
|
|
632
|
+
return {
|
|
633
|
+
urls: ordered.map((p) => p.url),
|
|
634
|
+
pages: ordered,
|
|
635
|
+
errors,
|
|
636
|
+
aborted
|
|
637
|
+
};
|
|
638
|
+
}
|
|
639
|
+
/**
|
|
640
|
+
* @typedef {Object} CrawlPage
|
|
641
|
+
* @property {string} url Requested URL (cache key).
|
|
642
|
+
* @property {string} finalUrl URL after redirects — use as the document's identity.
|
|
643
|
+
* @property {number} depth Link distance from the start document.
|
|
644
|
+
* @property {number|null} status HTTP status (304 served from cache keeps original text).
|
|
645
|
+
* @property {string} contentType Response Content-Type.
|
|
646
|
+
* @property {"html"|"text"} kind Cheap sniff hint — interpretation is the consumer's job.
|
|
647
|
+
* @property {boolean} stale True if served from cache due to 5xx/network failure.
|
|
648
|
+
* @property {string} text Raw response body.
|
|
649
|
+
* @property {string[]} links Absolute http(s) outgoing links, deduped, in discovery order.
|
|
650
|
+
*/
|
|
651
|
+
/**
|
|
652
|
+
* @typedef {Object} CrawlResult
|
|
653
|
+
* @property {string[]} urls Pages in BFS discovery order.
|
|
654
|
+
* @property {CrawlPage[]} pages
|
|
655
|
+
* @property {Array<{url: string, kind: "http"|"network"|"binary"|"callback"|"internal",
|
|
656
|
+
* status?: number, message: string}>} errors
|
|
657
|
+
* @property {boolean} aborted
|
|
658
|
+
*/
|
|
659
|
+
//#endregion
|
|
660
|
+
export { crawl, extractLinks, memoryCache };
|