@mdream/crawl 1.4.1 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -2
- package/dist/_chunks/crawl.mjs +200 -96
- package/dist/cli.mjs +34 -2
- package/dist/index.d.mts +63 -4
- package/dist/index.mjs +2 -2
- package/package.json +7 -7
package/README.md
CHANGED
|
@@ -49,6 +49,7 @@ npx @mdream/crawl -u https://docs.example.com
|
|
|
49
49
|
| `--crawl-delay <seconds>` | | Delay between requests in seconds | from `robots.txt` or none |
|
|
50
50
|
| `--exclude <pattern>` | | Exclude URLs matching glob patterns (repeatable) | none |
|
|
51
51
|
| `--skip-sitemap` | | Skip `sitemap.xml` and `robots.txt` discovery | `false` |
|
|
52
|
+
| `--sitemap <url>` | | Use an explicit sitemap URL instead of auto-discovery (repeatable for multi-part sitemaps) | auto-discovered |
|
|
52
53
|
| `--allow-subdomains` | | Crawl across subdomains of the same root domain | `false` |
|
|
53
54
|
| `--verbose` | `-v` | Enable verbose logging | `false` |
|
|
54
55
|
| `--help` | `-h` | Show help message | |
|
|
@@ -131,6 +132,7 @@ const results = await crawlAndGenerate({
|
|
|
131
132
|
| `exclude` | `string[]` | `[]` | Glob patterns for URLs to exclude |
|
|
132
133
|
| `crawlDelay` | `number` | from `robots.txt` | Delay between requests in seconds |
|
|
133
134
|
| `skipSitemap` | `boolean` | `false` | Skip `sitemap.xml` and `robots.txt` discovery |
|
|
135
|
+
| `sitemapUrls` | `string[]` | auto-discovered | Explicit sitemap URL(s) to use instead of auto-discovery. Supports non-standard locations and multiple parts (all loaded and merged). Ignored when `skipSitemap` is set |
|
|
134
136
|
| `allowSubdomains` | `boolean` | `false` | Crawl across subdomains of the same root domain (e.g. `docs.example.com` + `blog.example.com`). Output files are namespaced by hostname to avoid collisions |
|
|
135
137
|
| `useChrome` | `boolean` | `false` | Use system Chrome instead of Playwright's bundled browser (Playwright driver only) |
|
|
136
138
|
| `chunkSize` | `number` | | Chunk size passed to mdream for markdown conversion |
|
|
@@ -294,6 +296,7 @@ CLI arguments override config file values. Array options like `exclude` are conc
|
|
|
294
296
|
| `maxPages` | `number` | Maximum pages to crawl |
|
|
295
297
|
| `crawlDelay` | `number` | Delay between requests (seconds) |
|
|
296
298
|
| `skipSitemap` | `boolean` | Skip sitemap discovery |
|
|
299
|
+
| `sitemap` | `string \| string[]` | Explicit sitemap URL(s) to use instead of auto-discovery |
|
|
297
300
|
| `allowSubdomains` | `boolean` | Crawl across subdomains |
|
|
298
301
|
| `verbose` | `boolean` | Enable verbose logging |
|
|
299
302
|
| `artifacts` | `string[]` | Output formats: `llms.txt`, `llms-full.txt`, `markdown` |
|
|
@@ -437,16 +440,47 @@ Waits for `networkidle` before extracting content. Automatically detects and use
|
|
|
437
440
|
By default, the crawler performs sitemap discovery before crawling:
|
|
438
441
|
|
|
439
442
|
1. Fetches `robots.txt` to find `Sitemap:` directives and `Crawl-delay` values
|
|
440
|
-
2. Loads sitemaps referenced in `robots.txt`
|
|
443
|
+
2. Loads sitemaps referenced in `robots.txt` (authoritative, tried first)
|
|
441
444
|
3. Falls back to `/sitemap.xml`
|
|
442
445
|
4. Tries common alternatives: `/sitemap_index.xml`, `/sitemaps.xml`, `/sitemap-index.xml`
|
|
443
|
-
5. Supports sitemap index files (recursively loads child sitemaps)
|
|
446
|
+
5. Supports sitemap index files (recursively loads child sitemaps, with cycle protection)
|
|
444
447
|
6. Filters discovered URLs against glob patterns and exclusion rules
|
|
445
448
|
|
|
449
|
+
The first candidate that yields matching URLs wins, so a `robots.txt` sitemap is never overwritten by a well-known fallback. `<loc>` values are CDATA-unwrapped and XML-entity-decoded (`&` → `&`).
|
|
450
|
+
|
|
446
451
|
The home page is always included for metadata extraction (site name, description).
|
|
447
452
|
|
|
448
453
|
Disable with `--skip-sitemap` or `skipSitemap: true`.
|
|
449
454
|
|
|
455
|
+
### Overriding the sitemap location
|
|
456
|
+
|
|
457
|
+
When a site keeps its sitemap at a non-standard URL, or splits it across several links, pin the location(s) explicitly instead of relying on discovery:
|
|
458
|
+
|
|
459
|
+
```bash
|
|
460
|
+
# Single non-standard location
|
|
461
|
+
npx @mdream/crawl -u example.com --sitemap https://example.com/custom/sitemap.xml
|
|
462
|
+
|
|
463
|
+
# Multiple parts (repeatable), all loaded and merged
|
|
464
|
+
npx @mdream/crawl -u example.com \
|
|
465
|
+
--sitemap https://example.com/sitemap-posts.xml \
|
|
466
|
+
--sitemap https://example.com/sitemap-pages.xml
|
|
467
|
+
```
|
|
468
|
+
|
|
469
|
+
Or in `mdream.config.ts`:
|
|
470
|
+
|
|
471
|
+
```ts
|
|
472
|
+
import { defineConfig } from '@mdream/crawl'
|
|
473
|
+
|
|
474
|
+
export default defineConfig({
|
|
475
|
+
sitemap: [
|
|
476
|
+
'https://example.com/sitemap-posts.xml',
|
|
477
|
+
'https://example.com/sitemap-pages.xml',
|
|
478
|
+
],
|
|
479
|
+
})
|
|
480
|
+
```
|
|
481
|
+
|
|
482
|
+
An explicit sitemap replaces auto-discovery entirely (`robots.txt` is still read for `Crawl-delay`).
|
|
483
|
+
|
|
450
484
|
## Output Formats
|
|
451
485
|
|
|
452
486
|
### Individual Markdown Files
|
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -9,6 +9,113 @@ import { withHttps } from "ufo";
|
|
|
9
9
|
import picomatch from "picomatch";
|
|
10
10
|
import { getDomain } from "tldts";
|
|
11
11
|
import * as p from "@clack/prompts";
|
|
12
|
+
const DEFAULT_BOILERPLATE_THRESHOLD = .5;
|
|
13
|
+
const TOKEN_RE = /\S+/g;
|
|
14
|
+
function hashToken(s, start, end) {
|
|
15
|
+
let h = 2166136261;
|
|
16
|
+
for (let i = start; i < end; i++) {
|
|
17
|
+
h ^= s.charCodeAt(i);
|
|
18
|
+
h = Math.imul(h, 16777619) >>> 0;
|
|
19
|
+
}
|
|
20
|
+
return h >>> 0;
|
|
21
|
+
}
|
|
22
|
+
function tokenize(md, k) {
|
|
23
|
+
const starts = [];
|
|
24
|
+
const ends = [];
|
|
25
|
+
const tokenHashes = [];
|
|
26
|
+
TOKEN_RE.lastIndex = 0;
|
|
27
|
+
let m;
|
|
28
|
+
while ((m = TOKEN_RE.exec(md)) !== null) {
|
|
29
|
+
const start = m.index;
|
|
30
|
+
const end = TOKEN_RE.lastIndex;
|
|
31
|
+
starts.push(start);
|
|
32
|
+
ends.push(end);
|
|
33
|
+
tokenHashes.push(hashToken(md, start, end));
|
|
34
|
+
}
|
|
35
|
+
const n = tokenHashes.length;
|
|
36
|
+
const shingleHashes = [];
|
|
37
|
+
if (n === 0) return {
|
|
38
|
+
starts,
|
|
39
|
+
ends,
|
|
40
|
+
tokenHashes,
|
|
41
|
+
shingleHashes
|
|
42
|
+
};
|
|
43
|
+
if (n < k) shingleHashes.push(mixHashes(tokenHashes, 0, n));
|
|
44
|
+
else for (let i = 0; i + k <= n; i++) shingleHashes.push(mixHashes(tokenHashes, i, i + k));
|
|
45
|
+
return {
|
|
46
|
+
starts,
|
|
47
|
+
ends,
|
|
48
|
+
tokenHashes,
|
|
49
|
+
shingleHashes
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
function mixHashes(tokenHashes, from, to) {
|
|
53
|
+
let h = 2166136261;
|
|
54
|
+
for (let i = from; i < to; i++) {
|
|
55
|
+
h ^= tokenHashes[i];
|
|
56
|
+
h = Math.imul(h, 16777619) >>> 0;
|
|
57
|
+
}
|
|
58
|
+
return h >>> 0;
|
|
59
|
+
}
|
|
60
|
+
function detectChromeShingles(docs, threshold) {
|
|
61
|
+
const counts = /* @__PURE__ */ new Map();
|
|
62
|
+
for (let d = 0; d < docs.length; d++) {
|
|
63
|
+
const shingles = docs[d].shingleHashes;
|
|
64
|
+
const seen = /* @__PURE__ */ new Set();
|
|
65
|
+
for (let i = 0; i < shingles.length; i++) {
|
|
66
|
+
const h = shingles[i];
|
|
67
|
+
if (seen.has(h)) continue;
|
|
68
|
+
seen.add(h);
|
|
69
|
+
counts.set(h, (counts.get(h) ?? 0) + 1);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const minCount = Math.max(2, Math.ceil(threshold * docs.length));
|
|
73
|
+
const chrome = /* @__PURE__ */ new Set();
|
|
74
|
+
for (const [h, c] of counts) if (c >= minCount) chrome.add(h);
|
|
75
|
+
return chrome;
|
|
76
|
+
}
|
|
77
|
+
function stripDoc(md, doc, chrome, k, minRun) {
|
|
78
|
+
const n = doc.tokenHashes.length;
|
|
79
|
+
if (n === 0) return md;
|
|
80
|
+
const covered = new Uint8Array(n);
|
|
81
|
+
const shingles = doc.shingleHashes;
|
|
82
|
+
if (n < k) {
|
|
83
|
+
if (shingles.length === 1 && chrome.has(shingles[0])) covered.fill(1);
|
|
84
|
+
} else for (let i = 0; i < shingles.length; i++) if (chrome.has(shingles[i])) for (let j = i; j < i + k; j++) covered[j] = 1;
|
|
85
|
+
const keptRanges = [];
|
|
86
|
+
let runStart = 0;
|
|
87
|
+
let i = 0;
|
|
88
|
+
while (i < n) {
|
|
89
|
+
if (!covered[i]) {
|
|
90
|
+
i++;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
let j = i;
|
|
94
|
+
while (j < n && covered[j]) j++;
|
|
95
|
+
if (j - i >= minRun) {
|
|
96
|
+
if (i > runStart) keptRanges.push([runStart, i]);
|
|
97
|
+
runStart = j;
|
|
98
|
+
}
|
|
99
|
+
i = j;
|
|
100
|
+
}
|
|
101
|
+
if (runStart < n) keptRanges.push([runStart, n]);
|
|
102
|
+
if (keptRanges.length === 1 && keptRanges[0][0] === 0 && keptRanges[0][1] === n) return md;
|
|
103
|
+
if (keptRanges.length === 0) return md;
|
|
104
|
+
const pieces = [];
|
|
105
|
+
for (const [s, e] of keptRanges) pieces.push(md.slice(doc.starts[s], doc.ends[e - 1]).trim());
|
|
106
|
+
return pieces.filter(Boolean).join("\n\n");
|
|
107
|
+
}
|
|
108
|
+
function stripBoilerplateFromCorpus(contents, options = {}) {
|
|
109
|
+
const threshold = options.threshold ?? .5;
|
|
110
|
+
const minDocs = options.minDocs ?? 3;
|
|
111
|
+
const k = Math.max(2, options.shingleSize ?? 6);
|
|
112
|
+
const minRun = Math.max(k, options.minRun ?? 12);
|
|
113
|
+
if (contents.length < minDocs) return contents.slice();
|
|
114
|
+
const docs = contents.map((md) => tokenize(md, k));
|
|
115
|
+
const chrome = detectChromeShingles(docs, threshold);
|
|
116
|
+
if (chrome.size === 0) return contents.slice();
|
|
117
|
+
return contents.map((md, idx) => stripDoc(md, docs[idx], chrome, k, minRun));
|
|
118
|
+
}
|
|
12
119
|
function stripGlobTail(s) {
|
|
13
120
|
const idx = s.indexOf("*");
|
|
14
121
|
return idx === -1 ? s : s.slice(0, idx);
|
|
@@ -134,7 +241,7 @@ const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
|
|
|
134
241
|
const URL_TRAILING_SLASH_RE = /\/$/;
|
|
135
242
|
const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
|
|
136
243
|
const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
|
|
137
|
-
const TRACKING_PARAMS = new Set([
|
|
244
|
+
const TRACKING_PARAMS = /* @__PURE__ */ new Set([
|
|
138
245
|
"utm_source",
|
|
139
246
|
"utm_medium",
|
|
140
247
|
"utm_campaign",
|
|
@@ -191,6 +298,23 @@ function extractCdataUrl(url) {
|
|
|
191
298
|
if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) return url.slice(9, -3);
|
|
192
299
|
return url;
|
|
193
300
|
}
|
|
301
|
+
const XML_ENTITIES = {
|
|
302
|
+
"<": "<",
|
|
303
|
+
">": ">",
|
|
304
|
+
""": "\"",
|
|
305
|
+
"'": "'",
|
|
306
|
+
"&": "&"
|
|
307
|
+
};
|
|
308
|
+
const XML_ENTITY_RE = /&(?:lt|gt|quot|apos|amp);/g;
|
|
309
|
+
function decodeXmlEntities(value) {
|
|
310
|
+
if (!value.includes("&")) return value;
|
|
311
|
+
return value.replace(XML_ENTITY_RE, (m) => XML_ENTITIES[m]);
|
|
312
|
+
}
|
|
313
|
+
function cleanLoc(raw) {
|
|
314
|
+
const value = raw.trim();
|
|
315
|
+
if (value.startsWith("<![CDATA[") && value.endsWith("]]>")) return extractCdataUrl(value);
|
|
316
|
+
return decodeXmlEntities(value);
|
|
317
|
+
}
|
|
194
318
|
function normalizeUrl(resolved) {
|
|
195
319
|
resolved.hash = "";
|
|
196
320
|
const params = resolved.searchParams;
|
|
@@ -198,7 +322,9 @@ function normalizeUrl(resolved) {
|
|
|
198
322
|
params.sort();
|
|
199
323
|
return resolved.href.replace(URL_TRAILING_SLASH_RE, "") || resolved.href;
|
|
200
324
|
}
|
|
201
|
-
async function loadSitemap(sitemapUrl) {
|
|
325
|
+
async function loadSitemap(sitemapUrl, visited = /* @__PURE__ */ new Set()) {
|
|
326
|
+
if (visited.has(sitemapUrl)) return [];
|
|
327
|
+
visited.add(sitemapUrl);
|
|
202
328
|
const xmlContent = await ofetch(sitemapUrl, {
|
|
203
329
|
headers: FETCH_HEADERS,
|
|
204
330
|
timeout: 1e4,
|
|
@@ -213,9 +339,9 @@ async function loadSitemap(sitemapUrl) {
|
|
|
213
339
|
while (true) {
|
|
214
340
|
match = SITEMAP_INDEX_LOC_RE.exec(xmlContent);
|
|
215
341
|
if (match === null) break;
|
|
216
|
-
childSitemaps.push(
|
|
342
|
+
childSitemaps.push(cleanLoc(match[1]));
|
|
217
343
|
}
|
|
218
|
-
const childResults = await Promise.allSettled(childSitemaps.map((url) => loadSitemap(url)));
|
|
344
|
+
const childResults = await Promise.allSettled(childSitemaps.map((url) => loadSitemap(url, visited)));
|
|
219
345
|
const allUrls = [];
|
|
220
346
|
for (const result of childResults) if (result.status === "fulfilled") allUrls.push(...result.value);
|
|
221
347
|
return allUrls;
|
|
@@ -226,7 +352,7 @@ async function loadSitemap(sitemapUrl) {
|
|
|
226
352
|
while (true) {
|
|
227
353
|
match = SITEMAP_URL_LOC_RE.exec(xmlContent);
|
|
228
354
|
if (match === null) break;
|
|
229
|
-
urls.push(
|
|
355
|
+
urls.push(cleanLoc(match[1]));
|
|
230
356
|
}
|
|
231
357
|
return urls;
|
|
232
358
|
}
|
|
@@ -291,7 +417,7 @@ async function runConcurrent(items, concurrency, fn) {
|
|
|
291
417
|
await Promise.all(workers);
|
|
292
418
|
}
|
|
293
419
|
async function crawlAndGenerate(options, onProgress) {
|
|
294
|
-
const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay: userCrawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, allowSubdomains = false, hooks: hooksConfig, onPage } = options;
|
|
420
|
+
const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay: userCrawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, sitemapUrls: explicitSitemapUrls, allowSubdomains = false, stripBoilerplate = true, boilerplateThreshold, hooks: hooksConfig, onPage } = options;
|
|
295
421
|
const logger = resolveLogger(options);
|
|
296
422
|
const hooks = createHooks();
|
|
297
423
|
if (hooksConfig) hooks.addHooks(hooksConfig);
|
|
@@ -332,6 +458,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
332
458
|
if (startingUrls.length > 0 && !skipSitemap && !singlePageMode) {
|
|
333
459
|
const baseUrl = new URL(startingUrls[0]).origin;
|
|
334
460
|
const homePageUrl = baseUrl;
|
|
461
|
+
const hasExplicitSitemaps = !!explicitSitemapUrls && explicitSitemapUrls.length > 0;
|
|
335
462
|
onProgress?.(progress);
|
|
336
463
|
let robotsContent = null;
|
|
337
464
|
try {
|
|
@@ -349,90 +476,58 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
349
476
|
logger.info(`[ROBOTS] Crawl-delay: ${crawlDelay}s`);
|
|
350
477
|
}
|
|
351
478
|
}
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
const robotsSitemaps = sitemapMatches.map((match) => match.replace(ROBOTS_SITEMAP_PREFIX_RE, "").trim());
|
|
359
|
-
for (const sitemapUrl of robotsSitemaps) try {
|
|
360
|
-
const robotsUrls = await loadSitemap(sitemapUrl);
|
|
361
|
-
sitemapAttempts.push({
|
|
362
|
-
url: sitemapUrl,
|
|
363
|
-
success: true
|
|
364
|
-
});
|
|
365
|
-
const filteredUrls = filterSitemapUrls(robotsUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
|
|
366
|
-
if (hasGlobPatterns) {
|
|
367
|
-
startingUrls = filteredUrls;
|
|
368
|
-
progress.sitemap.processed = filteredUrls.length;
|
|
369
|
-
onProgress?.(progress);
|
|
370
|
-
break;
|
|
371
|
-
} else if (filteredUrls.length > 0) {
|
|
372
|
-
startingUrls = filteredUrls;
|
|
373
|
-
progress.sitemap.processed = filteredUrls.length;
|
|
374
|
-
onProgress?.(progress);
|
|
375
|
-
break;
|
|
376
|
-
}
|
|
377
|
-
} catch (error) {
|
|
378
|
-
sitemapAttempts.push({
|
|
379
|
-
url: sitemapUrl,
|
|
380
|
-
success: false,
|
|
381
|
-
error: error instanceof Error ? error.message : "Unknown error"
|
|
382
|
-
});
|
|
383
|
-
}
|
|
479
|
+
const candidateSitemaps = [];
|
|
480
|
+
if (hasExplicitSitemaps) candidateSitemaps.push(...explicitSitemapUrls);
|
|
481
|
+
else {
|
|
482
|
+
if (robotsContent) {
|
|
483
|
+
const sitemapMatches = robotsContent.match(ROBOTS_SITEMAP_RE);
|
|
484
|
+
if (sitemapMatches) for (const m of sitemapMatches) candidateSitemaps.push(m.replace(ROBOTS_SITEMAP_PREFIX_RE, "").trim());
|
|
384
485
|
}
|
|
486
|
+
candidateSitemaps.push(`${baseUrl}/sitemap.xml`, `${baseUrl}/sitemap_index.xml`, `${baseUrl}/sitemaps.xml`, `${baseUrl}/sitemap-index.xml`);
|
|
385
487
|
}
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
488
|
+
if (candidateSitemaps.length > 0) {
|
|
489
|
+
progress.sitemap.status = "processing";
|
|
490
|
+
progress.sitemap.found = candidateSitemaps.length;
|
|
491
|
+
onProgress?.(progress);
|
|
492
|
+
}
|
|
493
|
+
if (hasExplicitSitemaps) {
|
|
494
|
+
const merged = [];
|
|
495
|
+
for (const sitemapUrl of candidateSitemaps) try {
|
|
496
|
+
merged.push(...await loadSitemap(sitemapUrl));
|
|
497
|
+
sitemapAttempts.push({
|
|
498
|
+
url: sitemapUrl,
|
|
499
|
+
success: true
|
|
500
|
+
});
|
|
501
|
+
} catch (error) {
|
|
502
|
+
sitemapAttempts.push({
|
|
503
|
+
url: sitemapUrl,
|
|
504
|
+
success: false,
|
|
505
|
+
error: error instanceof Error ? error.message : "Unknown error"
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
const dedupedUrls = merged.length > 0 ? [...new Set(merged)] : merged;
|
|
509
|
+
const filteredUrls = filterSitemapUrls(dedupedUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
|
|
510
|
+
if (hasGlobPatterns || filteredUrls.length > 0) {
|
|
402
511
|
startingUrls = filteredUrls;
|
|
403
|
-
progress.sitemap.found =
|
|
512
|
+
progress.sitemap.found = dedupedUrls.length;
|
|
404
513
|
progress.sitemap.processed = filteredUrls.length;
|
|
405
514
|
onProgress?.(progress);
|
|
406
|
-
mainSitemapProcessed = true;
|
|
407
515
|
}
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
const commonSitemaps = [
|
|
416
|
-
`${baseUrl}/sitemap_index.xml`,
|
|
417
|
-
`${baseUrl}/sitemaps.xml`,
|
|
418
|
-
`${baseUrl}/sitemap-index.xml`
|
|
419
|
-
];
|
|
420
|
-
for (const sitemapUrl of commonSitemaps) try {
|
|
421
|
-
const altUrls = await loadSitemap(sitemapUrl);
|
|
516
|
+
} else {
|
|
517
|
+
const seenCandidates = /* @__PURE__ */ new Set();
|
|
518
|
+
for (const sitemapUrl of candidateSitemaps) {
|
|
519
|
+
if (seenCandidates.has(sitemapUrl)) continue;
|
|
520
|
+
seenCandidates.add(sitemapUrl);
|
|
521
|
+
try {
|
|
522
|
+
const foundUrls = await loadSitemap(sitemapUrl);
|
|
422
523
|
sitemapAttempts.push({
|
|
423
524
|
url: sitemapUrl,
|
|
424
525
|
success: true
|
|
425
526
|
});
|
|
426
|
-
const filteredUrls = filterSitemapUrls(
|
|
427
|
-
if (hasGlobPatterns) {
|
|
428
|
-
startingUrls = filteredUrls;
|
|
429
|
-
progress.sitemap.found = altUrls.length;
|
|
430
|
-
progress.sitemap.processed = filteredUrls.length;
|
|
431
|
-
onProgress?.(progress);
|
|
432
|
-
break;
|
|
433
|
-
} else if (filteredUrls.length > 0) {
|
|
527
|
+
const filteredUrls = filterSitemapUrls(foundUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
|
|
528
|
+
if (hasGlobPatterns || filteredUrls.length > 0) {
|
|
434
529
|
startingUrls = filteredUrls;
|
|
435
|
-
progress.sitemap.found =
|
|
530
|
+
progress.sitemap.found = foundUrls.length;
|
|
436
531
|
progress.sitemap.processed = filteredUrls.length;
|
|
437
532
|
onProgress?.(progress);
|
|
438
533
|
break;
|
|
@@ -448,6 +543,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
448
543
|
}
|
|
449
544
|
const successfulSitemaps = sitemapAttempts.filter((a) => a.success);
|
|
450
545
|
const failedSitemaps = sitemapAttempts.filter((a) => !a.success);
|
|
546
|
+
if (hasExplicitSitemaps && failedSitemaps.length > 0) logger.warn(`Sitemap: ${failedSitemaps.length} explicit sitemap(s) failed to load; crawl may be incomplete: ${failedSitemaps.map((a) => `${a.url} (${a.error})`).join(", ")}`);
|
|
451
547
|
if (successfulSitemaps.length > 0 && progress.sitemap.processed > 0) {
|
|
452
548
|
sitemapProvidedUrls = true;
|
|
453
549
|
logger.info(`Sitemap: ${progress.sitemap.processed} URLs from ${successfulSitemaps[0].url}`);
|
|
@@ -553,22 +649,8 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
553
649
|
const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
|
|
554
650
|
const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
|
|
555
651
|
const safeSegments = [...hostPrefix, ...pathSegments.map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"))];
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
url,
|
|
559
|
-
title,
|
|
560
|
-
content: md,
|
|
561
|
-
filePath
|
|
562
|
-
};
|
|
563
|
-
await hooks.callHook("crawl:content", contentCtx);
|
|
564
|
-
md = contentCtx.content;
|
|
565
|
-
filePath = contentCtx.filePath;
|
|
566
|
-
const fileDir = dirname(filePath);
|
|
567
|
-
if (fileDir && !createdDirs.has(fileDir)) {
|
|
568
|
-
await mkdir(fileDir, { recursive: true });
|
|
569
|
-
createdDirs.add(fileDir);
|
|
570
|
-
}
|
|
571
|
-
await writeFile(filePath, md, "utf-8");
|
|
652
|
+
const safeFilename = normalize(`${safeSegments.length > 0 ? safeSegments.join("/") : "index"}.md`);
|
|
653
|
+
filePath = join(outputDir, safeFilename);
|
|
572
654
|
}
|
|
573
655
|
const isHomePage = parsedUrl.pathname === "/" && parsedUrl.origin === normalizedHomePageUrl;
|
|
574
656
|
if (shouldProcessMarkdown || isHomePage) {
|
|
@@ -763,6 +845,28 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
763
845
|
progress.generation.status = "generating";
|
|
764
846
|
onProgress?.(progress);
|
|
765
847
|
const successfulResults = results.filter((r) => r.success);
|
|
848
|
+
if (stripBoilerplate && !singlePageMode) {
|
|
849
|
+
const cleaned = stripBoilerplateFromCorpus(successfulResults.map((r) => r.content), boilerplateThreshold !== void 0 ? { threshold: boilerplateThreshold } : {});
|
|
850
|
+
for (let i = 0; i < successfulResults.length; i++) successfulResults[i].content = cleaned[i];
|
|
851
|
+
}
|
|
852
|
+
if (generateIndividualMd) for (const result of successfulResults) {
|
|
853
|
+
if (!result.filePath) continue;
|
|
854
|
+
const contentCtx = {
|
|
855
|
+
url: result.url,
|
|
856
|
+
title: result.title,
|
|
857
|
+
content: result.content,
|
|
858
|
+
filePath: result.filePath
|
|
859
|
+
};
|
|
860
|
+
await hooks.callHook("crawl:content", contentCtx);
|
|
861
|
+
result.content = contentCtx.content;
|
|
862
|
+
result.filePath = contentCtx.filePath;
|
|
863
|
+
const fileDir = dirname(result.filePath);
|
|
864
|
+
if (fileDir && !createdDirs.has(fileDir)) {
|
|
865
|
+
await mkdir(fileDir, { recursive: true });
|
|
866
|
+
createdDirs.add(fileDir);
|
|
867
|
+
}
|
|
868
|
+
await writeFile(result.filePath, result.content, "utf-8");
|
|
869
|
+
}
|
|
766
870
|
const firstUrl = new URL(withHttps(urls[0]));
|
|
767
871
|
const originUrl = firstUrl.origin;
|
|
768
872
|
const homePageResult = successfulResults.find((r) => {
|
|
@@ -813,4 +917,4 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
813
917
|
}
|
|
814
918
|
return results;
|
|
815
919
|
}
|
|
816
|
-
export { crawlAndGenerate, createClackLogger, parseUrlPattern, resolveLogger, silentLogger, validateGlobPattern };
|
|
920
|
+
export { DEFAULT_BOILERPLATE_THRESHOLD, crawlAndGenerate, createClackLogger, parseUrlPattern, resolveLogger, silentLogger, stripBoilerplateFromCorpus, validateGlobPattern };
|
package/dist/cli.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { crawlAndGenerate, parseUrlPattern, resolveLogger, validateGlobPattern } from "./_chunks/crawl.mjs";
|
|
1
|
+
import { DEFAULT_BOILERPLATE_THRESHOLD, crawlAndGenerate, parseUrlPattern, resolveLogger, validateGlobPattern } from "./_chunks/crawl.mjs";
|
|
2
2
|
import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { dirname, join, relative, resolve } from "pathe";
|
|
4
4
|
import { withHttps } from "ufo";
|
|
@@ -12,7 +12,7 @@ async function loadMdreamConfig(cwd) {
|
|
|
12
12
|
});
|
|
13
13
|
return config || {};
|
|
14
14
|
}
|
|
15
|
-
const QUIET_FLAGS = new Set([
|
|
15
|
+
const QUIET_FLAGS = /* @__PURE__ */ new Set([
|
|
16
16
|
"--silent",
|
|
17
17
|
"--quiet",
|
|
18
18
|
"-q"
|
|
@@ -224,7 +224,10 @@ Options:
|
|
|
224
224
|
--crawl-delay <seconds> Crawl delay in seconds
|
|
225
225
|
--exclude <pattern> Exclude URLs matching glob patterns (can be used multiple times)
|
|
226
226
|
--skip-sitemap Skip sitemap.xml and robots.txt discovery
|
|
227
|
+
--sitemap <url> Use an explicit sitemap URL instead of auto-discovery (repeatable for multi-part sitemaps)
|
|
227
228
|
--allow-subdomains Crawl across subdomains of the same root domain
|
|
229
|
+
--keep-boilerplate Keep repeated site chrome (nav/footer) in per-page output (default: stripped)
|
|
230
|
+
--boilerplate-threshold <n> Fraction of pages a block must repeat in to count as chrome (0-1, default: ${DEFAULT_BOILERPLATE_THRESHOLD})
|
|
228
231
|
-v, --verbose Enable verbose logging
|
|
229
232
|
-q, --quiet, --silent Suppress all logs (clean stdout for JSON-RPC/MCP)
|
|
230
233
|
-h, --help Show this help message
|
|
@@ -238,6 +241,7 @@ Examples:
|
|
|
238
241
|
@mdream/crawl -u example.com --exclude "*/admin/*" --exclude "*/api/*"
|
|
239
242
|
@mdream/crawl -u example.com --verbose
|
|
240
243
|
@mdream/crawl -u example.com --skip-sitemap
|
|
244
|
+
@mdream/crawl -u example.com --sitemap https://example.com/custom/sitemap.xml
|
|
241
245
|
@mdream/crawl -u example.com --driver playwright --single-page
|
|
242
246
|
`);
|
|
243
247
|
process.exit(0);
|
|
@@ -349,7 +353,28 @@ Examples:
|
|
|
349
353
|
const patterns = [parsed];
|
|
350
354
|
const verbose = args.includes("--verbose") || args.includes("-v");
|
|
351
355
|
const skipSitemap = args.includes("--skip-sitemap");
|
|
356
|
+
const sitemapUrls = getArgValues("--sitemap").map((u) => withHttps(u));
|
|
357
|
+
if (args.filter((a) => a === "--sitemap" || a === "-sitemap").length > sitemapUrls.length) {
|
|
358
|
+
logger.error("Error: --sitemap requires a URL value");
|
|
359
|
+
process.exit(1);
|
|
360
|
+
}
|
|
361
|
+
for (const sitemapUrl of sitemapUrls) try {
|
|
362
|
+
new URL(sitemapUrl);
|
|
363
|
+
} catch {
|
|
364
|
+
logger.error(`Error: Invalid sitemap URL: ${sitemapUrl}`);
|
|
365
|
+
process.exit(1);
|
|
366
|
+
}
|
|
352
367
|
const allowSubdomains = args.includes("--allow-subdomains");
|
|
368
|
+
const stripBoilerplate = args.includes("--keep-boilerplate") ? false : void 0;
|
|
369
|
+
const boilerplateThresholdStr = getArgValue("--boilerplate-threshold");
|
|
370
|
+
let boilerplateThreshold;
|
|
371
|
+
if (boilerplateThresholdStr !== void 0) {
|
|
372
|
+
boilerplateThreshold = Number(boilerplateThresholdStr);
|
|
373
|
+
if (Number.isNaN(boilerplateThreshold) || boilerplateThreshold <= 0 || boilerplateThreshold > 1) {
|
|
374
|
+
logger.error("Error: Boilerplate threshold must be a number between 0 (exclusive) and 1");
|
|
375
|
+
process.exit(1);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
353
378
|
if (skipSitemap && parsed.isGlob) logger.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
|
|
354
379
|
return {
|
|
355
380
|
urls: [url],
|
|
@@ -369,7 +394,10 @@ Examples:
|
|
|
369
394
|
exclude: excludePatterns.length > 0 ? excludePatterns : void 0,
|
|
370
395
|
verbose,
|
|
371
396
|
skipSitemap,
|
|
397
|
+
sitemapUrls: sitemapUrls.length > 0 ? sitemapUrls : void 0,
|
|
372
398
|
allowSubdomains,
|
|
399
|
+
stripBoilerplate,
|
|
400
|
+
boilerplateThreshold,
|
|
373
401
|
silent
|
|
374
402
|
};
|
|
375
403
|
}
|
|
@@ -389,7 +417,10 @@ async function main() {
|
|
|
389
417
|
maxDepth: cliOptions.maxDepth ?? fileConfig.maxDepth,
|
|
390
418
|
crawlDelay: cliOptions.crawlDelay ?? fileConfig.crawlDelay,
|
|
391
419
|
skipSitemap: cliOptions.skipSitemap || fileConfig.skipSitemap || false,
|
|
420
|
+
sitemapUrls: cliOptions.sitemapUrls ?? (fileConfig.sitemap ? Array.isArray(fileConfig.sitemap) ? fileConfig.sitemap : [fileConfig.sitemap] : void 0),
|
|
392
421
|
allowSubdomains: cliOptions.allowSubdomains || fileConfig.allowSubdomains || false,
|
|
422
|
+
stripBoilerplate: cliOptions.stripBoilerplate ?? fileConfig.stripBoilerplate ?? true,
|
|
423
|
+
boilerplateThreshold: cliOptions.boilerplateThreshold ?? fileConfig.boilerplateThreshold,
|
|
393
424
|
verbose: cliOptions.verbose || fileConfig.verbose || false,
|
|
394
425
|
silent,
|
|
395
426
|
exclude: configExclude.length > 0 || cliExclude.length > 0 ? [...configExclude, ...cliExclude] : void 0,
|
|
@@ -407,6 +438,7 @@ async function main() {
|
|
|
407
438
|
`Formats: ${formats.join(", ")}`,
|
|
408
439
|
options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
|
|
409
440
|
options.skipSitemap && `Skip sitemap: Yes`,
|
|
441
|
+
options.sitemapUrls && options.sitemapUrls.length > 0 && `Sitemap: ${options.sitemapUrls.join(", ")}`,
|
|
410
442
|
options.allowSubdomains && `Allow subdomains: Yes`,
|
|
411
443
|
options.verbose && `Verbose: Enabled`
|
|
412
444
|
].filter(Boolean);
|
package/dist/index.d.mts
CHANGED
|
@@ -1,3 +1,34 @@
|
|
|
1
|
+
interface BoilerplateOptions {
|
|
2
|
+
/**
|
|
3
|
+
* Fraction of documents a shingle must appear in to count as chrome (0..1).
|
|
4
|
+
* Higher = stricter. Defaults to `DEFAULT_BOILERPLATE_THRESHOLD` (0.5).
|
|
5
|
+
*/
|
|
6
|
+
threshold?: number;
|
|
7
|
+
/**
|
|
8
|
+
* Minimum number of documents before detection runs at all.
|
|
9
|
+
* Defaults to `DEFAULT_BOILERPLATE_MIN_DOCS` (3).
|
|
10
|
+
*/
|
|
11
|
+
minDocs?: number;
|
|
12
|
+
/**
|
|
13
|
+
* Number of consecutive tokens per shingle. Smaller catches shorter repeated
|
|
14
|
+
* spans but risks matching common phrases; larger is more conservative.
|
|
15
|
+
* Defaults to `DEFAULT_BOILERPLATE_SHINGLE_SIZE` (6).
|
|
16
|
+
*/
|
|
17
|
+
shingleSize?: number;
|
|
18
|
+
/**
|
|
19
|
+
* Minimum length (in tokens) of a covered run before it is removed. Stops short
|
|
20
|
+
* incidental matches inside unique prose from being nibbled away.
|
|
21
|
+
* Defaults to `DEFAULT_BOILERPLATE_MIN_RUN` (12).
|
|
22
|
+
*/
|
|
23
|
+
minRun?: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Strip repeated cross-page boilerplate from a corpus of markdown documents.
|
|
27
|
+
*
|
|
28
|
+
* Returns a new array aligned with `contents`. Documents that are unchanged (no
|
|
29
|
+
* detected chrome, or detection skipped) are returned byte-for-byte as given.
|
|
30
|
+
*/
|
|
31
|
+
declare function stripBoilerplateFromCorpus(contents: string[], options?: BoilerplateOptions): string[];
|
|
1
32
|
/**
|
|
2
33
|
* Minimal spinner surface used by the crawler. Mirrors the subset of
|
|
3
34
|
* `@clack/prompts` spinner methods the package relies on.
|
|
@@ -95,7 +126,28 @@ interface CrawlOptions {
|
|
|
95
126
|
descriptionOverride?: string;
|
|
96
127
|
verbose?: boolean;
|
|
97
128
|
skipSitemap?: boolean;
|
|
129
|
+
/**
|
|
130
|
+
* Explicit sitemap URL(s) to use instead of auto-discovery (robots.txt +
|
|
131
|
+
* well-known paths). Accepts non-standard locations and multiple parts, which
|
|
132
|
+
* are all loaded and merged. Ignored when `skipSitemap` is set.
|
|
133
|
+
*/
|
|
134
|
+
sitemapUrls?: string[];
|
|
98
135
|
allowSubdomains?: boolean;
|
|
136
|
+
/**
|
|
137
|
+
* Strip repeated site chrome (top nav, footer, newsletter walls) from per-page
|
|
138
|
+
* markdown and llms-full.txt page sections. Implemented as a post-processing
|
|
139
|
+
* pass over the generated markdown: blocks that repeat across the corpus are
|
|
140
|
+
* detected by frequency and removed only from the wrapping (leading/trailing)
|
|
141
|
+
* run of each page, so a page's main content is preserved. Does not affect the
|
|
142
|
+
* llms.txt link index. Defaults to true. Needs a corpus (>= 3 pages) to run, so
|
|
143
|
+
* single-page crawls are left unchanged.
|
|
144
|
+
*/
|
|
145
|
+
stripBoilerplate?: boolean;
|
|
146
|
+
/**
|
|
147
|
+
* Fraction of crawled pages a block must appear in to count as chrome (0..1).
|
|
148
|
+
* Higher is stricter. Defaults to `DEFAULT_BOILERPLATE_THRESHOLD` (0.5).
|
|
149
|
+
*/
|
|
150
|
+
boilerplateThreshold?: number;
|
|
99
151
|
/**
|
|
100
152
|
* Suppress all diagnostic/progress logging. Use when stdout must stay clean,
|
|
101
153
|
* e.g. an MCP server that only emits JSON-RPC (issue #100). Ignored when an
|
|
@@ -107,7 +159,7 @@ interface CrawlOptions {
|
|
|
107
159
|
* stderr) instead of the default `@clack/prompts` stdout output.
|
|
108
160
|
*/
|
|
109
161
|
logger?: CrawlLogger;
|
|
110
|
-
hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][] }>;
|
|
162
|
+
hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
|
|
111
163
|
onPage?: (page: PageData) => Promise<void> | void;
|
|
112
164
|
}
|
|
113
165
|
interface MdreamCrawlConfig {
|
|
@@ -117,12 +169,18 @@ interface MdreamCrawlConfig {
|
|
|
117
169
|
maxPages?: number;
|
|
118
170
|
crawlDelay?: number;
|
|
119
171
|
skipSitemap?: boolean;
|
|
172
|
+
/** Explicit sitemap URL(s) to use instead of auto-discovery. Supports non-standard locations and multiple parts. */
|
|
173
|
+
sitemap?: string | string[];
|
|
120
174
|
allowSubdomains?: boolean;
|
|
175
|
+
/** Strip repeated site chrome from per-page output. Defaults to true. */
|
|
176
|
+
stripBoilerplate?: boolean;
|
|
177
|
+
/** Fraction of pages a block must repeat in to count as chrome (0..1). Defaults to 0.5. */
|
|
178
|
+
boilerplateThreshold?: number;
|
|
121
179
|
verbose?: boolean;
|
|
122
180
|
/** Suppress all diagnostic/progress logging (issue #100). */
|
|
123
181
|
silent?: boolean;
|
|
124
182
|
artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
|
|
125
|
-
hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][] }>;
|
|
183
|
+
hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
|
|
126
184
|
}
|
|
127
185
|
declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
|
|
128
186
|
interface ParsedUrlPattern {
|
|
@@ -159,7 +217,8 @@ interface CrawlProgress {
|
|
|
159
217
|
total: number;
|
|
160
218
|
processed: number;
|
|
161
219
|
failed: number;
|
|
162
|
-
currentUrl?: string;
|
|
220
|
+
currentUrl?: string;
|
|
221
|
+
/** Page fetch latency stats in ms */
|
|
163
222
|
latency: {
|
|
164
223
|
total: number;
|
|
165
224
|
min: number;
|
|
@@ -173,4 +232,4 @@ interface CrawlProgress {
|
|
|
173
232
|
};
|
|
174
233
|
}
|
|
175
234
|
declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
|
|
176
|
-
export { type CrawlHooks, type CrawlLogger, type CrawlOptions, type CrawlResult, type CrawlSpinner, type MdreamCrawlConfig, type PageData, crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger };
|
|
235
|
+
export { type BoilerplateOptions, type CrawlHooks, type CrawlLogger, type CrawlOptions, type CrawlResult, type CrawlSpinner, type MdreamCrawlConfig, type PageData, crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger, stripBoilerplateFromCorpus };
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { crawlAndGenerate, createClackLogger, resolveLogger, silentLogger } from "./_chunks/crawl.mjs";
|
|
1
|
+
import { crawlAndGenerate, createClackLogger, resolveLogger, silentLogger, stripBoilerplateFromCorpus } from "./_chunks/crawl.mjs";
|
|
2
2
|
function defineConfig(config) {
|
|
3
3
|
return config;
|
|
4
4
|
}
|
|
5
|
-
export { crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger };
|
|
5
|
+
export { crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger, stripBoilerplateFromCorpus };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.5.2",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -54,17 +54,17 @@
|
|
|
54
54
|
}
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
|
-
"@clack/prompts": "^1.
|
|
57
|
+
"@clack/prompts": "^1.7.0",
|
|
58
58
|
"c12": "^3.3.4",
|
|
59
59
|
"hookable": "^6.1.1",
|
|
60
|
-
"nypm": "^0.6.
|
|
60
|
+
"nypm": "^0.6.8",
|
|
61
61
|
"ofetch": "^1.5.1",
|
|
62
62
|
"pathe": "^2.0.3",
|
|
63
|
-
"picomatch": "^4.0.
|
|
64
|
-
"tldts": "^7.4.
|
|
63
|
+
"picomatch": "^4.0.5",
|
|
64
|
+
"tldts": "^7.4.8",
|
|
65
65
|
"ufo": "^1.6.4",
|
|
66
|
-
"@mdream/js": "1.
|
|
67
|
-
"mdream": "1.
|
|
66
|
+
"@mdream/js": "1.5.2",
|
|
67
|
+
"mdream": "1.5.2"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
70
|
"@types/picomatch": "^4.0.3"
|