@mdream/crawl 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/crawl.mjs +172 -95
- package/dist/_chunks/playwright-utils.mjs +0 -2
- package/dist/cli.mjs +9 -17
- package/dist/index.d.mts +0 -4
- package/dist/index.mjs +0 -2
- package/package.json +9 -9
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -9,24 +9,14 @@ import { dirname, join, normalize, resolve } from "pathe";
|
|
|
9
9
|
import { withHttps } from "ufo";
|
|
10
10
|
import picomatch from "picomatch";
|
|
11
11
|
import { getDomain } from "tldts";
|
|
12
|
-
//#region src/glob-utils.ts
|
|
13
12
|
function stripGlobTail(s) {
|
|
14
13
|
const idx = s.indexOf("*");
|
|
15
14
|
return idx === -1 ? s : s.slice(0, idx);
|
|
16
15
|
}
|
|
17
16
|
const GLOB_CHAR_RE = /[*?[]/;
|
|
18
|
-
/**
|
|
19
|
-
* Extract the registrable domain from a hostname using the public suffix list.
|
|
20
|
-
* Handles multi-part TLDs (.co.uk, .github.io, etc.) correctly.
|
|
21
|
-
* Returns the hostname unchanged for IPs or when parsing fails.
|
|
22
|
-
*/
|
|
23
17
|
function getRegistrableDomain(hostname) {
|
|
24
18
|
return getDomain(hostname, { allowPrivateDomains: true }) || hostname;
|
|
25
19
|
}
|
|
26
|
-
/**
|
|
27
|
-
* Parse a URL that may contain glob patterns
|
|
28
|
-
* Example: https://nuxtseo.com/docs/** -> { baseUrl: "https://nuxtseo.com", pattern: "/docs/**", isGlob: true }
|
|
29
|
-
*/
|
|
30
20
|
function parseUrlPattern(input) {
|
|
31
21
|
if (!(input.includes("*") || input.includes("?") || input.includes("["))) return {
|
|
32
22
|
baseUrl: input,
|
|
@@ -47,9 +37,6 @@ function parseUrlPattern(input) {
|
|
|
47
37
|
throw new Error(`Invalid URL pattern: "${input}". Please provide a valid URL with glob patterns (e.g., "example.com/docs/*" or "https://example.com/api/**").`);
|
|
48
38
|
}
|
|
49
39
|
}
|
|
50
|
-
/**
|
|
51
|
-
* Check if a URL matches a glob pattern
|
|
52
|
-
*/
|
|
53
40
|
function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
|
|
54
41
|
if (!parsedPattern.isGlob) return true;
|
|
55
42
|
try {
|
|
@@ -69,10 +56,6 @@ function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
|
|
|
69
56
|
return false;
|
|
70
57
|
}
|
|
71
58
|
}
|
|
72
|
-
/**
|
|
73
|
-
* Get the starting URL for crawling from a glob pattern
|
|
74
|
-
* For https://nuxtseo.com/docs/**, we want to start at https://nuxtseo.com
|
|
75
|
-
*/
|
|
76
59
|
function getStartingUrl(parsedPattern) {
|
|
77
60
|
if (!parsedPattern.isGlob) return withHttps(parsedPattern.baseUrl);
|
|
78
61
|
const pattern = parsedPattern.pattern;
|
|
@@ -83,9 +66,6 @@ function getStartingUrl(parsedPattern) {
|
|
|
83
66
|
const pathBeforeGlob = lastSlash >= 0 ? beforeGlob.substring(0, lastSlash + 1) : "/";
|
|
84
67
|
return withHttps(parsedPattern.baseUrl + pathBeforeGlob);
|
|
85
68
|
}
|
|
86
|
-
/**
|
|
87
|
-
* Check if a URL should be excluded based on exclude patterns
|
|
88
|
-
*/
|
|
89
69
|
function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
|
|
90
70
|
if (!excludePatterns || excludePatterns.length === 0) return false;
|
|
91
71
|
try {
|
|
@@ -104,15 +84,9 @@ function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
|
|
|
104
84
|
return false;
|
|
105
85
|
}
|
|
106
86
|
}
|
|
107
|
-
/**
|
|
108
|
-
* Check if a string is valid sitemap XML content (not an HTML page or other non-sitemap response)
|
|
109
|
-
*/
|
|
110
87
|
function isValidSitemapXml(content) {
|
|
111
88
|
return content.includes("<urlset") || content.includes("<sitemapindex");
|
|
112
89
|
}
|
|
113
|
-
/**
|
|
114
|
-
* Validate glob pattern syntax
|
|
115
|
-
*/
|
|
116
90
|
function validateGlobPattern(pattern) {
|
|
117
91
|
try {
|
|
118
92
|
parseUrlPattern(pattern);
|
|
@@ -121,8 +95,6 @@ function validateGlobPattern(pattern) {
|
|
|
121
95
|
return `Invalid glob pattern: ${error instanceof Error ? error.message : error}`;
|
|
122
96
|
}
|
|
123
97
|
}
|
|
124
|
-
//#endregion
|
|
125
|
-
//#region src/crawl.ts
|
|
126
98
|
const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
|
|
127
99
|
const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
|
|
128
100
|
const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
|
|
@@ -131,15 +103,70 @@ const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
|
|
|
131
103
|
const URL_TRAILING_SLASH_RE = /\/$/;
|
|
132
104
|
const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
|
|
133
105
|
const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
|
|
106
|
+
const TRACKING_PARAMS = new Set([
|
|
107
|
+
"utm_source",
|
|
108
|
+
"utm_medium",
|
|
109
|
+
"utm_campaign",
|
|
110
|
+
"utm_term",
|
|
111
|
+
"utm_content",
|
|
112
|
+
"utm_id",
|
|
113
|
+
"fbclid",
|
|
114
|
+
"gclid",
|
|
115
|
+
"gclsrc",
|
|
116
|
+
"dclid",
|
|
117
|
+
"gbraid",
|
|
118
|
+
"wbraid",
|
|
119
|
+
"msclkid",
|
|
120
|
+
"twclid",
|
|
121
|
+
"li_fat_id",
|
|
122
|
+
"mc_cid",
|
|
123
|
+
"mc_eid",
|
|
124
|
+
"ref",
|
|
125
|
+
"source",
|
|
126
|
+
"sessionid",
|
|
127
|
+
"session_id",
|
|
128
|
+
"sid",
|
|
129
|
+
"_ga",
|
|
130
|
+
"_gl",
|
|
131
|
+
"_hsenc",
|
|
132
|
+
"_hsmi",
|
|
133
|
+
"_openstat",
|
|
134
|
+
"yclid",
|
|
135
|
+
"ymclid",
|
|
136
|
+
"spm",
|
|
137
|
+
"scm"
|
|
138
|
+
]);
|
|
139
|
+
const MAX_QUERY_VARIANTS_PER_PATH = 5;
|
|
134
140
|
const FETCH_HEADERS = {
|
|
135
141
|
"User-Agent": "mdream-crawler/1.0",
|
|
136
142
|
"Accept": "text/html,application/xhtml+xml,text/markdown"
|
|
137
143
|
};
|
|
138
144
|
const DEFAULT_CONCURRENCY = 20;
|
|
145
|
+
const IGNORED_PATH_PREFIXES = [
|
|
146
|
+
"/cdn-cgi/",
|
|
147
|
+
"/_next/",
|
|
148
|
+
"/_nuxt/",
|
|
149
|
+
"/__",
|
|
150
|
+
"/wp-admin/",
|
|
151
|
+
"/wp-json/",
|
|
152
|
+
"/wp-includes/",
|
|
153
|
+
"/wp-content/uploads/",
|
|
154
|
+
"/api/",
|
|
155
|
+
"/assets/",
|
|
156
|
+
"/static/"
|
|
157
|
+
];
|
|
158
|
+
const HTML_EXTENSIONS_RE = /\.(?:html?|php|aspx?|jsp)$/i;
|
|
139
159
|
function extractCdataUrl(url) {
|
|
140
160
|
if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) return url.slice(9, -3);
|
|
141
161
|
return url;
|
|
142
162
|
}
|
|
163
|
+
function normalizeUrl(resolved) {
|
|
164
|
+
resolved.hash = "";
|
|
165
|
+
const params = resolved.searchParams;
|
|
166
|
+
for (const key of [...params.keys()]) if (TRACKING_PARAMS.has(key)) params.delete(key);
|
|
167
|
+
params.sort();
|
|
168
|
+
return resolved.href.replace(URL_TRAILING_SLASH_RE, "") || resolved.href;
|
|
169
|
+
}
|
|
143
170
|
async function loadSitemap(sitemapUrl) {
|
|
144
171
|
const xmlContent = await ofetch(sitemapUrl, {
|
|
145
172
|
headers: FETCH_HEADERS,
|
|
@@ -186,7 +213,7 @@ function extractMetadataInline(parsedUrl, allowedDomains) {
|
|
|
186
213
|
const href = el.attributes.href;
|
|
187
214
|
if (href) try {
|
|
188
215
|
const resolved = new URL(href, url);
|
|
189
|
-
const absoluteUrl = resolved
|
|
216
|
+
const absoluteUrl = normalizeUrl(resolved);
|
|
190
217
|
if (allowedDomains) {
|
|
191
218
|
const domain = getRegistrableDomain(resolved.hostname);
|
|
192
219
|
if (domain && allowedDomains.has(domain)) links.add(absoluteUrl);
|
|
@@ -269,6 +296,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
269
296
|
generation: { status: "idle" }
|
|
270
297
|
};
|
|
271
298
|
const sitemapAttempts = [];
|
|
299
|
+
let sitemapProvidedUrls = false;
|
|
272
300
|
if (startingUrls.length > 0 && !skipSitemap && !singlePageMode) {
|
|
273
301
|
const baseUrl = new URL(startingUrls[0]).origin;
|
|
274
302
|
const homePageUrl = baseUrl;
|
|
@@ -388,15 +416,11 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
388
416
|
}
|
|
389
417
|
const successfulSitemaps = sitemapAttempts.filter((a) => a.success);
|
|
390
418
|
const failedSitemaps = sitemapAttempts.filter((a) => !a.success);
|
|
391
|
-
if (successfulSitemaps.length > 0) {
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
const firstAttempt = failedSitemaps[0];
|
|
397
|
-
if (firstAttempt.error?.includes("404")) p.note(`No sitemap found, using crawler to discover pages`, "Sitemap Discovery");
|
|
398
|
-
else p.note(`Could not access sitemap: ${firstAttempt.error}`, "Sitemap Discovery");
|
|
399
|
-
}
|
|
419
|
+
if (successfulSitemaps.length > 0 && progress.sitemap.processed > 0) {
|
|
420
|
+
sitemapProvidedUrls = true;
|
|
421
|
+
p.log.info(`Sitemap: ${progress.sitemap.processed} URLs from ${successfulSitemaps[0].url}`);
|
|
422
|
+
} else if (successfulSitemaps.length > 0) p.log.info(`Sitemap: found but no URLs matched filters`);
|
|
423
|
+
else if (failedSitemaps.length > 0) p.log.info(`Sitemap: not found, discovering pages via crawler`);
|
|
400
424
|
if (!startingUrls.includes(homePageUrl)) startingUrls.unshift(homePageUrl);
|
|
401
425
|
progress.sitemap.status = "completed";
|
|
402
426
|
progress.crawling.total = startingUrls.length;
|
|
@@ -418,6 +442,16 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
418
442
|
return "";
|
|
419
443
|
}
|
|
420
444
|
}).filter(Boolean)) : void 0;
|
|
445
|
+
const isContentUrl = (url) => {
|
|
446
|
+
try {
|
|
447
|
+
const pathname = new URL(url).pathname;
|
|
448
|
+
for (let i = 0; i < IGNORED_PATH_PREFIXES.length; i++) if (pathname.startsWith(IGNORED_PATH_PREFIXES[i])) return false;
|
|
449
|
+
if (pathname.lastIndexOf(".") > pathname.lastIndexOf("/")) return HTML_EXTENSIONS_RE.test(pathname);
|
|
450
|
+
return true;
|
|
451
|
+
} catch {
|
|
452
|
+
return false;
|
|
453
|
+
}
|
|
454
|
+
};
|
|
421
455
|
const shouldCrawlUrl = (url) => {
|
|
422
456
|
if (isUrlExcluded(url, exclude, allowSubdomains)) return false;
|
|
423
457
|
if (!hasGlobPatterns) {
|
|
@@ -440,6 +474,8 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
440
474
|
const normalizedHomePageUrl = (startingUrls.length > 0 ? new URL(startingUrls[0]).origin : "").replace(URL_TRAILING_SLASH_RE, "");
|
|
441
475
|
const createdDirs = /* @__PURE__ */ new Set();
|
|
442
476
|
const sharedOrigin = origin || "";
|
|
477
|
+
const pendingUrls = [];
|
|
478
|
+
const queryVariantCounts = /* @__PURE__ */ new Map();
|
|
443
479
|
const processPage = async (url, content, initialTitle, depth, isMarkdown = false) => {
|
|
444
480
|
const parsedUrl = new URL(url);
|
|
445
481
|
const shouldProcessMarkdown = shouldCrawlUrl(url);
|
|
@@ -511,12 +547,28 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
511
547
|
progress.crawling.processed = results.length;
|
|
512
548
|
onProgress?.(progress);
|
|
513
549
|
}
|
|
514
|
-
if (followLinks && !singlePageMode && depth < maxDepth) {
|
|
515
|
-
const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link));
|
|
516
|
-
for (const link of filteredLinks)
|
|
550
|
+
if (followLinks && !singlePageMode && !sitemapProvidedUrls && depth < maxDepth) {
|
|
551
|
+
const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link) && isContentUrl(link));
|
|
552
|
+
for (const link of filteredLinks) {
|
|
553
|
+
if (processedUrls.has(link)) continue;
|
|
554
|
+
const linkUrl = new URL(link);
|
|
555
|
+
if (linkUrl.search) {
|
|
556
|
+
const pathname = linkUrl.pathname;
|
|
557
|
+
const count = queryVariantCounts.get(pathname) || 0;
|
|
558
|
+
if (count >= MAX_QUERY_VARIANTS_PER_PATH) continue;
|
|
559
|
+
queryVariantCounts.set(pathname, count + 1);
|
|
560
|
+
}
|
|
561
|
+
processedUrls.add(link);
|
|
562
|
+
pendingUrls.push({
|
|
563
|
+
url: link,
|
|
564
|
+
depth: depth + 1
|
|
565
|
+
});
|
|
566
|
+
}
|
|
517
567
|
}
|
|
518
568
|
};
|
|
519
569
|
const urlsToProcess = startingUrls.slice(0, maxRequestsPerCrawl);
|
|
570
|
+
for (const url of urlsToProcess) processedUrls.add(url.replace(URL_TRAILING_SLASH_RE, "") || url);
|
|
571
|
+
let totalProcessed = 0;
|
|
520
572
|
progress.crawling.status = "processing";
|
|
521
573
|
progress.crawling.total = urlsToProcess.length;
|
|
522
574
|
onProgress?.(progress);
|
|
@@ -569,12 +621,24 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
569
621
|
useChrome
|
|
570
622
|
};
|
|
571
623
|
const crawler = new PlaywrightCrawler(crawlerOptions);
|
|
572
|
-
const
|
|
624
|
+
const allRequests = [...urlsToProcess.map((url) => ({
|
|
573
625
|
url,
|
|
574
626
|
userData: { depth: 0 }
|
|
575
|
-
}));
|
|
627
|
+
}))];
|
|
576
628
|
try {
|
|
577
|
-
await crawler.run(
|
|
629
|
+
await crawler.run(allRequests);
|
|
630
|
+
totalProcessed += urlsToProcess.length;
|
|
631
|
+
while (pendingUrls.length > 0 && totalProcessed < maxRequestsPerCrawl) {
|
|
632
|
+
const batch = pendingUrls.splice(0, maxRequestsPerCrawl - totalProcessed);
|
|
633
|
+
progress.crawling.total += batch.length;
|
|
634
|
+
onProgress?.(progress);
|
|
635
|
+
const batchRequests = batch.map((item) => ({
|
|
636
|
+
url: item.url,
|
|
637
|
+
userData: { depth: item.depth }
|
|
638
|
+
}));
|
|
639
|
+
await crawler.run(batchRequests);
|
|
640
|
+
totalProcessed += batch.length;
|
|
641
|
+
}
|
|
578
642
|
} catch (error) {
|
|
579
643
|
const msg = error instanceof Error ? error.message : "";
|
|
580
644
|
if (msg.includes("wmic") || msg.includes("ENOENT")) throw new Error(`Crawlee failed to spawn a system process (${msg}). On Windows 11+, wmic.exe is no longer available. Upgrade crawlee to >=3.16.0 or use the HTTP driver instead (--driver http).`);
|
|
@@ -585,60 +649,74 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
585
649
|
throw error;
|
|
586
650
|
}
|
|
587
651
|
await purgeDefaultStorages();
|
|
588
|
-
} else
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
skip: false
|
|
598
|
-
};
|
|
599
|
-
await hooks.callHook("crawl:url", urlCtx);
|
|
600
|
-
if (urlCtx.skip) return;
|
|
601
|
-
try {
|
|
602
|
-
const fetchStart = Date.now();
|
|
603
|
-
const response = await ofetch.raw(url, {
|
|
604
|
-
headers: FETCH_HEADERS,
|
|
605
|
-
responseType: "text",
|
|
606
|
-
retry: 2,
|
|
607
|
-
retryDelay: 500,
|
|
608
|
-
timeout: 1e4,
|
|
609
|
-
onResponseError({ response }) {
|
|
610
|
-
if (response.status === 429) {
|
|
611
|
-
const retryAfter = response.headers.get("retry-after");
|
|
612
|
-
const delaySec = retryAfter ? Number.parseInt(retryAfter) || 1 : 2;
|
|
613
|
-
if (!crawlDelay || delaySec > crawlDelay) crawlDelay = delaySec;
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
});
|
|
617
|
-
recordLatency(Date.now() - fetchStart);
|
|
618
|
-
const body = response._data ?? "";
|
|
619
|
-
const contentType = response.headers.get("content-type") || "";
|
|
620
|
-
await processPage(url, body, "", 0, contentType.includes("text/markdown") || contentType.includes("text/x-markdown"));
|
|
621
|
-
} catch (error) {
|
|
622
|
-
if (verbose) console.error(`[ERROR] URL: ${url}, Error: ${error instanceof Error ? error.message : "Unknown"}`);
|
|
623
|
-
progress.crawling.failed++;
|
|
624
|
-
results.push({
|
|
652
|
+
} else {
|
|
653
|
+
const fetchPage = async (url, depth) => {
|
|
654
|
+
progress.crawling.currentUrl = url;
|
|
655
|
+
onProgress?.(progress);
|
|
656
|
+
if (crawlDelay) {
|
|
657
|
+
const delay = crawlDelay;
|
|
658
|
+
await new Promise((resolve) => setTimeout(resolve, delay * 1e3));
|
|
659
|
+
}
|
|
660
|
+
const urlCtx = {
|
|
625
661
|
url,
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
662
|
+
skip: false
|
|
663
|
+
};
|
|
664
|
+
await hooks.callHook("crawl:url", urlCtx);
|
|
665
|
+
if (urlCtx.skip) return;
|
|
666
|
+
try {
|
|
667
|
+
const fetchStart = Date.now();
|
|
668
|
+
const response = await ofetch.raw(url, {
|
|
669
|
+
headers: FETCH_HEADERS,
|
|
670
|
+
responseType: "text",
|
|
671
|
+
retry: 2,
|
|
672
|
+
retryDelay: 500,
|
|
673
|
+
timeout: 1e4,
|
|
674
|
+
onResponseError({ response }) {
|
|
675
|
+
if (response.status === 429) {
|
|
676
|
+
const retryAfter = response.headers.get("retry-after");
|
|
677
|
+
const delaySec = retryAfter ? Number.parseInt(retryAfter) || 1 : 2;
|
|
678
|
+
if (!crawlDelay || delaySec > crawlDelay) crawlDelay = delaySec;
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
});
|
|
682
|
+
recordLatency(Date.now() - fetchStart);
|
|
683
|
+
const body = response._data ?? "";
|
|
684
|
+
const contentType = response.headers.get("content-type") || "";
|
|
685
|
+
const isHtml = contentType.includes("text/html") || contentType.includes("application/xhtml");
|
|
686
|
+
const isMarkdown = contentType.includes("text/markdown") || contentType.includes("text/x-markdown");
|
|
687
|
+
if (!isHtml && !isMarkdown) return;
|
|
688
|
+
await processPage(url, body, "", depth, isMarkdown);
|
|
689
|
+
} catch (error) {
|
|
690
|
+
if (verbose) console.error(`[ERROR] URL: ${url}, Error: ${error instanceof Error ? error.message : "Unknown"}`);
|
|
691
|
+
progress.crawling.failed++;
|
|
692
|
+
results.push({
|
|
693
|
+
url,
|
|
632
694
|
title: "",
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
695
|
+
content: "",
|
|
696
|
+
timestamp: Date.now(),
|
|
697
|
+
success: false,
|
|
698
|
+
error: error instanceof Error ? error.message : "Unknown error",
|
|
699
|
+
metadata: {
|
|
700
|
+
title: "",
|
|
701
|
+
description: "",
|
|
702
|
+
links: []
|
|
703
|
+
},
|
|
704
|
+
depth
|
|
705
|
+
});
|
|
706
|
+
progress.crawling.processed = results.length;
|
|
707
|
+
onProgress?.(progress);
|
|
708
|
+
}
|
|
709
|
+
};
|
|
710
|
+
await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, (url) => fetchPage(url, 0));
|
|
711
|
+
totalProcessed += urlsToProcess.length;
|
|
712
|
+
while (pendingUrls.length > 0 && totalProcessed < maxRequestsPerCrawl) {
|
|
713
|
+
const batch = pendingUrls.splice(0, maxRequestsPerCrawl - totalProcessed);
|
|
714
|
+
progress.crawling.total += batch.length;
|
|
639
715
|
onProgress?.(progress);
|
|
716
|
+
await runConcurrent(batch, DEFAULT_CONCURRENCY, (item) => fetchPage(item.url, item.depth));
|
|
717
|
+
totalProcessed += batch.length;
|
|
640
718
|
}
|
|
641
|
-
}
|
|
719
|
+
}
|
|
642
720
|
progress.crawling.status = "completed";
|
|
643
721
|
onProgress?.(progress);
|
|
644
722
|
await hooks.callHook("crawl:done", { results });
|
|
@@ -696,5 +774,4 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
696
774
|
}
|
|
697
775
|
return results;
|
|
698
776
|
}
|
|
699
|
-
//#endregion
|
|
700
777
|
export { parseUrlPattern as n, validateGlobPattern as r, crawlAndGenerate as t };
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import * as p from "@clack/prompts";
|
|
2
2
|
import { addDependency } from "nypm";
|
|
3
|
-
//#region src/playwright-utils.ts
|
|
4
3
|
async function checkPlaywrightInstallation() {
|
|
5
4
|
try {
|
|
6
5
|
await import("playwright");
|
|
@@ -55,5 +54,4 @@ async function isUseChromeSupported() {
|
|
|
55
54
|
} catch {}
|
|
56
55
|
return false;
|
|
57
56
|
}
|
|
58
|
-
//#endregion
|
|
59
57
|
export { ensurePlaywrightInstalled, isUseChromeSupported };
|
package/dist/cli.mjs
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { n as parseUrlPattern, r as validateGlobPattern, t as crawlAndGenerate } from "./_chunks/crawl.mjs";
|
|
2
2
|
import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
3
|
import * as p from "@clack/prompts";
|
|
4
|
-
import { dirname, join, resolve } from "pathe";
|
|
4
|
+
import { dirname, join, relative, resolve } from "pathe";
|
|
5
5
|
import { withHttps } from "ufo";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { loadConfig } from "c12";
|
|
8
|
-
//#region src/config.ts
|
|
9
8
|
async function loadMdreamConfig(cwd) {
|
|
10
9
|
const { config } = await loadConfig({
|
|
11
10
|
name: "mdream",
|
|
@@ -13,8 +12,6 @@ async function loadMdreamConfig(cwd) {
|
|
|
13
12
|
});
|
|
14
13
|
return config || {};
|
|
15
14
|
}
|
|
16
|
-
//#endregion
|
|
17
|
-
//#region src/cli.ts
|
|
18
15
|
const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
|
|
19
16
|
const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
|
|
20
17
|
function checkOutputDirectoryPermissions(outputDir) {
|
|
@@ -152,7 +149,7 @@ async function interactiveCrawl() {
|
|
|
152
149
|
});
|
|
153
150
|
const summary = [
|
|
154
151
|
`URLs: ${urls.join(", ")}`,
|
|
155
|
-
`Output: ${outputDir}`,
|
|
152
|
+
`Output: ${relative(process.cwd(), outputDir) || outputDir}`,
|
|
156
153
|
`Driver: ${crawlerOptions.driver}`,
|
|
157
154
|
`Max pages: Unlimited`,
|
|
158
155
|
`Follow links: Yes (depth 3)`,
|
|
@@ -179,19 +176,16 @@ async function interactiveCrawl() {
|
|
|
179
176
|
};
|
|
180
177
|
}
|
|
181
178
|
async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
messages.push(`📄 ${successful} pages \u00B7 ⏱️ ${durationStr}`);
|
|
185
|
-
if (failed > 0) messages.push(`⚠️ ${failed} failed`);
|
|
179
|
+
let line = `${successful} pages in ${`${durationSeconds.toFixed(1)}s`}`;
|
|
180
|
+
if (failed > 0) line += ` (${failed} failed)`;
|
|
186
181
|
if (latency && latency.count > 0) {
|
|
187
182
|
const avg = Math.round(latency.total / latency.count);
|
|
188
183
|
const min = latency.min === Infinity ? 0 : Math.round(latency.min);
|
|
189
184
|
const max = Math.round(latency.max);
|
|
190
|
-
|
|
185
|
+
line += ` \u00B7 HTTP Latency: avg ${avg}ms, min ${min}ms, max ${max}ms`;
|
|
191
186
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
p.note(messages.join("\n"), "✅ Complete");
|
|
187
|
+
p.log.success(line);
|
|
188
|
+
p.log.info(`${generatedFiles.join(", ")} \u2192 ${relative(process.cwd(), outputDir) || "."}`);
|
|
195
189
|
}
|
|
196
190
|
function parseCliArgs() {
|
|
197
191
|
const args = process.argv.slice(2);
|
|
@@ -391,9 +385,8 @@ async function main() {
|
|
|
391
385
|
if (options.generateIndividualMd) formats.push("Individual MD files");
|
|
392
386
|
const summary = [
|
|
393
387
|
`URL: ${options.urls.join(", ")}`,
|
|
394
|
-
`Output: ${options.outputDir}`,
|
|
395
|
-
`Driver: ${options.driver}`,
|
|
396
|
-
`Depth: ${options.maxDepth}`,
|
|
388
|
+
`Output: ${relative(process.cwd(), options.outputDir) || "."}`,
|
|
389
|
+
`Driver: ${options.driver} \u00B7 Depth: ${options.maxDepth}`,
|
|
397
390
|
`Formats: ${formats.join(", ")}`,
|
|
398
391
|
options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
|
|
399
392
|
options.skipSitemap && `Skip sitemap: Yes`,
|
|
@@ -481,5 +474,4 @@ main().catch((error) => {
|
|
|
481
474
|
else p.log.error(`Unexpected error: ${msg}`);
|
|
482
475
|
process.exit(1);
|
|
483
476
|
});
|
|
484
|
-
//#endregion
|
|
485
477
|
export {};
|
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
//#region src/types.d.ts
|
|
2
1
|
interface PageData {
|
|
3
2
|
url: string;
|
|
4
3
|
html: string;
|
|
@@ -82,8 +81,6 @@ interface CrawlResult {
|
|
|
82
81
|
metadata?: PageMetadata;
|
|
83
82
|
depth?: number;
|
|
84
83
|
}
|
|
85
|
-
//#endregion
|
|
86
|
-
//#region src/crawl.d.ts
|
|
87
84
|
interface CrawlProgress {
|
|
88
85
|
sitemap: {
|
|
89
86
|
status: 'discovering' | 'processing' | 'completed';
|
|
@@ -109,5 +106,4 @@ interface CrawlProgress {
|
|
|
109
106
|
};
|
|
110
107
|
}
|
|
111
108
|
declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
|
|
112
|
-
//#endregion
|
|
113
109
|
export { type CrawlHooks, type CrawlOptions, type CrawlResult, type MdreamCrawlConfig, type PageData, crawlAndGenerate, defineConfig };
|
package/dist/index.mjs
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.0.
|
|
4
|
+
"version": "1.0.4",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -54,20 +54,20 @@
|
|
|
54
54
|
}
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
|
-
"@clack/prompts": "^1.
|
|
58
|
-
"c12": "^3.
|
|
59
|
-
"hookable": "^
|
|
57
|
+
"@clack/prompts": "^1.2.0",
|
|
58
|
+
"c12": "^3.3.4",
|
|
59
|
+
"hookable": "^6.1.0",
|
|
60
60
|
"nypm": "^0.6.5",
|
|
61
61
|
"ofetch": "^1.5.1",
|
|
62
62
|
"pathe": "^2.0.3",
|
|
63
|
-
"picomatch": "^4.0.
|
|
64
|
-
"tldts": "^7.0.
|
|
63
|
+
"picomatch": "^4.0.4",
|
|
64
|
+
"tldts": "^7.0.28",
|
|
65
65
|
"ufo": "^1.6.3",
|
|
66
|
-
"@mdream/js": "1.0.
|
|
67
|
-
"mdream": "1.0.
|
|
66
|
+
"@mdream/js": "1.0.4",
|
|
67
|
+
"mdream": "1.0.4"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
|
-
"@types/picomatch": "^4.0.
|
|
70
|
+
"@types/picomatch": "^4.0.3"
|
|
71
71
|
},
|
|
72
72
|
"scripts": {
|
|
73
73
|
"build": "obuild",
|