@mdream/crawl 1.0.3 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/crawl.mjs +61 -44
- package/dist/_chunks/playwright-utils.mjs +0 -2
- package/dist/cli.mjs +9 -17
- package/dist/index.d.mts +0 -4
- package/dist/index.mjs +0 -2
- package/package.json +8 -8
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -9,24 +9,14 @@ import { dirname, join, normalize, resolve } from "pathe";
|
|
|
9
9
|
import { withHttps } from "ufo";
|
|
10
10
|
import picomatch from "picomatch";
|
|
11
11
|
import { getDomain } from "tldts";
|
|
12
|
-
//#region src/glob-utils.ts
|
|
13
12
|
function stripGlobTail(s) {
|
|
14
13
|
const idx = s.indexOf("*");
|
|
15
14
|
return idx === -1 ? s : s.slice(0, idx);
|
|
16
15
|
}
|
|
17
16
|
const GLOB_CHAR_RE = /[*?[]/;
|
|
18
|
-
/**
|
|
19
|
-
* Extract the registrable domain from a hostname using the public suffix list.
|
|
20
|
-
* Handles multi-part TLDs (.co.uk, .github.io, etc.) correctly.
|
|
21
|
-
* Returns the hostname unchanged for IPs or when parsing fails.
|
|
22
|
-
*/
|
|
23
17
|
function getRegistrableDomain(hostname) {
|
|
24
18
|
return getDomain(hostname, { allowPrivateDomains: true }) || hostname;
|
|
25
19
|
}
|
|
26
|
-
/**
|
|
27
|
-
* Parse a URL that may contain glob patterns
|
|
28
|
-
* Example: https://nuxtseo.com/docs/** -> { baseUrl: "https://nuxtseo.com", pattern: "/docs/**", isGlob: true }
|
|
29
|
-
*/
|
|
30
20
|
function parseUrlPattern(input) {
|
|
31
21
|
if (!(input.includes("*") || input.includes("?") || input.includes("["))) return {
|
|
32
22
|
baseUrl: input,
|
|
@@ -47,9 +37,6 @@ function parseUrlPattern(input) {
|
|
|
47
37
|
throw new Error(`Invalid URL pattern: "${input}". Please provide a valid URL with glob patterns (e.g., "example.com/docs/*" or "https://example.com/api/**").`);
|
|
48
38
|
}
|
|
49
39
|
}
|
|
50
|
-
/**
|
|
51
|
-
* Check if a URL matches a glob pattern
|
|
52
|
-
*/
|
|
53
40
|
function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
|
|
54
41
|
if (!parsedPattern.isGlob) return true;
|
|
55
42
|
try {
|
|
@@ -69,10 +56,6 @@ function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
|
|
|
69
56
|
return false;
|
|
70
57
|
}
|
|
71
58
|
}
|
|
72
|
-
/**
|
|
73
|
-
* Get the starting URL for crawling from a glob pattern
|
|
74
|
-
* For https://nuxtseo.com/docs/**, we want to start at https://nuxtseo.com
|
|
75
|
-
*/
|
|
76
59
|
function getStartingUrl(parsedPattern) {
|
|
77
60
|
if (!parsedPattern.isGlob) return withHttps(parsedPattern.baseUrl);
|
|
78
61
|
const pattern = parsedPattern.pattern;
|
|
@@ -83,9 +66,6 @@ function getStartingUrl(parsedPattern) {
|
|
|
83
66
|
const pathBeforeGlob = lastSlash >= 0 ? beforeGlob.substring(0, lastSlash + 1) : "/";
|
|
84
67
|
return withHttps(parsedPattern.baseUrl + pathBeforeGlob);
|
|
85
68
|
}
|
|
86
|
-
/**
|
|
87
|
-
* Check if a URL should be excluded based on exclude patterns
|
|
88
|
-
*/
|
|
89
69
|
function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
|
|
90
70
|
if (!excludePatterns || excludePatterns.length === 0) return false;
|
|
91
71
|
try {
|
|
@@ -104,15 +84,9 @@ function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
|
|
|
104
84
|
return false;
|
|
105
85
|
}
|
|
106
86
|
}
|
|
107
|
-
/**
|
|
108
|
-
* Check if a string is valid sitemap XML content (not an HTML page or other non-sitemap response)
|
|
109
|
-
*/
|
|
110
87
|
function isValidSitemapXml(content) {
|
|
111
88
|
return content.includes("<urlset") || content.includes("<sitemapindex");
|
|
112
89
|
}
|
|
113
|
-
/**
|
|
114
|
-
* Validate glob pattern syntax
|
|
115
|
-
*/
|
|
116
90
|
function validateGlobPattern(pattern) {
|
|
117
91
|
try {
|
|
118
92
|
parseUrlPattern(pattern);
|
|
@@ -121,8 +95,6 @@ function validateGlobPattern(pattern) {
|
|
|
121
95
|
return `Invalid glob pattern: ${error instanceof Error ? error.message : error}`;
|
|
122
96
|
}
|
|
123
97
|
}
|
|
124
|
-
//#endregion
|
|
125
|
-
//#region src/crawl.ts
|
|
126
98
|
const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
|
|
127
99
|
const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
|
|
128
100
|
const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
|
|
@@ -131,6 +103,40 @@ const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
|
|
|
131
103
|
const URL_TRAILING_SLASH_RE = /\/$/;
|
|
132
104
|
const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
|
|
133
105
|
const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
|
|
106
|
+
const TRACKING_PARAMS = new Set([
|
|
107
|
+
"utm_source",
|
|
108
|
+
"utm_medium",
|
|
109
|
+
"utm_campaign",
|
|
110
|
+
"utm_term",
|
|
111
|
+
"utm_content",
|
|
112
|
+
"utm_id",
|
|
113
|
+
"fbclid",
|
|
114
|
+
"gclid",
|
|
115
|
+
"gclsrc",
|
|
116
|
+
"dclid",
|
|
117
|
+
"gbraid",
|
|
118
|
+
"wbraid",
|
|
119
|
+
"msclkid",
|
|
120
|
+
"twclid",
|
|
121
|
+
"li_fat_id",
|
|
122
|
+
"mc_cid",
|
|
123
|
+
"mc_eid",
|
|
124
|
+
"ref",
|
|
125
|
+
"source",
|
|
126
|
+
"sessionid",
|
|
127
|
+
"session_id",
|
|
128
|
+
"sid",
|
|
129
|
+
"_ga",
|
|
130
|
+
"_gl",
|
|
131
|
+
"_hsenc",
|
|
132
|
+
"_hsmi",
|
|
133
|
+
"_openstat",
|
|
134
|
+
"yclid",
|
|
135
|
+
"ymclid",
|
|
136
|
+
"spm",
|
|
137
|
+
"scm"
|
|
138
|
+
]);
|
|
139
|
+
const MAX_QUERY_VARIANTS_PER_PATH = 5;
|
|
134
140
|
const FETCH_HEADERS = {
|
|
135
141
|
"User-Agent": "mdream-crawler/1.0",
|
|
136
142
|
"Accept": "text/html,application/xhtml+xml,text/markdown"
|
|
@@ -149,11 +155,18 @@ const IGNORED_PATH_PREFIXES = [
|
|
|
149
155
|
"/assets/",
|
|
150
156
|
"/static/"
|
|
151
157
|
];
|
|
152
|
-
const HTML_EXTENSIONS_RE = /\.(html?|php|aspx?|jsp)$/i;
|
|
158
|
+
const HTML_EXTENSIONS_RE = /\.(?:html?|php|aspx?|jsp)$/i;
|
|
153
159
|
function extractCdataUrl(url) {
|
|
154
160
|
if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) return url.slice(9, -3);
|
|
155
161
|
return url;
|
|
156
162
|
}
|
|
163
|
+
function normalizeUrl(resolved) {
|
|
164
|
+
resolved.hash = "";
|
|
165
|
+
const params = resolved.searchParams;
|
|
166
|
+
for (const key of [...params.keys()]) if (TRACKING_PARAMS.has(key)) params.delete(key);
|
|
167
|
+
params.sort();
|
|
168
|
+
return resolved.href.replace(URL_TRAILING_SLASH_RE, "") || resolved.href;
|
|
169
|
+
}
|
|
157
170
|
async function loadSitemap(sitemapUrl) {
|
|
158
171
|
const xmlContent = await ofetch(sitemapUrl, {
|
|
159
172
|
headers: FETCH_HEADERS,
|
|
@@ -200,8 +213,7 @@ function extractMetadataInline(parsedUrl, allowedDomains) {
|
|
|
200
213
|
const href = el.attributes.href;
|
|
201
214
|
if (href) try {
|
|
202
215
|
const resolved = new URL(href, url);
|
|
203
|
-
|
|
204
|
-
const absoluteUrl = resolved.href.replace(URL_TRAILING_SLASH_RE, "") || resolved.href;
|
|
216
|
+
const absoluteUrl = normalizeUrl(resolved);
|
|
205
217
|
if (allowedDomains) {
|
|
206
218
|
const domain = getRegistrableDomain(resolved.hostname);
|
|
207
219
|
if (domain && allowedDomains.has(domain)) links.add(absoluteUrl);
|
|
@@ -284,6 +296,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
284
296
|
generation: { status: "idle" }
|
|
285
297
|
};
|
|
286
298
|
const sitemapAttempts = [];
|
|
299
|
+
let sitemapProvidedUrls = false;
|
|
287
300
|
if (startingUrls.length > 0 && !skipSitemap && !singlePageMode) {
|
|
288
301
|
const baseUrl = new URL(startingUrls[0]).origin;
|
|
289
302
|
const homePageUrl = baseUrl;
|
|
@@ -403,15 +416,11 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
403
416
|
}
|
|
404
417
|
const successfulSitemaps = sitemapAttempts.filter((a) => a.success);
|
|
405
418
|
const failedSitemaps = sitemapAttempts.filter((a) => !a.success);
|
|
406
|
-
if (successfulSitemaps.length > 0) {
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
const firstAttempt = failedSitemaps[0];
|
|
412
|
-
if (firstAttempt.error?.includes("404")) p.note(`No sitemap found, using crawler to discover pages`, "Sitemap Discovery");
|
|
413
|
-
else p.note(`Could not access sitemap: ${firstAttempt.error}`, "Sitemap Discovery");
|
|
414
|
-
}
|
|
419
|
+
if (successfulSitemaps.length > 0 && progress.sitemap.processed > 0) {
|
|
420
|
+
sitemapProvidedUrls = true;
|
|
421
|
+
p.log.info(`Sitemap: ${progress.sitemap.processed} URLs from ${successfulSitemaps[0].url}`);
|
|
422
|
+
} else if (successfulSitemaps.length > 0) p.log.info(`Sitemap: found but no URLs matched filters`);
|
|
423
|
+
else if (failedSitemaps.length > 0) p.log.info(`Sitemap: not found, discovering pages via crawler`);
|
|
415
424
|
if (!startingUrls.includes(homePageUrl)) startingUrls.unshift(homePageUrl);
|
|
416
425
|
progress.sitemap.status = "completed";
|
|
417
426
|
progress.crawling.total = startingUrls.length;
|
|
@@ -465,6 +474,8 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
465
474
|
const normalizedHomePageUrl = (startingUrls.length > 0 ? new URL(startingUrls[0]).origin : "").replace(URL_TRAILING_SLASH_RE, "");
|
|
466
475
|
const createdDirs = /* @__PURE__ */ new Set();
|
|
467
476
|
const sharedOrigin = origin || "";
|
|
477
|
+
const pendingUrls = [];
|
|
478
|
+
const queryVariantCounts = /* @__PURE__ */ new Map();
|
|
468
479
|
const processPage = async (url, content, initialTitle, depth, isMarkdown = false) => {
|
|
469
480
|
const parsedUrl = new URL(url);
|
|
470
481
|
const shouldProcessMarkdown = shouldCrawlUrl(url);
|
|
@@ -536,9 +547,17 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
536
547
|
progress.crawling.processed = results.length;
|
|
537
548
|
onProgress?.(progress);
|
|
538
549
|
}
|
|
539
|
-
if (followLinks && !singlePageMode && depth < maxDepth) {
|
|
550
|
+
if (followLinks && !singlePageMode && !sitemapProvidedUrls && depth < maxDepth) {
|
|
540
551
|
const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link) && isContentUrl(link));
|
|
541
|
-
for (const link of filteredLinks)
|
|
552
|
+
for (const link of filteredLinks) {
|
|
553
|
+
if (processedUrls.has(link)) continue;
|
|
554
|
+
const linkUrl = new URL(link);
|
|
555
|
+
if (linkUrl.search) {
|
|
556
|
+
const pathname = linkUrl.pathname;
|
|
557
|
+
const count = queryVariantCounts.get(pathname) || 0;
|
|
558
|
+
if (count >= MAX_QUERY_VARIANTS_PER_PATH) continue;
|
|
559
|
+
queryVariantCounts.set(pathname, count + 1);
|
|
560
|
+
}
|
|
542
561
|
processedUrls.add(link);
|
|
543
562
|
pendingUrls.push({
|
|
544
563
|
url: link,
|
|
@@ -547,7 +566,6 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
547
566
|
}
|
|
548
567
|
}
|
|
549
568
|
};
|
|
550
|
-
const pendingUrls = [];
|
|
551
569
|
const urlsToProcess = startingUrls.slice(0, maxRequestsPerCrawl);
|
|
552
570
|
for (const url of urlsToProcess) processedUrls.add(url.replace(URL_TRAILING_SLASH_RE, "") || url);
|
|
553
571
|
let totalProcessed = 0;
|
|
@@ -756,5 +774,4 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
756
774
|
}
|
|
757
775
|
return results;
|
|
758
776
|
}
|
|
759
|
-
//#endregion
|
|
760
777
|
export { parseUrlPattern as n, validateGlobPattern as r, crawlAndGenerate as t };
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import * as p from "@clack/prompts";
|
|
2
2
|
import { addDependency } from "nypm";
|
|
3
|
-
//#region src/playwright-utils.ts
|
|
4
3
|
async function checkPlaywrightInstallation() {
|
|
5
4
|
try {
|
|
6
5
|
await import("playwright");
|
|
@@ -55,5 +54,4 @@ async function isUseChromeSupported() {
|
|
|
55
54
|
} catch {}
|
|
56
55
|
return false;
|
|
57
56
|
}
|
|
58
|
-
//#endregion
|
|
59
57
|
export { ensurePlaywrightInstalled, isUseChromeSupported };
|
package/dist/cli.mjs
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { n as parseUrlPattern, r as validateGlobPattern, t as crawlAndGenerate } from "./_chunks/crawl.mjs";
|
|
2
2
|
import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
3
|
import * as p from "@clack/prompts";
|
|
4
|
-
import { dirname, join, resolve } from "pathe";
|
|
4
|
+
import { dirname, join, relative, resolve } from "pathe";
|
|
5
5
|
import { withHttps } from "ufo";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { loadConfig } from "c12";
|
|
8
|
-
//#region src/config.ts
|
|
9
8
|
async function loadMdreamConfig(cwd) {
|
|
10
9
|
const { config } = await loadConfig({
|
|
11
10
|
name: "mdream",
|
|
@@ -13,8 +12,6 @@ async function loadMdreamConfig(cwd) {
|
|
|
13
12
|
});
|
|
14
13
|
return config || {};
|
|
15
14
|
}
|
|
16
|
-
//#endregion
|
|
17
|
-
//#region src/cli.ts
|
|
18
15
|
const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
|
|
19
16
|
const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
|
|
20
17
|
function checkOutputDirectoryPermissions(outputDir) {
|
|
@@ -152,7 +149,7 @@ async function interactiveCrawl() {
|
|
|
152
149
|
});
|
|
153
150
|
const summary = [
|
|
154
151
|
`URLs: ${urls.join(", ")}`,
|
|
155
|
-
`Output: ${outputDir}`,
|
|
152
|
+
`Output: ${relative(process.cwd(), outputDir) || outputDir}`,
|
|
156
153
|
`Driver: ${crawlerOptions.driver}`,
|
|
157
154
|
`Max pages: Unlimited`,
|
|
158
155
|
`Follow links: Yes (depth 3)`,
|
|
@@ -179,19 +176,16 @@ async function interactiveCrawl() {
|
|
|
179
176
|
};
|
|
180
177
|
}
|
|
181
178
|
async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
messages.push(`📄 ${successful} pages \u00B7 ⏱️ ${durationStr}`);
|
|
185
|
-
if (failed > 0) messages.push(`⚠️ ${failed} failed`);
|
|
179
|
+
let line = `${successful} pages in ${`${durationSeconds.toFixed(1)}s`}`;
|
|
180
|
+
if (failed > 0) line += ` (${failed} failed)`;
|
|
186
181
|
if (latency && latency.count > 0) {
|
|
187
182
|
const avg = Math.round(latency.total / latency.count);
|
|
188
183
|
const min = latency.min === Infinity ? 0 : Math.round(latency.min);
|
|
189
184
|
const max = Math.round(latency.max);
|
|
190
|
-
|
|
185
|
+
line += ` \u00B7 HTTP Latency: avg ${avg}ms, min ${min}ms, max ${max}ms`;
|
|
191
186
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
p.note(messages.join("\n"), "✅ Complete");
|
|
187
|
+
p.log.success(line);
|
|
188
|
+
p.log.info(`${generatedFiles.join(", ")} \u2192 ${relative(process.cwd(), outputDir) || "."}`);
|
|
195
189
|
}
|
|
196
190
|
function parseCliArgs() {
|
|
197
191
|
const args = process.argv.slice(2);
|
|
@@ -391,9 +385,8 @@ async function main() {
|
|
|
391
385
|
if (options.generateIndividualMd) formats.push("Individual MD files");
|
|
392
386
|
const summary = [
|
|
393
387
|
`URL: ${options.urls.join(", ")}`,
|
|
394
|
-
`Output: ${options.outputDir}`,
|
|
395
|
-
`Driver: ${options.driver}`,
|
|
396
|
-
`Depth: ${options.maxDepth}`,
|
|
388
|
+
`Output: ${relative(process.cwd(), options.outputDir) || "."}`,
|
|
389
|
+
`Driver: ${options.driver} \u00B7 Depth: ${options.maxDepth}`,
|
|
397
390
|
`Formats: ${formats.join(", ")}`,
|
|
398
391
|
options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
|
|
399
392
|
options.skipSitemap && `Skip sitemap: Yes`,
|
|
@@ -481,5 +474,4 @@ main().catch((error) => {
|
|
|
481
474
|
else p.log.error(`Unexpected error: ${msg}`);
|
|
482
475
|
process.exit(1);
|
|
483
476
|
});
|
|
484
|
-
//#endregion
|
|
485
477
|
export {};
|
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
//#region src/types.d.ts
|
|
2
1
|
interface PageData {
|
|
3
2
|
url: string;
|
|
4
3
|
html: string;
|
|
@@ -82,8 +81,6 @@ interface CrawlResult {
|
|
|
82
81
|
metadata?: PageMetadata;
|
|
83
82
|
depth?: number;
|
|
84
83
|
}
|
|
85
|
-
//#endregion
|
|
86
|
-
//#region src/crawl.d.ts
|
|
87
84
|
interface CrawlProgress {
|
|
88
85
|
sitemap: {
|
|
89
86
|
status: 'discovering' | 'processing' | 'completed';
|
|
@@ -109,5 +106,4 @@ interface CrawlProgress {
|
|
|
109
106
|
};
|
|
110
107
|
}
|
|
111
108
|
declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
|
|
112
|
-
//#endregion
|
|
113
109
|
export { type CrawlHooks, type CrawlOptions, type CrawlResult, type MdreamCrawlConfig, type PageData, crawlAndGenerate, defineConfig };
|
package/dist/index.mjs
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.0.
|
|
4
|
+
"version": "1.0.4",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -54,20 +54,20 @@
|
|
|
54
54
|
}
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
|
-
"@clack/prompts": "^1.
|
|
58
|
-
"c12": "^3.3.
|
|
57
|
+
"@clack/prompts": "^1.2.0",
|
|
58
|
+
"c12": "^3.3.4",
|
|
59
59
|
"hookable": "^6.1.0",
|
|
60
60
|
"nypm": "^0.6.5",
|
|
61
61
|
"ofetch": "^1.5.1",
|
|
62
62
|
"pathe": "^2.0.3",
|
|
63
|
-
"picomatch": "^4.0.
|
|
64
|
-
"tldts": "^7.0.
|
|
63
|
+
"picomatch": "^4.0.4",
|
|
64
|
+
"tldts": "^7.0.28",
|
|
65
65
|
"ufo": "^1.6.3",
|
|
66
|
-
"@mdream/js": "1.0.
|
|
67
|
-
"mdream": "1.0.
|
|
66
|
+
"@mdream/js": "1.0.4",
|
|
67
|
+
"mdream": "1.0.4"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
|
-
"@types/picomatch": "^4.0.
|
|
70
|
+
"@types/picomatch": "^4.0.3"
|
|
71
71
|
},
|
|
72
72
|
"scripts": {
|
|
73
73
|
"build": "obuild",
|