@mdream/crawl 2.0.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/crawl.mjs +35 -7
- package/package.json +4 -4
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -5,7 +5,7 @@ import { createHooks } from "hookable";
|
|
|
5
5
|
import { htmlToMarkdown } from "mdream";
|
|
6
6
|
import { ofetch } from "ofetch";
|
|
7
7
|
import { dirname, join, normalize, resolve } from "pathe";
|
|
8
|
-
import { withHttps } from "ufo";
|
|
8
|
+
import { hasProtocol, withHttps } from "ufo";
|
|
9
9
|
import picomatch from "picomatch";
|
|
10
10
|
import { getDomain } from "tldts";
|
|
11
11
|
import * as p from "@clack/prompts";
|
|
@@ -164,14 +164,14 @@ function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
|
|
|
164
164
|
}
|
|
165
165
|
}
|
|
166
166
|
function getStartingUrl(parsedPattern) {
|
|
167
|
-
|
|
167
|
+
const baseUrl = hasProtocol(parsedPattern.baseUrl) ? parsedPattern.baseUrl : withHttps(parsedPattern.baseUrl);
|
|
168
|
+
if (!parsedPattern.isGlob) return baseUrl;
|
|
168
169
|
const pattern = parsedPattern.pattern;
|
|
169
170
|
const firstGlobIndex = pattern.search(GLOB_CHAR_RE);
|
|
170
|
-
if (firstGlobIndex === -1) return
|
|
171
|
+
if (firstGlobIndex === -1) return baseUrl + pattern;
|
|
171
172
|
const beforeGlob = pattern.substring(0, firstGlobIndex);
|
|
172
173
|
const lastSlash = beforeGlob.lastIndexOf("/");
|
|
173
|
-
|
|
174
|
-
return withHttps(parsedPattern.baseUrl + pathBeforeGlob);
|
|
174
|
+
return baseUrl + (lastSlash >= 0 ? beforeGlob.substring(0, lastSlash + 1) : "/");
|
|
175
175
|
}
|
|
176
176
|
function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
|
|
177
177
|
if (!excludePatterns || excludePatterns.length === 0) return false;
|
|
@@ -455,6 +455,34 @@ const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
|
|
|
455
455
|
const URL_TRAILING_SLASH_RE = /\/$/;
|
|
456
456
|
const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
|
|
457
457
|
const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
|
|
458
|
+
const WINDOWS_DEVICE_NAMES = /* @__PURE__ */ new Set([
|
|
459
|
+
"con",
|
|
460
|
+
"prn",
|
|
461
|
+
"aux",
|
|
462
|
+
"nul",
|
|
463
|
+
"com1",
|
|
464
|
+
"com2",
|
|
465
|
+
"com3",
|
|
466
|
+
"com4",
|
|
467
|
+
"com5",
|
|
468
|
+
"com6",
|
|
469
|
+
"com7",
|
|
470
|
+
"com8",
|
|
471
|
+
"com9",
|
|
472
|
+
"lpt1",
|
|
473
|
+
"lpt2",
|
|
474
|
+
"lpt3",
|
|
475
|
+
"lpt4",
|
|
476
|
+
"lpt5",
|
|
477
|
+
"lpt6",
|
|
478
|
+
"lpt7",
|
|
479
|
+
"lpt8",
|
|
480
|
+
"lpt9"
|
|
481
|
+
]);
|
|
482
|
+
function sanitizeOutputSegment(segment) {
|
|
483
|
+
const safe = segment.replace(URL_PATH_UNSAFE_CHARS_RE, "-");
|
|
484
|
+
return safe.length >= 3 && safe.length <= 4 && WINDOWS_DEVICE_NAMES.has(safe.toLowerCase()) ? `~${safe}` : safe;
|
|
485
|
+
}
|
|
458
486
|
const TRACKING_PARAMS = /* @__PURE__ */ new Set([
|
|
459
487
|
"utm_source",
|
|
460
488
|
"utm_medium",
|
|
@@ -859,9 +887,9 @@ async function runCrawl(options, onProgress) {
|
|
|
859
887
|
let filePath;
|
|
860
888
|
if (shouldProcessMarkdown && writeMarkdown) {
|
|
861
889
|
const urlPath = parsedUrl.pathname === "/" ? "/index" : parsedUrl.pathname;
|
|
862
|
-
const hostPrefix = allowSubdomains ? [parsedUrl.hostname
|
|
890
|
+
const hostPrefix = allowSubdomains ? [sanitizeOutputSegment(parsedUrl.hostname)] : [];
|
|
863
891
|
const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
|
|
864
|
-
const safeSegments = [...hostPrefix, ...pathSegments.map(
|
|
892
|
+
const safeSegments = [...hostPrefix, ...pathSegments.map(sanitizeOutputSegment)];
|
|
865
893
|
const filename = safeSegments.length > 0 ? safeSegments.join("/") : "index";
|
|
866
894
|
const safeFilename = normalize(`${filename}.md`);
|
|
867
895
|
filePath = join(outputDir, safeFilename);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "2.0.
|
|
4
|
+
"version": "2.0.2",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -65,15 +65,15 @@
|
|
|
65
65
|
},
|
|
66
66
|
"dependencies": {
|
|
67
67
|
"@clack/prompts": "^1.8.1",
|
|
68
|
-
"@mdream/js": "2.0.
|
|
68
|
+
"@mdream/js": "2.0.2",
|
|
69
69
|
"c12": "4.0.0-rc.2",
|
|
70
70
|
"hookable": "^6.1.2",
|
|
71
|
-
"mdream": "2.0.
|
|
71
|
+
"mdream": "2.0.2",
|
|
72
72
|
"nypm": "^0.6.10",
|
|
73
73
|
"ofetch": "^1.5.1",
|
|
74
74
|
"pathe": "^2.0.3",
|
|
75
75
|
"picomatch": "^4.0.7",
|
|
76
|
-
"tldts": "^7.4.
|
|
76
|
+
"tldts": "^7.4.18",
|
|
77
77
|
"ufo": "^1.6.4"
|
|
78
78
|
},
|
|
79
79
|
"devDependencies": {
|