@mdream/crawl 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,7 +5,7 @@ import { createHooks } from "hookable";
5
5
  import { htmlToMarkdown } from "mdream";
6
6
  import { ofetch } from "ofetch";
7
7
  import { dirname, join, normalize, resolve } from "pathe";
8
- import { withHttps } from "ufo";
8
+ import { hasProtocol, withHttps } from "ufo";
9
9
  import picomatch from "picomatch";
10
10
  import { getDomain } from "tldts";
11
11
  import * as p from "@clack/prompts";
@@ -164,14 +164,14 @@ function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
164
164
  }
165
165
  }
166
166
  function getStartingUrl(parsedPattern) {
167
- if (!parsedPattern.isGlob) return withHttps(parsedPattern.baseUrl);
167
+ const baseUrl = hasProtocol(parsedPattern.baseUrl) ? parsedPattern.baseUrl : withHttps(parsedPattern.baseUrl);
168
+ if (!parsedPattern.isGlob) return baseUrl;
168
169
  const pattern = parsedPattern.pattern;
169
170
  const firstGlobIndex = pattern.search(GLOB_CHAR_RE);
170
- if (firstGlobIndex === -1) return withHttps(parsedPattern.baseUrl + pattern);
171
+ if (firstGlobIndex === -1) return baseUrl + pattern;
171
172
  const beforeGlob = pattern.substring(0, firstGlobIndex);
172
173
  const lastSlash = beforeGlob.lastIndexOf("/");
173
- const pathBeforeGlob = lastSlash >= 0 ? beforeGlob.substring(0, lastSlash + 1) : "/";
174
- return withHttps(parsedPattern.baseUrl + pathBeforeGlob);
174
+ return baseUrl + (lastSlash >= 0 ? beforeGlob.substring(0, lastSlash + 1) : "/");
175
175
  }
176
176
  function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
177
177
  if (!excludePatterns || excludePatterns.length === 0) return false;
@@ -455,6 +455,34 @@ const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
455
455
  const URL_TRAILING_SLASH_RE = /\/$/;
456
456
  const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
457
457
  const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
458
+ const WINDOWS_DEVICE_NAMES = /* @__PURE__ */ new Set([
459
+ "con",
460
+ "prn",
461
+ "aux",
462
+ "nul",
463
+ "com1",
464
+ "com2",
465
+ "com3",
466
+ "com4",
467
+ "com5",
468
+ "com6",
469
+ "com7",
470
+ "com8",
471
+ "com9",
472
+ "lpt1",
473
+ "lpt2",
474
+ "lpt3",
475
+ "lpt4",
476
+ "lpt5",
477
+ "lpt6",
478
+ "lpt7",
479
+ "lpt8",
480
+ "lpt9"
481
+ ]);
482
+ function sanitizeOutputSegment(segment) {
483
+ const safe = segment.replace(URL_PATH_UNSAFE_CHARS_RE, "-");
484
+ return safe.length >= 3 && safe.length <= 4 && WINDOWS_DEVICE_NAMES.has(safe.toLowerCase()) ? `~${safe}` : safe;
485
+ }
458
486
  const TRACKING_PARAMS = /* @__PURE__ */ new Set([
459
487
  "utm_source",
460
488
  "utm_medium",
@@ -859,9 +887,9 @@ async function runCrawl(options, onProgress) {
859
887
  let filePath;
860
888
  if (shouldProcessMarkdown && writeMarkdown) {
861
889
  const urlPath = parsedUrl.pathname === "/" ? "/index" : parsedUrl.pathname;
862
- const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
890
+ const hostPrefix = allowSubdomains ? [sanitizeOutputSegment(parsedUrl.hostname)] : [];
863
891
  const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
864
- const safeSegments = [...hostPrefix, ...pathSegments.map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"))];
892
+ const safeSegments = [...hostPrefix, ...pathSegments.map(sanitizeOutputSegment)];
865
893
  const filename = safeSegments.length > 0 ? safeSegments.join("/") : "index";
866
894
  const safeFilename = normalize(`${filename}.md`);
867
895
  filePath = join(outputDir, safeFilename);
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/crawl",
3
3
  "type": "module",
4
- "version": "2.0.0",
4
+ "version": "2.0.1",
5
5
  "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -65,15 +65,15 @@
65
65
  },
66
66
  "dependencies": {
67
67
  "@clack/prompts": "^1.8.1",
68
- "@mdream/js": "2.0.0",
68
+ "@mdream/js": "2.0.1",
69
69
  "c12": "4.0.0-rc.2",
70
70
  "hookable": "^6.1.2",
71
- "mdream": "2.0.0",
71
+ "mdream": "2.0.1",
72
72
  "nypm": "^0.6.10",
73
73
  "ofetch": "^1.5.1",
74
74
  "pathe": "^2.0.3",
75
75
  "picomatch": "^4.0.7",
76
- "tldts": "^7.4.16",
76
+ "tldts": "^7.4.18",
77
77
  "ufo": "^1.6.4"
78
78
  },
79
79
  "devDependencies": {