@mdream/crawl 0.8.4 → 0.8.5

package/dist/_chunks/{crawl-DYXGzu7W.mjs → crawl-BtuYX2_u.mjs}

@@ -5,24 +5,10 @@ import { HttpCrawler, PlaywrightCrawler, log, purgeDefaultStorages } from "crawl
 import { generateLlmsTxtArtifacts, htmlToMarkdown } from "mdream";
 import { withMinimalPreset } from "mdream/preset/minimal";
 import { dirname, join, normalize, resolve } from "pathe";
+import { withHttps } from "ufo";
 import picomatch from "picomatch";
 import { extractionPlugin } from "mdream/plugins";
 
-//#region ../../node_modules/.pnpm/ufo@1.6.1/node_modules/ufo/dist/index.mjs
-const r = String.fromCharCode;
-const PROTOCOL_REGEX = /^[\s\w\0+.-]{2,}:([/\\]{2})?/;
-function withHttps(input) {
-	return withProtocol(input, "https://");
-}
-function withProtocol(input, protocol) {
-	let match = input.match(PROTOCOL_REGEX);
-	if (!match) match = input.match(/^\/{2,}/);
-	if (!match) return protocol + input;
-	return protocol + input.slice(match[0].length);
-}
-const protocolRelative = Symbol.for("ufo:protocolRelative");
-
-//#endregion
 //#region src/glob-utils.ts
 /**
 * Parse a URL that may contain glob patterns
@@ -467,14 +453,14 @@ async function crawlAndGenerate(options, onProgress) {
 	await crawler.run(initialRequests);
 	progress.crawling.status = "completed";
 	onProgress?.(progress);
-	if (results.some((r$1) => r$1.success)) {
+	if (results.some((r) => r.success)) {
 		progress.generation.status = "generating";
 		onProgress?.(progress);
-		const successfulResults = results.filter((r$1) => r$1.success);
+		const successfulResults = results.filter((r) => r.success);
 		const firstUrl = new URL(withHttps(urls[0]));
 		const origin$1 = firstUrl.origin;
-		const homePageResult = successfulResults.find((r$1) => {
-			const resultUrl = new URL(withHttps(r$1.url));
+		const homePageResult = successfulResults.find((r) => {
+			const resultUrl = new URL(withHttps(r.url));
 			return resultUrl.href === origin$1 || resultUrl.href === `${origin$1}/`;
 		});
 		const siteName = siteNameOverride || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
@@ -528,4 +514,4 @@ async function crawlAndGenerate(options, onProgress) {
 }
 
 //#endregion
-export { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps };
+export { crawlAndGenerate, parseUrlPattern, validateGlobPattern };

package/dist/cli.mjs

@@ -1,9 +1,10 @@
-import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-DYXGzu7W.mjs";
+import { crawlAndGenerate, parseUrlPattern, validateGlobPattern } from "./_chunks/crawl-BtuYX2_u.mjs";
 import { readFileSync } from "node:fs";
 import * as p$1 from "@clack/prompts";
 import * as p from "@clack/prompts";
 import { PlaywrightCrawler } from "crawlee";
 import { dirname, join, resolve } from "pathe";
+import { withHttps } from "ufo";
 import { fileURLToPath } from "node:url";
 import { addDependency } from "nypm";
 

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { crawlAndGenerate } from "./_chunks/crawl-DYXGzu7W.mjs";
+import { crawlAndGenerate } from "./_chunks/crawl-BtuYX2_u.mjs";
 import { writeFile } from "node:fs/promises";
 import { basename, sep } from "pathe";
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "0.8.4",
+  "version": "0.8.5",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -50,7 +50,8 @@
     "nypm": "^0.6.0",
     "pathe": "^2.0.3",
     "picomatch": "^4.0.3",
-    "mdream": "0.8.4"
+    "ufo": "^1.6.1",
+    "mdream": "0.8.5"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.1"