@mdream/crawl 1.6.3 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/crawl.mjs +3 -1
- package/package.json +3 -3
package/dist/_chunks/crawl.mjs
CHANGED
|
@@ -602,6 +602,7 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
602
602
|
const normalizedHomePageUrl = (startingUrls.length > 0 ? new URL(startingUrls[0]).origin : "").replace(URL_TRAILING_SLASH_RE, "");
|
|
603
603
|
const createdDirs = /* @__PURE__ */ new Set();
|
|
604
604
|
const sharedOrigin = origin || "";
|
|
605
|
+
const sharedOriginBase = sharedOrigin.replace(URL_TRAILING_SLASH_RE, "");
|
|
605
606
|
const pendingUrls = [];
|
|
606
607
|
const queryVariantCounts = /* @__PURE__ */ new Map();
|
|
607
608
|
const processPage = async (url, content, initialTitle, depth, isMarkdown = false) => {
|
|
@@ -624,9 +625,10 @@ async function crawlAndGenerate(options, onProgress) {
|
|
|
624
625
|
};
|
|
625
626
|
await hooks.callHook("crawl:html", htmlCtx);
|
|
626
627
|
content = htmlCtx.html;
|
|
628
|
+
const pageBaseUrl = sharedOriginBase ? `${sharedOriginBase}${parsedUrl.pathname}` : url;
|
|
627
629
|
const { extraction, getMetadata } = extractMetadataInline(parsedUrl, allowedRegistrableDomains);
|
|
628
630
|
md = htmlToMarkdown(content, {
|
|
629
|
-
origin:
|
|
631
|
+
origin: pageBaseUrl,
|
|
630
632
|
extraction
|
|
631
633
|
});
|
|
632
634
|
metadata = getMetadata();
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/crawl",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.7.1",
|
|
5
5
|
"description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -63,8 +63,8 @@
|
|
|
63
63
|
"picomatch": "^4.0.5",
|
|
64
64
|
"tldts": "^7.4.10",
|
|
65
65
|
"ufo": "^1.6.4",
|
|
66
|
-
"@mdream/js": "1.
|
|
67
|
-
"mdream": "1.
|
|
66
|
+
"@mdream/js": "1.7.1",
|
|
67
|
+
"mdream": "1.7.1"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
70
|
"@types/picomatch": "^4.0.3"
|