@mdream/crawl 1.7.0 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -602,6 +602,7 @@ async function crawlAndGenerate(options, onProgress) {
602
602
  const normalizedHomePageUrl = (startingUrls.length > 0 ? new URL(startingUrls[0]).origin : "").replace(URL_TRAILING_SLASH_RE, "");
603
603
  const createdDirs = /* @__PURE__ */ new Set();
604
604
  const sharedOrigin = origin || "";
605
+ const sharedOriginBase = sharedOrigin.replace(URL_TRAILING_SLASH_RE, "");
605
606
  const pendingUrls = [];
606
607
  const queryVariantCounts = /* @__PURE__ */ new Map();
607
608
  const processPage = async (url, content, initialTitle, depth, isMarkdown = false) => {
@@ -624,9 +625,10 @@ async function crawlAndGenerate(options, onProgress) {
624
625
  };
625
626
  await hooks.callHook("crawl:html", htmlCtx);
626
627
  content = htmlCtx.html;
628
+ const pageBaseUrl = sharedOriginBase ? `${sharedOriginBase}${parsedUrl.pathname}` : url;
627
629
  const { extraction, getMetadata } = extractMetadataInline(parsedUrl, allowedRegistrableDomains);
628
630
  md = htmlToMarkdown(content, {
629
- origin: pageOrigin,
631
+ origin: pageBaseUrl,
630
632
  extraction
631
633
  });
632
634
  metadata = getMetadata();
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/crawl",
3
3
  "type": "module",
4
- "version": "1.7.0",
4
+ "version": "1.7.1",
5
5
  "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -63,8 +63,8 @@
63
63
  "picomatch": "^4.0.5",
64
64
  "tldts": "^7.4.10",
65
65
  "ufo": "^1.6.4",
66
- "@mdream/js": "1.7.0",
67
- "mdream": "1.7.0"
66
+ "@mdream/js": "1.7.1",
67
+ "mdream": "1.7.1"
68
68
  },
69
69
  "devDependencies": {
70
70
  "@types/picomatch": "^4.0.3"