@mastra/rag 2.6.4 → 2.6.5-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/SKILL.md +1 -1
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/document/transformers/html.d.ts.map +1 -1
- package/dist/index.cjs +57 -16
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +57 -16
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
package/dist/index.js
CHANGED
|
@@ -4547,6 +4547,61 @@ var RecursiveCharacterTransformer = class RecursiveCharacterTransformer extends
|
|
|
4547
4547
|
};
|
|
4548
4548
|
//#endregion
|
|
4549
4549
|
//#region src/document/transformers/html.ts
|
|
4550
|
+
const BLOCK_TAGS = /* @__PURE__ */ new Set([
|
|
4551
|
+
"address",
|
|
4552
|
+
"article",
|
|
4553
|
+
"aside",
|
|
4554
|
+
"blockquote",
|
|
4555
|
+
"br",
|
|
4556
|
+
"caption",
|
|
4557
|
+
"center",
|
|
4558
|
+
"dd",
|
|
4559
|
+
"details",
|
|
4560
|
+
"dialog",
|
|
4561
|
+
"div",
|
|
4562
|
+
"dl",
|
|
4563
|
+
"dt",
|
|
4564
|
+
"fieldset",
|
|
4565
|
+
"figcaption",
|
|
4566
|
+
"figure",
|
|
4567
|
+
"footer",
|
|
4568
|
+
"form",
|
|
4569
|
+
"h1",
|
|
4570
|
+
"h2",
|
|
4571
|
+
"h3",
|
|
4572
|
+
"h4",
|
|
4573
|
+
"h5",
|
|
4574
|
+
"h6",
|
|
4575
|
+
"header",
|
|
4576
|
+
"hgroup",
|
|
4577
|
+
"hr",
|
|
4578
|
+
"legend",
|
|
4579
|
+
"li",
|
|
4580
|
+
"main",
|
|
4581
|
+
"menu",
|
|
4582
|
+
"nav",
|
|
4583
|
+
"ol",
|
|
4584
|
+
"p",
|
|
4585
|
+
"pre",
|
|
4586
|
+
"search",
|
|
4587
|
+
"section",
|
|
4588
|
+
"summary",
|
|
4589
|
+
"table",
|
|
4590
|
+
"tbody",
|
|
4591
|
+
"td",
|
|
4592
|
+
"tfoot",
|
|
4593
|
+
"th",
|
|
4594
|
+
"thead",
|
|
4595
|
+
"tr",
|
|
4596
|
+
"ul"
|
|
4597
|
+
]);
|
|
4598
|
+
function extractText(node) {
|
|
4599
|
+
if (!node) return "";
|
|
4600
|
+
if (!node.tagName) return node.text || "";
|
|
4601
|
+
let content = "";
|
|
4602
|
+
for (const child of node.childNodes ?? []) content += extractText(child);
|
|
4603
|
+
return BLOCK_TAGS.has(node.tagName.toLowerCase()) ? " " + content + " " : content;
|
|
4604
|
+
}
|
|
4550
4605
|
var HTMLHeaderTransformer = class {
|
|
4551
4606
|
headersToSplitOn;
|
|
4552
4607
|
returnEachElement;
|
|
@@ -4605,14 +4660,7 @@ var HTMLHeaderTransformer = class {
|
|
|
4605
4660
|
return "/" + parts.join("/");
|
|
4606
4661
|
}
|
|
4607
4662
|
getTextContent(element) {
|
|
4608
|
-
|
|
4609
|
-
if (!element.tagName) return element.text || "";
|
|
4610
|
-
let content = element.text || "";
|
|
4611
|
-
if (element.childNodes) for (const child of element.childNodes) {
|
|
4612
|
-
const childText = this.getTextContent(child);
|
|
4613
|
-
if (childText) content += " " + childText;
|
|
4614
|
-
}
|
|
4615
|
-
return content.trim();
|
|
4663
|
+
return extractText(element).replace(/\s+/g, " ").trim();
|
|
4616
4664
|
}
|
|
4617
4665
|
aggregateElementsToChunks(elements) {
|
|
4618
4666
|
const aggregatedChunks = [];
|
|
@@ -4690,14 +4738,7 @@ var HTMLSectionTransformer = class {
|
|
|
4690
4738
|
return "/" + parts.join("/");
|
|
4691
4739
|
}
|
|
4692
4740
|
getTextContent(element) {
|
|
4693
|
-
|
|
4694
|
-
if (!element.tagName) return element.text || "";
|
|
4695
|
-
let content = element.text || "";
|
|
4696
|
-
if (element.childNodes) for (const child of element.childNodes) {
|
|
4697
|
-
const childText = this.getTextContent(child);
|
|
4698
|
-
if (childText) content += " " + childText;
|
|
4699
|
-
}
|
|
4700
|
-
return content.trim();
|
|
4741
|
+
return extractText(element).replace(/\s+/g, " ").trim();
|
|
4701
4742
|
}
|
|
4702
4743
|
splitHtmlByHeaders(htmlDoc) {
|
|
4703
4744
|
const sections = [];
|