@mastra/rag 2.0.0-beta.1 → 2.0.0-beta.3

package/CHANGELOG.md

@@ -1,5 +1,99 @@
 # @mastra/rag
 
+## 2.0.0-beta.3
+
+### Patch Changes
+
+- Add maxSize support for HTML chunking strategies ([#10654](https://github.com/mastra-ai/mastra/pull/10654))
+
+  Added support for the `maxSize` option in HTML chunking strategies (`headers` and `sections`), allowing users to control the maximum chunk size when chunking HTML documents. Previously, HTML chunks could be excessively large when sections contained substantial content.
+
+  **Changes:**
+  - Added `maxSize` support to `headers` strategy - applies `RecursiveCharacterTransformer` after header-based splitting
+  - Added `maxSize` support to `sections` strategy - applies `RecursiveCharacterTransformer` after section-based splitting
+  - Fixed `splitHtmlByHeaders` content extraction bug - changed from broken `nextElementSibling` to working `parentNode.childNodes` approach
+  - Added comprehensive test coverage including integration test with real arXiv paper
+
+  **Usage:**
+
+  ```typescript
+  import { MDocument } from '@mastra/rag';
+
+  const doc = MDocument.fromHTML(htmlContent);
+
+  const chunks = await doc.chunk({
+    strategy: 'html',
+    headers: [
+      ['h1', 'Header 1'],
+      ['h2', 'Header 2'],
+      ['h3', 'Header 3'],
+    ],
+    maxSize: 512, // Control chunk size
+    overlap: 50, // Optional overlap for context
+  });
+  ```
+
+  **Results from real arXiv paper test:**
+  - Without maxSize: 22 chunks, max 45,531 chars (too big!)
+  - With maxSize=512: 499 chunks, max 512 chars (properly sized)
+
+  Fixes #7942
+
+- Updated dependencies [[`ac0d2f4`](https://github.com/mastra-ai/mastra/commit/ac0d2f4ff8831f72c1c66c2be809706d17f65789), [`1a0d3fc`](https://github.com/mastra-ai/mastra/commit/1a0d3fc811482c9c376cdf79ee615c23bae9b2d6), [`85a628b`](https://github.com/mastra-ai/mastra/commit/85a628b1224a8f64cd82ea7f033774bf22df7a7e), [`c237233`](https://github.com/mastra-ai/mastra/commit/c23723399ccedf7f5744b3f40997b79246bfbe64), [`15f9e21`](https://github.com/mastra-ai/mastra/commit/15f9e216177201ea6e3f6d0bfb063fcc0953444f), [`ff94dea`](https://github.com/mastra-ai/mastra/commit/ff94dea935f4e34545c63bcb6c29804732698809), [`5b2ff46`](https://github.com/mastra-ai/mastra/commit/5b2ff4651df70c146523a7fca773f8eb0a2272f8), [`db41688`](https://github.com/mastra-ai/mastra/commit/db4168806d007417e2e60b4f68656dca4e5f40c9), [`5ca599d`](https://github.com/mastra-ai/mastra/commit/5ca599d0bb59a1595f19f58473fcd67cc71cef58), [`bff1145`](https://github.com/mastra-ai/mastra/commit/bff114556b3cbadad9b2768488708f8ad0e91475), [`5c8ca24`](https://github.com/mastra-ai/mastra/commit/5c8ca247094e0cc2cdbd7137822fb47241f86e77), [`e191844`](https://github.com/mastra-ai/mastra/commit/e1918444ca3f80e82feef1dad506cd4ec6e2875f), [`22553f1`](https://github.com/mastra-ai/mastra/commit/22553f11c63ee5e966a9c034a349822249584691), [`7237163`](https://github.com/mastra-ai/mastra/commit/72371635dbf96a87df4b073cc48fc655afbdce3d), [`2500740`](https://github.com/mastra-ai/mastra/commit/2500740ea23da067d6e50ec71c625ab3ce275e64), [`873ecbb`](https://github.com/mastra-ai/mastra/commit/873ecbb517586aa17d2f1e99283755b3ebb2863f), [`4f9bbe5`](https://github.com/mastra-ai/mastra/commit/4f9bbe5968f42c86f4930b8193de3c3c17e5bd36), [`02e51fe`](https://github.com/mastra-ai/mastra/commit/02e51feddb3d4155cfbcc42624fd0d0970d032c0), [`8f3fa3a`](https://github.com/mastra-ai/mastra/commit/8f3fa3a652bb77da092f913ec51ae46e3a7e27dc), [`cd29ad2`](https://github.com/mastra-ai/mastra/commit/cd29ad23a255534e8191f249593849ed29160886), [`bdf4d8c`](https://github.com/mastra-ai/mastra/commit/bdf4d8cdc656d8a2c21d81834bfa3bfa70f56c16), [`854e3da`](https://github.com/mastra-ai/mastra/commit/854e3dad5daac17a91a20986399d3a51f54bf68b), [`ce18d38`](https://github.com/mastra-ai/mastra/commit/ce18d38678c65870350d123955014a8432075fd9), [`cccf9c8`](https://github.com/mastra-ai/mastra/commit/cccf9c8b2d2dfc1a5e63919395b83d78c89682a0), [`61a5705`](https://github.com/mastra-ai/mastra/commit/61a570551278b6743e64243b3ce7d73de915ca8a), [`db70a48`](https://github.com/mastra-ai/mastra/commit/db70a48aeeeeb8e5f92007e8ede52c364ce15287), [`f0fdc14`](https://github.com/mastra-ai/mastra/commit/f0fdc14ee233d619266b3d2bbdeea7d25cfc6d13), [`db18bc9`](https://github.com/mastra-ai/mastra/commit/db18bc9c3825e2c1a0ad9a183cc9935f6691bfa1), [`9b37b56`](https://github.com/mastra-ai/mastra/commit/9b37b565e1f2a76c24f728945cc740c2b09be9da), [`41a23c3`](https://github.com/mastra-ai/mastra/commit/41a23c32f9877d71810f37e24930515df2ff7a0f), [`5d171ad`](https://github.com/mastra-ai/mastra/commit/5d171ad9ef340387276b77c2bb3e83e83332d729), [`f03ae60`](https://github.com/mastra-ai/mastra/commit/f03ae60500fe350c9d828621006cdafe1975fdd8), [`d1e74a0`](https://github.com/mastra-ai/mastra/commit/d1e74a0a293866dece31022047f5dbab65a304d0), [`39e7869`](https://github.com/mastra-ai/mastra/commit/39e7869bc7d0ee391077ce291474d8a84eedccff), [`5761926`](https://github.com/mastra-ai/mastra/commit/57619260c4a2cdd598763abbacd90de594c6bc76), [`c900fdd`](https://github.com/mastra-ai/mastra/commit/c900fdd504c41348efdffb205cfe80d48c38fa33), [`604a79f`](https://github.com/mastra-ai/mastra/commit/604a79fecf276e26a54a3fe01bb94e65315d2e0e), [`887f0b4`](https://github.com/mastra-ai/mastra/commit/887f0b4746cdbd7cb7d6b17ac9f82aeb58037ea5), [`2562143`](https://github.com/mastra-ai/mastra/commit/256214336b4faa78646c9c1776612393790d8784), [`ef11a61`](https://github.com/mastra-ai/mastra/commit/ef11a61920fa0ed08a5b7ceedd192875af119749)]:
+  - @mastra/core@1.0.0-beta.6
+
+## 2.0.0-beta.2
+
+### Patch Changes
+
+- Add table support to markdown transformer ([#10487](https://github.com/mastra-ai/mastra/pull/10487))
+
+  Added support for markdown tables in the `MarkdownHeaderTransformer` to prevent tables from being split in the middle during document chunking. Tables are now treated as semantic units similar to code blocks.
+
+  **Changes:**
+  - Updated `MarkdownHeaderTransformer` to detect and preserve markdown tables during chunking
+  - Tables are identified by lines containing pipe characters (`|`)
+  - Tables are kept together as a single block, preventing splits that would break table structure
+  - Added comprehensive test coverage for table handling in various scenarios
+  - Works with both simple and complex tables, including multi-row tables and tables with different formatting
+
+  **Usage:**
+
+  ```typescript
+  import { MDocument } from '@mastra/rag';
+
+  const doc = MDocument.fromMarkdown(`
+  # Data Report
+  
+  ## Results
+  
+  | Name | Score | Status |
+  |------|-------|--------|
+  | Alice | 95   | Pass   |
+  | Bob   | 87   | Pass   |
+  | Carol | 78   | Pass   |
+  
+  ## Summary
+  
+  The results show...
+  `);
+
+  const chunks = await doc.chunk({
+    strategy: 'markdown',
+    headers: [
+      ['#', 'title'],
+      ['##', 'section'],
+    ],
+  });
+
+  // Tables will now be preserved intact within chunks
+  ```
+
+- Fix invalid filter handling in vector queries and graph-rag searches. Invalid filter inputs now throw explicit errors instead of silently falling back to empty filters, preventing unintended unfiltered results. ([#10375](https://github.com/mastra-ai/mastra/pull/10375))
+
+- Updated dependencies [[`21a15de`](https://github.com/mastra-ai/mastra/commit/21a15de369fe82aac26bb642ed7be73505475e8b), [`feb7ee4`](https://github.com/mastra-ai/mastra/commit/feb7ee4d09a75edb46c6669a3beaceec78811747), [`b0e2ea5`](https://github.com/mastra-ai/mastra/commit/b0e2ea5b52c40fae438b9e2f7baee6f0f89c5442), [`c456e01`](https://github.com/mastra-ai/mastra/commit/c456e0149e3c176afcefdbd9bb1d2c5917723725), [`ab035c2`](https://github.com/mastra-ai/mastra/commit/ab035c2ef6d8cc7bb25f06f1a38508bd9e6f126b), [`1a46a56`](https://github.com/mastra-ai/mastra/commit/1a46a566f45a3fcbadc1cf36bf86d351f264bfa3), [`3cf540b`](https://github.com/mastra-ai/mastra/commit/3cf540b9fbfea8f4fc8d3a2319a4e6c0b0cbfd52), [`1c6ce51`](https://github.com/mastra-ai/mastra/commit/1c6ce51f875915ab57fd36873623013699a2a65d), [`898a972`](https://github.com/mastra-ai/mastra/commit/898a9727d286c2510d6b702dfd367e6aaf5c6b0f), [`a97003a`](https://github.com/mastra-ai/mastra/commit/a97003aa1cf2f4022a41912324a1e77263b326b8), [`ccc141e`](https://github.com/mastra-ai/mastra/commit/ccc141ed27da0abc3a3fc28e9e5128152e8e37f4), [`fe3b897`](https://github.com/mastra-ai/mastra/commit/fe3b897c2ccbcd2b10e81b099438c7337feddf89), [`00123ba`](https://github.com/mastra-ai/mastra/commit/00123ba96dc9e5cd0b110420ebdba56d8f237b25), [`29c4309`](https://github.com/mastra-ai/mastra/commit/29c4309f818b24304c041bcb4a8f19b5f13f6b62), [`16785ce`](https://github.com/mastra-ai/mastra/commit/16785ced928f6f22638f4488cf8a125d99211799), [`de8239b`](https://github.com/mastra-ai/mastra/commit/de8239bdcb1d8c0cfa06da21f1569912a66bbc8a), [`b5e6cd7`](https://github.com/mastra-ai/mastra/commit/b5e6cd77fc8c8e64e0494c1d06cee3d84e795d1e), [`3759cb0`](https://github.com/mastra-ai/mastra/commit/3759cb064935b5f74c65ac2f52a1145f7352899d), [`651e772`](https://github.com/mastra-ai/mastra/commit/651e772eb1475fb13e126d3fcc01751297a88214), [`b61b93f`](https://github.com/mastra-ai/mastra/commit/b61b93f9e058b11dd2eec169853175d31dbdd567), [`bae33d9`](https://github.com/mastra-ai/mastra/commit/bae33d91a63fbb64d1e80519e1fc1acaed1e9013), [`c0b731f`](https://github.com/mastra-ai/mastra/commit/c0b731fb27d712dc8582e846df5c0332a6a0c5ba), [`43ca8f2`](https://github.com/mastra-ai/mastra/commit/43ca8f2c7334851cc7b4d3d2f037d8784bfbdd5f), [`2ca67cc`](https://github.com/mastra-ai/mastra/commit/2ca67cc3bb1f6a617353fdcab197d9efebe60d6f), [`9e67002`](https://github.com/mastra-ai/mastra/commit/9e67002b52c9be19936c420a489dbee9c5fd6a78), [`35edc49`](https://github.com/mastra-ai/mastra/commit/35edc49ac0556db609189641d6341e76771b81fc)]:
+  - @mastra/core@1.0.0-beta.5
+
 ## 2.0.0-beta.1
 
 ### Patch Changes

package/dist/document/document.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../../src/document/document.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,IAAI,KAAK,EAAgC,MAAM,UAAU,CAAC;AAU3E,OAAO,KAAK,EACV,WAAW,EAEX,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,oBAAoB,EACpB,4BAA4B,EAC5B,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAErB,MAAM,SAAS,CAAC;AAGjB,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,IAAI,CAAS;gBAET,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAAE,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAOhG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC;IAmDjG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYhF,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAY9E,OAAO,CAAC,eAAe;IAevB,OAAO,CAAC,YAAY,CAAC,CAA4E;IAEjG,OAAO,KAAK,WAAW,GAetB;YAEa,OAAO;IASf,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9D,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAU9D,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBpD,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBpD,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAMtD,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAUtD,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa5D,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB5D,qBAAqB,CAAC,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,IAAI,CAAC;IAU5E,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAiBnD,OAAO,IAAI,KAAK,EAAE;IAIlB,OAAO,IAAI,MAAM,EAAE;IAInB,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;CAGrC"}
+{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../../src/document/document.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,IAAI,KAAK,EAAgC,MAAM,UAAU,CAAC;AAU3E,OAAO,KAAK,EACV,WAAW,EAEX,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,oBAAoB,EACpB,4BAA4B,EAC5B,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAErB,MAAM,SAAS,CAAC;AAGjB,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,IAAI,CAAS;gBAET,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAAE,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAOhG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC;IAmDjG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYhF,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAY9E,OAAO,CAAC,eAAe;IAevB,OAAO,CAAC,YAAY,CAAC,CAA4E;IAEjG,OAAO,KAAK,WAAW,GAetB;YAEa,OAAO;IASf,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9D,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAU9D,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IA8CpD,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBpD,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAMtD,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAUtD,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa5D,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB5D,qBAAqB,CAAC,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,IAAI,CAAC;IAU5E,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAiBnD,OAAO,IAAI,KAAK,EAAE;IAIlB,OAAO,IAAI,MAAM,EAAE;IAInB,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;CAGrC"}

package/dist/document/transformers/html.d.ts

@@ -23,6 +23,7 @@ export declare class HTMLSectionTransformer {
     });
     splitText(text: string): Document[];
     private getXPath;
+    private getTextContent;
     private splitHtmlByHeaders;
     splitDocuments(documents: Document[]): Promise<Document[]>;
     createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[];

package/dist/document/transformers/html.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAWjD,qBAAa,qBAAqB;IAChC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,iBAAiB,CAAU;gBAEvB,OAAO,EAAE,gBAAgB,GAAG;QAAE,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;KAAE;IAKvE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAwDjD,OAAO,CAAC,QAAQ;IA2BhB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,yBAAyB;IAyBjC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA8B/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,YAAY,CAAgC;gBAExC,OAAO,EAAE,gBAAgB,GAAG;QAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;KAAE;IAKxE,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE;IAenC,OAAO,CAAC,QAAQ;IAwBhB,OAAO,CAAC,kBAAkB;IA8CpB,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAahE,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA+B/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}
+{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAWjD,qBAAa,qBAAqB;IAChC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,iBAAiB,CAAU;gBAEvB,OAAO,EAAE,gBAAgB,GAAG;QAAE,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;KAAE;IAKvE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAwDjD,OAAO,CAAC,QAAQ;IA2BhB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,yBAAyB;IAyBjC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA8B/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,YAAY,CAAgC;gBAExC,OAAO,EAAE,gBAAgB,GAAG;QAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;KAAE;IAKxE,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE;IAenC,OAAO,CAAC,QAAQ;IAwBhB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,kBAAkB;IA6DpB,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAahE,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA+B/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}

package/dist/document/transformers/markdown.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAa5D,qBAAa,mBAAoB,SAAQ,6BAA6B;gBACxD,OAAO,GAAE,gBAAqB;CAI3C;AAED,qBAAa,yBAAyB;IACpC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,YAAY,CAAU;gBAElB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,cAAc,GAAE,OAAe,EAAE,YAAY,GAAE,OAAc;IAM/G,OAAO,CAAC,sBAAsB;IAuD9B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAqHjD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IAmB/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAa5D,qBAAa,mBAAoB,SAAQ,6BAA6B;gBACxD,OAAO,GAAE,gBAAqB;CAI3C;AAED,qBAAa,yBAAyB;IACpC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,YAAY,CAAU;gBAElB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,cAAc,GAAE,OAAe,EAAE,YAAY,GAAE,OAAc;IAM/G,OAAO,CAAC,sBAAsB;IAuD9B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAsIjD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IAmB/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}

package/dist/graph-rag/index.d.ts

@@ -6,11 +6,12 @@
  *  - Improve graph traversal and querying using types
  */
 type SupportedEdgeType = 'semantic';
+type GraphMetadata = Record<string, any>;
 export interface GraphNode {
     id: string;
     content: string;
     embedding?: number[];
-    metadata?: Record<string, any>;
+    metadata?: GraphMetadata;
 }
 export interface RankedNode extends GraphNode {
     score: number;
@@ -46,11 +47,21 @@ export declare class GraphRAG {
     createGraph(chunks: GraphChunk[], embeddings: GraphEmbedding[]): void;
     private selectWeightedNeighbor;
     private randomWalkWithRestart;
-    query({ query, topK, randomWalkSteps, restartProb, }: {
+    /**
+     * Query the graph with a dense embedding and optional metadata filter.
+     *
+     * @param query - The embedding vector to query.
+     * @param topK - Number of top results to return.
+     * @param randomWalkSteps - Steps for random walk reranking.
+     * @param restartProb - Restart probability for random walk.
+     * @param filter - Optional strict metadata filter. All key-value pairs must match exactly.
+     */
+    query({ query, topK, randomWalkSteps, restartProb, filter, }: {
         query: number[];
         topK?: number;
         randomWalkSteps?: number;
         restartProb?: number;
+        filter?: Partial<GraphMetadata>;
     }): RankedNode[];
 }
 export {};

package/dist/graph-rag/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/graph-rag/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,KAAK,iBAAiB,GAAG,UAAU,CAAC;AAGpC,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,UAAW,SAAQ,SAAS;IAC3C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,iBAAiB,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAyB;IACtC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAS;gBAEd,SAAS,GAAE,MAAa,EAAE,SAAS,GAAE,MAAY;IAQ7D,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAW9B,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAe9B,QAAQ,IAAI,SAAS,EAAE;IAKvB,QAAQ,IAAI,SAAS,EAAE;IAIvB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IAIzC,KAAK,IAAI,IAAI;IAKb,iBAAiB,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IASvD,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,gBAAgB;IAgCxB,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,cAAc,EAAE;IAuC9D,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,qBAAqB;IAoC7B,KAAK,CAAC,EACJ,KAAK,EACL,IAAS,EACT,eAAqB,EACrB,WAAkB,GACnB,EAAE;QACD,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,GAAG,UAAU,EAAE;CAoDjB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/graph-rag/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,KAAK,iBAAiB,GAAG,UAAU,CAAC;AACpC,KAAK,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAGzC,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,UAAW,SAAQ,SAAS;IAC3C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,iBAAiB,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAyB;IACtC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAS;gBAEd,SAAS,GAAE,MAAa,EAAE,SAAS,GAAE,MAAY;IAQ7D,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAW9B,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAe9B,QAAQ,IAAI,SAAS,EAAE;IAKvB,QAAQ,IAAI,SAAS,EAAE;IAIvB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IAIzC,KAAK,IAAI,IAAI;IAKb,iBAAiB,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IASvD,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,gBAAgB;IAgCxB,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,cAAc,EAAE;IAuC9D,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,qBAAqB;IA2C7B;;;;;;;;OAQG;IAEH,KAAK,CAAC,EACJ,KAAK,EACL,IAAS,EACT,eAAqB,EACrB,WAAkB,EAClB,MAAM,GACP,EAAE;QACD,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;KACjC,GAAG,UAAU,EAAE;CA+DjB"}

package/dist/index.cjs

@@ -272,7 +272,7 @@ Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'
 var defaultQuestionExtractPrompt = new PromptTemplate({
   templateVars: ["numQuestions", "context"],
   template: `(
-  "Given the contextual informations below, generate {numQuestions} questions this context can provides specific answers to which are unlikely to be found else where. Higher-level summaries of surrounding context may be provided as well. "
+  "Given the contextual information below, generate {numQuestions} questions this context can provide specific answers to which are unlikely to be found elsewhere. Higher-level summaries of surrounding context may be provided as well. "
   "Try using these summaries to generate better questions that this context can answer."
   "---------------------"
   "{context}"
@@ -4999,23 +4999,47 @@ var HTMLSectionTransformer = class {
     }
     return "/" + parts.join("/");
   }
+  getTextContent(element) {
+    if (!element) return "";
+    if (!element.tagName) {
+      return element.text || "";
+    }
+    let content = element.text || "";
+    if (element.childNodes) {
+      for (const child of element.childNodes) {
+        const childText = this.getTextContent(child);
+        if (childText) {
+          content += " " + childText;
+        }
+      }
+    }
+    return content.trim();
+  }
   splitHtmlByHeaders(htmlDoc) {
     const sections = [];
     const root = nodeHtmlBetterParser.parse(htmlDoc);
     const headers = Object.keys(this.headersToSplitOn);
     const headerElements = root.querySelectorAll(headers.join(","));
-    headerElements.forEach((headerElement, index) => {
+    headerElements.forEach((headerElement) => {
       const header = headerElement.text?.trim() || "";
       const tagName = headerElement.tagName;
       const xpath = this.getXPath(headerElement);
       let content = "";
-      let currentElement = headerElement.nextElementSibling;
-      const nextHeader = headerElements[index + 1];
-      while (currentElement && (!nextHeader || currentElement !== nextHeader)) {
-        if (currentElement.text) {
-          content += currentElement.text.trim() + " ";
+      const parentNode = headerElement.parentNode;
+      if (parentNode && parentNode.childNodes) {
+        let foundHeader = false;
+        for (const node of parentNode.childNodes) {
+          if (node === headerElement) {
+            foundHeader = true;
+            continue;
+          }
+          if (foundHeader && node.tagName && headers.includes(node.tagName.toLowerCase())) {
+            break;
+          }
+          if (foundHeader) {
+            content += this.getTextContent(node) + " ";
+          }
         }
-        currentElement = currentElement.nextElementSibling;
       }
       content = content.trim();
       sections.push({
@@ -5567,6 +5591,11 @@ var MarkdownHeaderTransformer = class {
         currentContent.push(line);
         continue;
       }
+      const isTableLine = strippedLine.includes("|") && strippedLine.length > 0;
+      if (isTableLine) {
+        currentContent.push(line);
+        continue;
+      }
       let headerMatched = false;
       for (const [sep, name14] of this.headersToSplitOn) {
         if (strippedLine.startsWith(sep) && (strippedLine.length === sep.length || strippedLine[sep.length] === " ")) {
@@ -6387,13 +6416,33 @@ var MDocument = class _MDocument {
   async chunkHTML(options) {
     if (options?.headers?.length) {
       const rt = new HTMLHeaderTransformer(options);
-      const textSplit = rt.transformDocuments(this.chunks);
+      let textSplit = rt.transformDocuments(this.chunks);
+      if (options?.maxSize) {
+        const textSplitter = new RecursiveCharacterTransformer({
+          maxSize: options.maxSize,
+          overlap: options.overlap,
+          keepSeparator: options.keepSeparator,
+          addStartIndex: options.addStartIndex,
+          stripWhitespace: options.stripWhitespace
+        });
+        textSplit = textSplitter.splitDocuments(textSplit);
+      }
       this.chunks = textSplit;
       return;
     }
     if (options?.sections?.length) {
       const rt = new HTMLSectionTransformer(options);
-      const textSplit = rt.transformDocuments(this.chunks);
+      let textSplit = rt.transformDocuments(this.chunks);
+      if (options?.maxSize) {
+        const textSplitter = new RecursiveCharacterTransformer({
+          maxSize: options.maxSize,
+          overlap: options.overlap,
+          keepSeparator: options.keepSeparator,
+          addStartIndex: options.addStartIndex,
+          stripWhitespace: options.stripWhitespace
+        });
+        textSplit = textSplitter.splitDocuments(textSplit);
+      }
       this.chunks = textSplit;
       return;
     }
@@ -6803,7 +6852,7 @@ var GraphRAG = class {
     return neighbors[neighbors.length - 1]?.id;
   }
   // Perform random walk with restart
-  randomWalkWithRestart(startNodeId, steps, restartProb) {
+  randomWalkWithRestart(startNodeId, steps, restartProb, allowedNodeIds) {
     const visits = /* @__PURE__ */ new Map();
     let currentNodeId = startNodeId;
     for (let step = 0; step < steps; step++) {
@@ -6812,7 +6861,10 @@ var GraphRAG = class {
         currentNodeId = startNodeId;
         continue;
       }
-      const neighbors = this.getNeighbors(currentNodeId);
+      let neighbors = this.getNeighbors(currentNodeId);
+      if (allowedNodeIds) {
+        neighbors = neighbors.filter((n) => allowedNodeIds.has(n.id));
+      }
       if (neighbors.length === 0) {
         currentNodeId = startNodeId;
         continue;
@@ -6826,12 +6878,22 @@ var GraphRAG = class {
     }
     return normalizedVisits;
   }
+  /**
+   * Query the graph with a dense embedding and optional metadata filter.
+   *
+   * @param query - The embedding vector to query.
+   * @param topK - Number of top results to return.
+   * @param randomWalkSteps - Steps for random walk reranking.
+   * @param restartProb - Restart probability for random walk.
+   * @param filter - Optional strict metadata filter. All key-value pairs must match exactly.
+   */
   // Retrieve relevant nodes using hybrid approach
   query({
     query,
     topK = 10,
     randomWalkSteps = 100,
-    restartProb = 0.15
+    restartProb = 0.15,
+    filter
   }) {
     if (!query || query.length !== this.dimension) {
       throw new Error(`Query embedding must have dimension ${this.dimension}`);
@@ -6845,15 +6907,20 @@ var GraphRAG = class {
     if (restartProb <= 0 || restartProb >= 1) {
       throw new Error("Restart probability must be between 0 and 1");
     }
-    const similarities = Array.from(this.nodes.values()).map((node) => ({
+    const filterEntries = Object.entries(filter ?? {});
+    const matchesFilter = (node) => filterEntries.length === 0 ? true : filterEntries.every(([key, value]) => node.metadata?.[key] === value);
+    const nodesToSearch = Array.from(this.nodes.values()).filter(matchesFilter);
+    const similarities = nodesToSearch.map((node) => ({
       node,
       similarity: this.cosineSimilarity(query, node.embedding)
     }));
     similarities.sort((a, b) => b.similarity - a.similarity);
     const topNodes = similarities.slice(0, topK);
+    const useFilter = filterEntries.length > 0;
+    const allowedNodeIds = useFilter ? new Set(nodesToSearch.map((n) => n.id)) : void 0;
     const rerankedNodes = /* @__PURE__ */ new Map();
     for (const { node, similarity } of topNodes) {
-      const walkScores = this.randomWalkWithRestart(node.id, randomWalkSteps, restartProb);
+      const walkScores = this.randomWalkWithRestart(node.id, randomWalkSteps, restartProb, allowedNodeIds);
       for (const [nodeId, walkScore] of walkScores) {
         const node2 = this.nodes.get(nodeId);
         const existingScore = rerankedNodes.get(nodeId)?.score || 0;
@@ -7136,9 +7203,9 @@ var createGraphRAGTool = (options) => {
               return typeof filter === "string" ? JSON.parse(filter) : filter;
             } catch (error) {
               if (logger) {
-                logger.warn("Failed to parse filter as JSON, using empty filter", { filter, error });
+                logger.error("Invalid filter", { filter, error });
               }
-              return {};
+              throw new Error(`Invalid filter format: ${error instanceof Error ? error.message : String(error)}`);
             }
           })();
         }
@@ -7263,9 +7330,9 @@ var createVectorQueryTool = (options) => {
               return typeof filter === "string" ? JSON.parse(filter) : filter;
             } catch (error) {
               if (logger) {
-                logger.warn("Failed to parse filter as JSON, using empty filter", { filter, error });
+                logger.error("Invalid filter", { filter, error });
               }
-              return {};
+              throw new Error(`Invalid filter format: ${error instanceof Error ? error.message : String(error)}`);
             }
           })();
         }