@mastra/rag 1.0.7 → 1.0.8-alpha.0

package/.turbo/turbo-build.log

@@ -1,4 +1,4 @@
 
-> @mastra/rag@1.0.7-alpha.1 build /home/runner/work/mastra/mastra/packages/rag
+> @mastra/rag@1.0.8-alpha.0 build /home/runner/work/mastra/mastra/packages/rag
 > tsup --silent --config tsup.config.ts
 

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @mastra/rag
 
+## 1.0.8-alpha.0
+
+### Patch Changes
+
+- 1be6004: Added semantic markdown chunking strategy.
+- Updated dependencies [8388649]
+- Updated dependencies [dd94a26]
+- Updated dependencies [3ba6772]
+- Updated dependencies [2fff911]
+  - @mastra/core@0.13.2-alpha.0
+
 ## 1.0.7
 
 ### Patch Changes

package/dist/document/document.d.ts

@@ -1,5 +1,5 @@
 import { Document as Chunk } from './schema/index.js';
-import type { ChunkParams, ExtractParams, HTMLChunkOptions, RecursiveChunkOptions, CharacterChunkOptions, TokenChunkOptions, MarkdownChunkOptions, JsonChunkOptions, LatexChunkOptions, SentenceChunkOptions } from './types.js';
+import type { ChunkParams, ExtractParams, HTMLChunkOptions, RecursiveChunkOptions, CharacterChunkOptions, TokenChunkOptions, MarkdownChunkOptions, SemanticMarkdownChunkOptions, JsonChunkOptions, LatexChunkOptions, SentenceChunkOptions } from './types.js';
 export declare class MDocument {
     private chunks;
     private type;
@@ -16,6 +16,8 @@ export declare class MDocument {
     static fromMarkdown(markdown: string, metadata?: Record<string, any>): MDocument;
     static fromJSON(jsonString: string, metadata?: Record<string, any>): MDocument;
     private defaultStrategy;
+    private _strategyMap?;
+    private get strategyMap();
     private chunkBy;
     chunkRecursive(options?: RecursiveChunkOptions): Promise<void>;
     chunkCharacter(options?: CharacterChunkOptions): Promise<void>;
@@ -25,6 +27,7 @@ export declare class MDocument {
     chunkToken(options?: TokenChunkOptions): Promise<void>;
     chunkMarkdown(options?: MarkdownChunkOptions): Promise<void>;
     chunkSentence(options?: SentenceChunkOptions): Promise<void>;
+    chunkSemanticMarkdown(options?: SemanticMarkdownChunkOptions): Promise<void>;
     chunk(params?: ChunkParams): Promise<Chunk[]>;
     getDocs(): Chunk[];
     getText(): string[];

package/dist/document/document.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../../src/document/document.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,IAAI,KAAK,EAAgC,MAAM,UAAU,CAAC;AAS3E,OAAO,KAAK,EACV,WAAW,EAEX,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAErB,MAAM,SAAS,CAAC;AAGjB,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,IAAI,CAAS;gBAET,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAAE,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAOhG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC;IAmDjG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYhF,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAY9E,OAAO,CAAC,eAAe;YAeT,OAAO;IAoBf,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9D,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAU9D,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBpD,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBpD,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAMtD,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAUtD,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa5D,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB5D,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAiBnD,OAAO,IAAI,KAAK,EAAE;IAIlB,OAAO,IAAI,MAAM,EAAE;IAInB,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;CAGrC"}
+{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../../src/document/document.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,IAAI,KAAK,EAAgC,MAAM,UAAU,CAAC;AAU3E,OAAO,KAAK,EACV,WAAW,EAEX,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,oBAAoB,EACpB,4BAA4B,EAC5B,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAErB,MAAM,SAAS,CAAC;AAGjB,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,IAAI,CAAS;gBAET,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAAE,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAOhG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC;IAmDjG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYhF,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAY9E,OAAO,CAAC,eAAe;IAevB,OAAO,CAAC,YAAY,CAAC,CAA4E;IAEjG,OAAO,KAAK,WAAW,GAetB;YAEa,OAAO;IASf,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9D,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAU9D,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBpD,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBpD,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAMtD,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAUtD,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa5D,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB5D,qBAAqB,CAAC,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,IAAI,CAAC;IAU5E,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAiBnD,OAAO,IAAI,KAAK,EAAE;IAIlB,OAAO,IAAI,MAAM,EAAE;IAInB,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;CAGrC"}

package/dist/document/transformers/semantic-markdown.d.ts

@@ -0,0 +1,25 @@
+import type { TiktokenModel, TiktokenEncoding } from 'js-tiktoken';
+import { Document } from '../schema/index.js';
+import type { SemanticMarkdownChunkOptions } from '../types.js';
+import { TextTransformer } from './text.js';
+export declare class SemanticMarkdownTransformer extends TextTransformer {
+    private tokenizer;
+    private joinThreshold;
+    private allowedSpecial;
+    private disallowedSpecial;
+    constructor({ joinThreshold, encodingName, modelName, allowedSpecial, disallowedSpecial, ...baseOptions }?: SemanticMarkdownChunkOptions);
+    private countTokens;
+    private splitMarkdownByHeaders;
+    private mergeSemanticSections;
+    splitText({ text }: {
+        text: string;
+    }): string[];
+    createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[];
+    transformDocuments(documents: Document[]): Document[];
+    static fromTikToken({ encodingName, modelName, options, }: {
+        encodingName?: TiktokenEncoding;
+        modelName?: TiktokenModel;
+        options?: SemanticMarkdownChunkOptions;
+    }): SemanticMarkdownTransformer;
+}
+//# sourceMappingURL=semantic-markdown.d.ts.map

package/dist/document/transformers/semantic-markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-markdown.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/semantic-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAY,MAAM,aAAa,CAAC;AAE7E,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,UAAU,CAAC;AAE7D,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AASzC,qBAAa,2BAA4B,SAAQ,eAAe;IAC9D,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,cAAc,CAAsB;IAC5C,OAAO,CAAC,iBAAiB,CAAsB;gBAEnC,EACV,aAAmB,EACnB,YAA4B,EAC5B,SAAS,EACT,cAA0B,EAC1B,iBAAyB,EACzB,GAAG,WAAW,EACf,GAAE,4BAAiC;IAcpC,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,sBAAsB;IA2D9B,OAAO,CAAC,qBAAqB;IA+B7B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAgB/C,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IAuB/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;IAYrD,MAAM,CAAC,YAAY,CAAC,EAClB,YAA4B,EAC5B,SAAS,EACT,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,OAAO,CAAC,EAAE,4BAA4B,CAAC;KACxC,GAAG,2BAA2B;CA4BhC"}

package/dist/document/types.d.ts

@@ -66,6 +66,13 @@ export type MarkdownChunkOptions = BaseChunkOptions & {
     returnEachLine?: boolean;
     stripHeaders?: boolean;
 };
+export type SemanticMarkdownChunkOptions = BaseChunkOptions & {
+    joinThreshold?: number;
+    encodingName?: TiktokenEncoding;
+    modelName?: TiktokenModel;
+    allowedSpecial?: Set<string> | 'all';
+    disallowedSpecial?: Set<string> | 'all';
+};
 export type HTMLChunkOptions = BaseChunkOptions & ({
     headers: [string, string][];
     sections?: never;
@@ -99,8 +106,9 @@ export type StrategyOptions = {
     json: JsonChunkOptions;
     latex: LatexChunkOptions;
     sentence: SentenceChunkOptions;
+    'semantic-markdown': SemanticMarkdownChunkOptions;
 };
-export type ChunkStrategy = 'recursive' | 'character' | 'token' | 'markdown' | 'html' | 'json' | 'latex' | 'sentence';
+export type ChunkStrategy = 'recursive' | 'character' | 'token' | 'markdown' | 'html' | 'json' | 'latex' | 'sentence' | 'semantic-markdown';
 export type ChunkParams = ({
     strategy?: 'character';
 } & CharacterChunkOptions & {
@@ -133,5 +141,9 @@ export type ChunkParams = ({
     strategy: 'sentence';
 } & SentenceChunkOptions & {
     extract?: ExtractParams;
+}) | ({
+    strategy: 'semantic-markdown';
+} & SemanticMarkdownChunkOptions & {
+    extract?: ExtractParams;
 });
 //# sourceMappingURL=types.d.ts.map

package/dist/document/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/document/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EACV,mBAAmB,EACnB,kBAAkB,EAClB,yBAAyB,EACzB,kBAAkB,EACnB,MAAM,cAAc,CAAC;AAEtB,oBAAY,QAAQ;IAClB,GAAG,QAAQ;IACX,EAAE,OAAO;IACT,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,EAAE,OAAO;IACT,EAAE,OAAO;IACT,GAAG,QAAQ;IACX,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;IACf,KAAK,UAAU;IACf,QAAQ,aAAa;IACrB,KAAK,UAAU;IACf,IAAI,SAAS;IACb,GAAG,QAAQ;IACX,MAAM,WAAW;IACjB,KAAK,UAAU;IACf,CAAC,MAAM;IACP,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,OAAO,YAAY;IACnB,MAAM,WAAW;IACjB,UAAU,eAAe;CAC1B;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,KAAK,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC;IACtC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC;IACvC,SAAS,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC;IAChD,QAAQ,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG,gBAAgB,GAAG;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG,gBAAgB,GAAG;IACrD,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,GAAG;IACjD,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;IACrC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,GAAG;IACpD,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,gBAAgB,GAC7C,CACI;IAAE,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAA;CAAE,GAC3E;IAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,KAAK,CAAA;CAAE,CACpD,GAAG;IAAE,cAAc,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAEnC,MAAM,MAAM,gBAAgB,GAAG,gBAAgB,GAAG;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,GAAG,EAAE,CAAC;AAEtD,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,GAAG;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,qBAAqB,CAAC;IACjC,SAAS,EAAE,qBAAqB,CAAC;IACjC,KAAK,EAAE,iBAAiB,CAAC;IACzB,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,IAAI,EAAE,gBAAgB,CAAC;IACvB,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,iBAAiB,CAAC;IACzB,QAAQ,EAAE,oBAAoB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,WAAW,GAAG,WAAW,GAAG,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,UAAU,CAAC;AAEtH,MAAM,MAAM,WAAW,GACnB,CAAC;IAAE,QAAQ,CAAC,EAAE,WAAW,CAAA;CAAE,GAAG,qBAAqB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GAClF,CAAC;IAAE,QAAQ,EAAE,WAAW,CAAA;CAAE,GAAG,qBAAqB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GACjF,CAAC;IAAE,QAAQ,EAAE,OAAO,CAAA;CAAE,GAAG,iBAAiB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GACzE,CAAC;IAAE,QAAQ,EAAE,UAAU,CAAA;CAAE,GAAG,oBAAoB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GAC/E,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,gBAAgB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GACvE,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,gBAAgB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GACvE,CAAC;IAAE,QAAQ,EAAE,OAAO,CAAA;CAAE,GAAG,iBAAiB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GACzE,CAAC;IAAE,QAAQ,EAAE,UAAU,CAAA;CAAE,GAAG,oBAAoB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/document/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EACV,mBAAmB,EACnB,kBAAkB,EAClB,yBAAyB,EACzB,kBAAkB,EACnB,MAAM,cAAc,CAAC;AAEtB,oBAAY,QAAQ;IAClB,GAAG,QAAQ;IACX,EAAE,OAAO;IACT,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,EAAE,OAAO;IACT,EAAE,OAAO;IACT,GAAG,QAAQ;IACX,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;IACf,KAAK,UAAU;IACf,QAAQ,aAAa;IACrB,KAAK,UAAU;IACf,IAAI,SAAS;IACb,GAAG,QAAQ;IACX,MAAM,WAAW;IACjB,KAAK,UAAU;IACf,CAAC,MAAM;IACP,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,OAAO,YAAY;IACnB,MAAM,WAAW;IACjB,UAAU,eAAe;CAC1B;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,KAAK,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC;IACtC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC;IACvC,SAAS,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC;IAChD,QAAQ,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG,gBAAgB,GAAG;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG,gBAAgB,GAAG;IACrD,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,GAAG;IACjD,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;IACrC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,GAAG;IACpD,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,4BAA4B,GAAG,gBAAgB,GAAG;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;IACrC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,gBAAgB,GAC7C,CACI;IAAE,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAA;CAAE,GAC3E;IAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,KAAK,CAAA;CAAE,CACpD,GAAG;IAAE,cAAc,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAEnC,MAAM,MAAM,gBAAgB,GAAG,gBAAgB,GAAG;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,GAAG,EAAE,CAAC;AAEtD,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,GAAG;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,qBAAqB,CAAC;IACjC,SAAS,EAAE,qBAAqB,CAAC;IACjC,KAAK,EAAE,iBAAiB,CAAC;IACzB,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,IAAI,EAAE,gBAAgB,CAAC;IACvB,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,iBAAiB,CAAC;IACzB,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,mBAAmB,EAAE,4BAA4B,CAAC;CACnD,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,WAAW,GACX,WAAW,GACX,OAAO,GACP,UAAU,GACV,MAAM,GACN,MAAM,GACN,OAAO,GACP,UAAU,GACV,mBAAmB,CAAC;AAExB,MAAM,MAAM,WAAW,GACnB,CAAC;IAAE,QAAQ,CAAC,EAAE,WAAW,CAAA;CAAE,GAAG,qBAAqB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GAClF,CAAC;IAAE,QAAQ,EAAE,WAAW,CAAA;CAAE,GAAG,qBAAqB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GACjF,CAAC;IAAE,QAAQ,EAAE,OAAO,CAAA;CAAE,GAAG,iBAAiB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GACzE,CAAC;IAAE,QAAQ,EAAE,UAAU,CAAA;CAAE,GAAG,oBAAoB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GAC/E,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,gBAAgB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GACvE,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,gBAAgB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GACvE,CAAC;IAAE,QAAQ,EAAE,OAAO,CAAA;CAAE,GAAG,iBAAiB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GACzE,CAAC;IAAE,QAAQ,EAAE,UAAU,CAAA;CAAE,GAAG,oBAAoB,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,GAC/E,CAAC;IAAE,QAAQ,EAAE,mBAAmB,CAAA;CAAE,GAAG,4BAA4B,GAAG;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAE,CAAC,CAAC"}

package/dist/document/validation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../src/document/validation.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AA2H7C,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,GAAG,IAAI,CAsB9E"}
+{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../src/document/validation.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAsI7C,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,GAAG,IAAI,CAsB9E"}

package/dist/index.cjs

@@ -5650,6 +5650,168 @@ var MarkdownHeaderTransformer = class {
     return this.createDocuments(texts, metadatas);
   }
 };
+var SemanticMarkdownTransformer = class _SemanticMarkdownTransformer extends TextTransformer {
+  tokenizer;
+  joinThreshold;
+  allowedSpecial;
+  disallowedSpecial;
+  constructor({
+    joinThreshold = 500,
+    encodingName = "cl100k_base",
+    modelName,
+    allowedSpecial = /* @__PURE__ */ new Set(),
+    disallowedSpecial = "all",
+    ...baseOptions
+  } = {}) {
+    super(baseOptions);
+    this.joinThreshold = joinThreshold;
+    this.allowedSpecial = allowedSpecial;
+    this.disallowedSpecial = disallowedSpecial;
+    try {
+      this.tokenizer = modelName ? jsTiktoken.encodingForModel(modelName) : jsTiktoken.getEncoding(encodingName);
+    } catch {
+      throw new Error("Could not load tiktoken encoding. Please install it with `npm install js-tiktoken`.");
+    }
+  }
+  countTokens(text) {
+    const allowed = this.allowedSpecial === "all" ? "all" : Array.from(this.allowedSpecial);
+    const disallowed = this.disallowedSpecial === "all" ? "all" : Array.from(this.disallowedSpecial);
+    const processedText = this.stripWhitespace ? text.trim() : text;
+    return this.tokenizer.encode(processedText, allowed, disallowed).length;
+  }
+  splitMarkdownByHeaders(markdown) {
+    const sections = [];
+    const lines = markdown.split("\n");
+    let currentContent = "";
+    let currentTitle = "";
+    let currentDepth = 0;
+    let inCodeBlock = false;
+    const headerRegex = /^(#+)\s+(.+)$/;
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      const headerMatch = line.match(headerRegex);
+      if (line.startsWith("```") || line.startsWith("~~~")) {
+        inCodeBlock = !inCodeBlock;
+      }
+      if (headerMatch && !inCodeBlock) {
+        if (currentContent.trim() !== "" || currentTitle && currentDepth > 0) {
+          sections.push({
+            title: currentTitle,
+            content: currentContent.trim(),
+            depth: currentDepth,
+            length: this.countTokens(currentContent.trim())
+          });
+        }
+        currentContent = "";
+        currentDepth = headerMatch[1].length;
+        currentTitle = headerMatch[2];
+      } else {
+        currentContent += line + "\n";
+      }
+    }
+    if (currentContent.trim() !== "") {
+      sections.push({
+        title: currentTitle,
+        content: currentContent.trim(),
+        depth: currentDepth,
+        length: this.countTokens(currentContent.trim())
+      });
+    }
+    if (sections.length > 1 && sections[0].title === "" && sections[0].content.trim() === "") {
+      sections.shift();
+    }
+    return sections;
+  }
+  mergeSemanticSections(sections) {
+    if (sections.length === 0) return sections;
+    const workingSections = [...sections];
+    const deepest = Math.max(...workingSections.map((s) => s.depth));
+    for (let depth = deepest; depth > 0; depth--) {
+      for (let j = 1; j < workingSections.length; j++) {
+        const current = workingSections[j];
+        if (current.depth === depth) {
+          const prev = workingSections[j - 1];
+          if (prev.length + current.length < this.joinThreshold && prev.depth <= current.depth) {
+            const title = `${"#".repeat(current.depth)} ${current.title}`;
+            const formattedTitle = `
+
+${title}`;
+            prev.content += `${formattedTitle}
+${current.content}`;
+            prev.length = this.countTokens(prev.content);
+            workingSections.splice(j, 1);
+            j--;
+          }
+        }
+      }
+    }
+    return workingSections;
+  }
+  splitText({ text }) {
+    if (!text.trim()) return [];
+    const initialSections = this.splitMarkdownByHeaders(text);
+    const mergedSections = this.mergeSemanticSections(initialSections);
+    return mergedSections.map((section) => {
+      if (section.title) {
+        const header = `${"#".repeat(section.depth)} ${section.title}`;
+        return `${header}
+${section.content}`;
+      }
+      return section.content;
+    });
+  }
+  createDocuments(texts, metadatas) {
+    const _metadatas = metadatas || Array(texts.length).fill({});
+    const documents = [];
+    texts.forEach((text, i) => {
+      this.splitText({ text }).forEach((chunk) => {
+        const metadata = {
+          ..._metadatas[i],
+          tokenCount: this.countTokens(chunk)
+        };
+        documents.push(
+          new Document({
+            text: chunk,
+            metadata
+          })
+        );
+      });
+    });
+    return documents;
+  }
+  transformDocuments(documents) {
+    const texts = [];
+    const metadatas = [];
+    for (const doc of documents) {
+      texts.push(doc.text);
+      metadatas.push(doc.metadata);
+    }
+    return this.createDocuments(texts, metadatas);
+  }
+  static fromTikToken({
+    encodingName = "cl100k_base",
+    modelName,
+    options = {}
+  }) {
+    let tokenizer;
+    try {
+      tokenizer = modelName ? jsTiktoken.encodingForModel(modelName) : jsTiktoken.getEncoding(encodingName);
+    } catch {
+      throw new Error("Could not load tiktoken encoding. Please install it with `npm install js-tiktoken`.");
+    }
+    const tikTokenCounter = (text) => {
+      const allowed = options.allowedSpecial === "all" ? "all" : options.allowedSpecial ? Array.from(options.allowedSpecial) : [];
+      const disallowed = options.disallowedSpecial === "all" ? "all" : options.disallowedSpecial ? Array.from(options.disallowedSpecial) : [];
+      return tokenizer.encode(text, allowed, disallowed).length;
+    };
+    return new _SemanticMarkdownTransformer({
+      ...options,
+      encodingName,
+      modelName,
+      lengthFunction: tikTokenCounter
+    });
+  }
+};
 
 // src/document/transformers/sentence.ts
 var SentenceTransformer = class extends TextTransformer {
@@ -6037,6 +6199,13 @@ var markdownChunkOptionsSchema = baseChunkOptionsSchema.extend({
   returnEachLine: zod.z.boolean().optional(),
   stripHeaders: zod.z.boolean().optional()
 }).strict();
+var semanticMarkdownChunkOptionsSchema = baseChunkOptionsSchema.extend({
+  joinThreshold: zod.z.number().positive().optional(),
+  encodingName: zod.z.string().optional(),
+  modelName: zod.z.string().optional(),
+  allowedSpecial: setOrAllSchema,
+  disallowedSpecial: setOrAllSchema
+}).strict();
 var latexChunkOptionsSchema = baseChunkOptionsSchema.strict();
 var validationSchemas = {
   character: characterChunkOptionsSchema.transform(handleDeprecatedSize),
@@ -6046,6 +6215,7 @@ var validationSchemas = {
   json: jsonChunkOptionsSchema.transform(handleDeprecatedSize),
   html: htmlChunkOptionsSchema.transform(handleDeprecatedSize),
   markdown: markdownChunkOptionsSchema.transform(handleDeprecatedSize),
+  "semantic-markdown": semanticMarkdownChunkOptionsSchema.transform(handleDeprecatedSize),
   latex: latexChunkOptionsSchema.transform(handleDeprecatedSize)
 };
 function validateChunkParams(strategy, params) {
@@ -6175,18 +6345,25 @@ var MDocument = class _MDocument {
         return "recursive";
     }
   }
+  _strategyMap;
+  get strategyMap() {
+    if (!this._strategyMap) {
+      this._strategyMap = {
+        recursive: (options) => this.chunkRecursive(options),
+        character: (options) => this.chunkCharacter(options),
+        token: (options) => this.chunkToken(options),
+        markdown: (options) => this.chunkMarkdown(options),
+        html: (options) => this.chunkHTML(options),
+        json: (options) => this.chunkJSON(options),
+        latex: (options) => this.chunkLatex(options),
+        sentence: (options) => this.chunkSentence(options),
+        "semantic-markdown": (options) => this.chunkSemanticMarkdown(options)
+      };
+    }
+    return this._strategyMap;
+  }
   async chunkBy(strategy, options) {
-    const strategyMap = {
-      recursive: (options2) => this.chunkRecursive(options2),
-      character: (options2) => this.chunkCharacter(options2),
-      token: (options2) => this.chunkToken(options2),
-      markdown: (options2) => this.chunkMarkdown(options2),
-      html: (options2) => this.chunkHTML(options2),
-      json: (options2) => this.chunkJSON(options2),
-      latex: (options2) => this.chunkLatex(options2),
-      sentence: (options2) => this.chunkSentence(options2)
-    };
-    const chunkingFunc = strategyMap[strategy];
+    const chunkingFunc = this.strategyMap[strategy];
     if (chunkingFunc) {
       await chunkingFunc(options);
     } else {
@@ -6288,6 +6465,15 @@ var MDocument = class _MDocument {
     const textSplit = rt.transformDocuments(this.chunks);
     this.chunks = textSplit;
   }
+  async chunkSemanticMarkdown(options) {
+    const rt = SemanticMarkdownTransformer.fromTikToken({
+      options,
+      encodingName: options?.encodingName,
+      modelName: options?.modelName
+    });
+    const textSplit = rt.transformDocuments(this.chunks);
+    this.chunks = textSplit;
+  }
   async chunk(params) {
     const { strategy: passedStrategy, extract, ...chunkOptions } = params || {};
     const strategy = passedStrategy || this.defaultStrategy();