@mastra/rag 1.0.6 → 1.0.7-alpha.1

package/.turbo/turbo-build.log

@@ -1,4 +1,4 @@
 
-> @mastra/rag@1.0.6-alpha.0 build /home/runner/work/mastra/mastra/packages/rag
+> @mastra/rag@1.0.7-alpha.1 build /home/runner/work/mastra/mastra/packages/rag
 > tsup --silent --config tsup.config.ts
 

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # @mastra/rag
 
+## 1.0.7-alpha.1
+
+### Patch Changes
+
+- 4a406ec: fixes TypeScript declaration file imports to ensure proper ESM compatibility
+- Updated dependencies [cb36de0]
+- Updated dependencies [a82b851]
+- Updated dependencies [41a0a0e]
+- Updated dependencies [2871020]
+- Updated dependencies [4a406ec]
+- Updated dependencies [5d377e5]
+  - @mastra/core@0.13.0-alpha.2
+
+## 1.0.7-alpha.0
+
+### Patch Changes
+
+- 351b36e: update evals and rag ai sdk package versions
+- ccd519c: Add sentence chunking strategy and strategy-specific parameter validation for all existing strategies.
+- Updated dependencies [ea0c5f2]
+- Updated dependencies [b0e43c1]
+- Updated dependencies [1fb812e]
+- Updated dependencies [35c5798]
+  - @mastra/core@0.13.0-alpha.1
+
 ## 1.0.6
 
 ### Patch Changes

package/dist/document/document.d.ts

@@ -1,5 +1,5 @@
-import { Document as Chunk } from './schema';
-import type { ChunkOptions, ChunkParams, ExtractParams } from './types';
+import { Document as Chunk } from './schema/index.js';
+import type { ChunkParams, ExtractParams, HTMLChunkOptions, RecursiveChunkOptions, CharacterChunkOptions, TokenChunkOptions, MarkdownChunkOptions, JsonChunkOptions, LatexChunkOptions, SentenceChunkOptions } from './types.js';
 export declare class MDocument {
     private chunks;
     private type;
@@ -17,13 +17,14 @@ export declare class MDocument {
     static fromJSON(jsonString: string, metadata?: Record<string, any>): MDocument;
     private defaultStrategy;
     private chunkBy;
-    chunkRecursive(options?: ChunkOptions): Promise<void>;
-    chunkCharacter(options?: ChunkOptions): Promise<void>;
-    chunkHTML(options?: ChunkOptions): Promise<void>;
-    chunkJSON(options?: ChunkOptions): Promise<void>;
-    chunkLatex(options?: ChunkOptions): Promise<void>;
-    chunkToken(options?: ChunkOptions): Promise<void>;
-    chunkMarkdown(options?: ChunkOptions): Promise<void>;
+    chunkRecursive(options?: RecursiveChunkOptions): Promise<void>;
+    chunkCharacter(options?: CharacterChunkOptions): Promise<void>;
+    chunkHTML(options?: HTMLChunkOptions): Promise<void>;
+    chunkJSON(options?: JsonChunkOptions): Promise<void>;
+    chunkLatex(options?: LatexChunkOptions): Promise<void>;
+    chunkToken(options?: TokenChunkOptions): Promise<void>;
+    chunkMarkdown(options?: MarkdownChunkOptions): Promise<void>;
+    chunkSentence(options?: SentenceChunkOptions): Promise<void>;
     chunk(params?: ChunkParams): Promise<Chunk[]>;
     getDocs(): Chunk[];
     getText(): string[];

package/dist/document/document.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../../src/document/document.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,IAAI,KAAK,EAAgC,MAAM,UAAU,CAAC;AAQ3E,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAiB,aAAa,EAAE,MAAM,SAAS,CAAC;AAEvF,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,IAAI,CAAS;gBAET,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAAE,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAOhG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC;IAmDjG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYhF,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAY9E,OAAO,CAAC,eAAe;YAeT,OAAO;IA4Bf,cAAc,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBrD,cAAc,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrD,SAAS,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBhD,SAAS,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBhD,UAAU,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAMjD,UAAU,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAUjD,aAAa,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAapD,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAenD,OAAO,IAAI,KAAK,EAAE;IAIlB,OAAO,IAAI,MAAM,EAAE;IAInB,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;CAGrC"}
+{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../../src/document/document.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,IAAI,KAAK,EAAgC,MAAM,UAAU,CAAC;AAS3E,OAAO,KAAK,EACV,WAAW,EAEX,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAErB,MAAM,SAAS,CAAC;AAGjB,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,IAAI,CAAS;gBAET,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAAE,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAOhG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC;IAmDjG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYhF,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAY9E,OAAO,CAAC,eAAe;YAeT,OAAO;IAoBf,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9D,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAU9D,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBpD,SAAS,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBpD,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAMtD,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAUtD,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa5D,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB5D,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAiBnD,OAAO,IAAI,KAAK,EAAE;IAIlB,OAAO,IAAI,MAAM,EAAE;IAInB,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;CAGrC"}

package/dist/document/extractors/base.d.ts

@@ -1,4 +1,4 @@
-import type { BaseNode } from '../schema';
+import type { BaseNode } from '../schema/index.js';
 export declare abstract class BaseExtractor {
     isTextNodeOnly: boolean;
     abstract extract(nodes: BaseNode[]): Promise<Record<string, any>[]>;

package/dist/document/extractors/index.d.ts

@@ -1,6 +1,6 @@
-export { TitleExtractor } from './title';
-export { SummaryExtractor } from './summary';
-export { QuestionsAnsweredExtractor } from './questions';
-export { KeywordExtractor } from './keywords';
-export type { KeywordExtractArgs, QuestionAnswerExtractArgs, SummaryExtractArgs, TitleExtractorsArgs } from './types';
+export { TitleExtractor } from './title.js';
+export { SummaryExtractor } from './summary.js';
+export { QuestionsAnsweredExtractor } from './questions.js';
+export { KeywordExtractor } from './keywords.js';
+export type { KeywordExtractArgs, QuestionAnswerExtractArgs, SummaryExtractArgs, TitleExtractorsArgs } from './types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/document/extractors/keywords.d.ts

@@ -1,8 +1,8 @@
 import type { MastraLanguageModel } from '@mastra/core/agent';
-import type { KeywordExtractPrompt } from '../prompts';
-import type { BaseNode } from '../schema';
-import { BaseExtractor } from './base';
-import type { KeywordExtractArgs } from './types';
+import type { KeywordExtractPrompt } from '../prompts/index.js';
+import type { BaseNode } from '../schema/index.js';
+import { BaseExtractor } from './base.js';
+import type { KeywordExtractArgs } from './types.js';
 type ExtractKeyword = {
     /**
      * Comma-separated keywords extracted from the node. May be empty if extraction fails.

package/dist/document/extractors/questions.d.ts

@@ -1,8 +1,8 @@
 import type { MastraLanguageModel } from '@mastra/core/agent';
-import type { QuestionExtractPrompt } from '../prompts';
-import type { BaseNode } from '../schema';
-import { BaseExtractor } from './base';
-import type { QuestionAnswerExtractArgs } from './types';
+import type { QuestionExtractPrompt } from '../prompts/index.js';
+import type { BaseNode } from '../schema/index.js';
+import { BaseExtractor } from './base.js';
+import type { QuestionAnswerExtractArgs } from './types.js';
 type ExtractQuestion = {
     /**
      * Questions extracted from the node as a string (may be empty if extraction fails).

package/dist/document/extractors/summary.d.ts

@@ -1,7 +1,7 @@
-import type { SummaryPrompt } from '../prompts';
-import type { BaseNode } from '../schema';
-import { BaseExtractor } from './base';
-import type { SummaryExtractArgs } from './types';
+import type { SummaryPrompt } from '../prompts/index.js';
+import type { BaseNode } from '../schema/index.js';
+import { BaseExtractor } from './base.js';
+import type { SummaryExtractArgs } from './types.js';
 type ExtractSummary = {
     sectionSummary?: string;
     prevSectionSummary?: string;

package/dist/document/extractors/title.d.ts

@@ -1,8 +1,8 @@
 import type { MastraLanguageModel } from '@mastra/core/agent';
-import type { TitleCombinePrompt, TitleExtractorPrompt } from '../prompts';
-import type { BaseNode } from '../schema';
-import { BaseExtractor } from './base';
-import type { TitleExtractorsArgs } from './types';
+import type { TitleCombinePrompt, TitleExtractorPrompt } from '../prompts/index.js';
+import type { BaseNode } from '../schema/index.js';
+import { BaseExtractor } from './base.js';
+import type { TitleExtractorsArgs } from './types.js';
 type ExtractTitle = {
     documentTitle: string;
 };

package/dist/document/extractors/types.d.ts

@@ -1,5 +1,5 @@
 import type { MastraLanguageModel } from '@mastra/core/agent';
-import type { KeywordExtractPrompt, QuestionExtractPrompt, SummaryPrompt, TitleExtractorPrompt, TitleCombinePrompt } from '../prompts';
+import type { KeywordExtractPrompt, QuestionExtractPrompt, SummaryPrompt, TitleExtractorPrompt, TitleCombinePrompt } from '../prompts/index.js';
 export type KeywordExtractArgs = {
     llm?: MastraLanguageModel;
     keywords?: number;

package/dist/document/index.d.ts

@@ -1,3 +1,3 @@
-export * from './document';
-export * from './types';
+export * from './document.js';
+export * from './types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/document/prompts/base.d.ts

@@ -1,4 +1,4 @@
-import type { BasePromptTemplateOptions, ChatMessage, PromptTemplateOptions } from './types';
+import type { BasePromptTemplateOptions, ChatMessage, PromptTemplateOptions } from './types.js';
 export declare abstract class BasePromptTemplate<const TemplatesVar extends readonly string[] = string[]> {
     templateVars: Set<string>;
     options: Partial<Record<TemplatesVar[number] | (string & {}), string>>;

package/dist/document/prompts/index.d.ts

@@ -1,4 +1,4 @@
-export { BasePromptTemplate, PromptTemplate } from './base';
-export { defaultKeywordExtractPrompt, defaultQuestionExtractPrompt, defaultSummaryPrompt, defaultTitleCombinePromptTemplate, defaultTitleExtractorPromptTemplate, } from './prompt';
-export type { KeywordExtractPrompt, QuestionExtractPrompt, SummaryPrompt, TitleCombinePrompt, TitleExtractorPrompt, } from './prompt';
+export { BasePromptTemplate, PromptTemplate } from './base.js';
+export { defaultKeywordExtractPrompt, defaultQuestionExtractPrompt, defaultSummaryPrompt, defaultTitleCombinePromptTemplate, defaultTitleExtractorPromptTemplate, } from './prompt.js';
+export type { KeywordExtractPrompt, QuestionExtractPrompt, SummaryPrompt, TitleCombinePrompt, TitleExtractorPrompt, } from './prompt.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/document/prompts/prompt.d.ts

@@ -1,4 +1,4 @@
-import { PromptTemplate } from './base';
+import { PromptTemplate } from './base.js';
 export type SummaryPrompt = PromptTemplate<['context']>;
 export type KeywordExtractPrompt = PromptTemplate<['context', 'maxKeywords']>;
 export type QuestionExtractPrompt = PromptTemplate<['context', 'numQuestions']>;

package/dist/document/schema/index.d.ts

@@ -1,4 +1,4 @@
-export { BaseNode, Document, TextNode } from './node';
-export { NodeRelationship, ObjectType } from './types';
-export type { Metadata, RelatedNodeInfo, RelatedNodeType, BaseNodeParams, TextNodeParams } from './types';
+export { BaseNode, Document, TextNode } from './node.js';
+export { NodeRelationship, ObjectType } from './types.js';
+export type { Metadata, RelatedNodeInfo, RelatedNodeType, BaseNodeParams, TextNodeParams } from './types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/document/schema/node.d.ts

@@ -1,5 +1,5 @@
-import { NodeRelationship, ObjectType } from './types';
-import type { Metadata, RelatedNodeInfo, RelatedNodeType, BaseNodeParams, TextNodeParams } from './types';
+import { NodeRelationship, ObjectType } from './types.js';
+import type { Metadata, RelatedNodeInfo, RelatedNodeType, BaseNodeParams, TextNodeParams } from './types.js';
 /**
  * Generic abstract class for retrievable nodes
  */

package/dist/document/transformers/character.d.ts

@@ -1,21 +1,10 @@
-import { Language } from '../types';
-import type { ChunkOptions } from '../types';
-import { TextTransformer } from './text';
+import { Language } from '../types.js';
+import type { BaseChunkOptions, CharacterChunkOptions, RecursiveChunkOptions } from '../types.js';
+import { TextTransformer } from './text.js';
 export declare class CharacterTransformer extends TextTransformer {
     protected separator: string;
     protected isSeparatorRegex: boolean;
-    constructor({ separator, isSeparatorRegex, options, }: {
-        separator?: string;
-        isSeparatorRegex?: boolean;
-        options?: {
-            size?: number;
-            overlap?: number;
-            lengthFunction?: (text: string) => number;
-            keepSeparator?: boolean | 'start' | 'end';
-            addStartIndex?: boolean;
-            stripWhitespace?: boolean;
-        };
-    });
+    constructor({ separator, isSeparatorRegex, ...baseOptions }?: CharacterChunkOptions);
     splitText({ text }: {
         text: string;
     }): string[];
@@ -24,23 +13,12 @@ export declare class CharacterTransformer extends TextTransformer {
 export declare class RecursiveCharacterTransformer extends TextTransformer {
     protected separators: string[];
     protected isSeparatorRegex: boolean;
-    constructor({ separators, isSeparatorRegex, options, }: {
-        separators?: string[];
-        isSeparatorRegex?: boolean;
-        options?: ChunkOptions;
-    });
+    constructor({ separators, isSeparatorRegex, language, ...baseOptions }?: RecursiveChunkOptions);
     private _splitText;
     splitText({ text }: {
         text: string;
     }): string[];
-    static fromLanguage(language: Language, options?: {
-        size?: number;
-        chunkOverlap?: number;
-        lengthFunction?: (text: string) => number;
-        keepSeparator?: boolean | 'start' | 'end';
-        addStartIndex?: boolean;
-        stripWhitespace?: boolean;
-    }): RecursiveCharacterTransformer;
+    static fromLanguage(language: Language, options?: BaseChunkOptions): RecursiveCharacterTransformer;
     static getSeparatorsForLanguage(language: Language): string[];
 }
 //# sourceMappingURL=character.d.ts.map

package/dist/document/transformers/character.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"character.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/character.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AA+CzC,qBAAa,oBAAqB,SAAQ,eAAe;IACvD,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC;gBAExB,EACV,SAAkB,EAClB,gBAAwB,EACxB,OAAY,GACb,EAAE;QACD,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;YAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;YAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;YACxB,eAAe,CAAC,EAAE,OAAO,CAAC;SAC3B,CAAC;KACH;IAMD,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAqB/C,OAAO,CAAC,YAAY;CAyBrB;AAED,qBAAa,6BAA8B,SAAQ,eAAe;IAChE,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC;gBAExB,EACV,UAAU,EACV,gBAAwB,EACxB,OAAY,GACb,EAAE;QACD,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,OAAO,CAAC,EAAE,YAAY,CAAC;KACxB;IAMD,OAAO,CAAC,UAAU;IAuDlB,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAI/C,MAAM,CAAC,YAAY,CACjB,QAAQ,EAAE,QAAQ,EAClB,OAAO,GAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,eAAe,CAAC,EAAE,OAAO,CAAC;KACtB,GACL,6BAA6B;IAKhC,MAAM,CAAC,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,EAAE;CA+E9D"}
+{"version":3,"file":"character.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/character.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,KAAK,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAE/F,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AA+CzC,qBAAa,oBAAqB,SAAQ,eAAe;IACvD,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC;gBAExB,EAAE,SAAkB,EAAE,gBAAwB,EAAE,GAAG,WAAW,EAAE,GAAE,qBAA0B;IAMxG,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAqB/C,OAAO,CAAC,YAAY;CAyBrB;AAED,qBAAa,6BAA8B,SAAQ,eAAe;IAChE,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC;gBAExB,EAAE,UAAU,EAAE,gBAAwB,EAAE,QAAQ,EAAE,GAAG,WAAW,EAAE,GAAE,qBAA0B;IAM1G,OAAO,CAAC,UAAU;IAuDlB,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAI/C,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,GAAE,gBAAqB,GAAG,6BAA6B;IAUtG,MAAM,CAAC,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,EAAE;CA+E9D"}

package/dist/document/transformers/html.d.ts

@@ -1,8 +1,11 @@
-import { Document } from '../schema';
+import { Document } from '../schema/index.js';
+import type { HTMLChunkOptions } from '../types.js';
 export declare class HTMLHeaderTransformer {
     private headersToSplitOn;
     private returnEachElement;
-    constructor(headersToSplitOn: [string, string][], returnEachElement?: boolean);
+    constructor(options: HTMLChunkOptions & {
+        headers: [string, string][];
+    });
     splitText({ text }: {
         text: string;
     }): Document[];
@@ -14,8 +17,10 @@ export declare class HTMLHeaderTransformer {
 }
 export declare class HTMLSectionTransformer {
     private headersToSplitOn;
-    private options;
-    constructor(headersToSplitOn: [string, string][], options?: Record<string, any>);
+    private textSplitter;
+    constructor(options: HTMLChunkOptions & {
+        sections: [string, string][];
+    });
     splitText(text: string): Document[];
     private getXPath;
     private splitHtmlByHeaders;

package/dist/document/transformers/html.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAWrC,qBAAa,qBAAqB;IAChC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,iBAAiB,CAAU;gBAEvB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,iBAAiB,GAAE,OAAe;IAKpF,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAwDjD,OAAO,CAAC,QAAQ;IA2BhB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,yBAAyB;IAyBjC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA8B/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,OAAO,CAAsB;gBAEzB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAKnF,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE;IAenC,OAAO,CAAC,QAAQ;IAwBhB,OAAO,CAAC,kBAAkB;IA8CpB,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAchE,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA+B/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}
+{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAWjD,qBAAa,qBAAqB;IAChC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,iBAAiB,CAAU;gBAEvB,OAAO,EAAE,gBAAgB,GAAG;QAAE,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;KAAE;IAKvE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAwDjD,OAAO,CAAC,QAAQ;IA2BhB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,yBAAyB;IAyBjC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA8B/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,YAAY,CAAgC;gBAExC,OAAO,EAAE,gBAAgB,GAAG;QAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;KAAE;IAKxE,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE;IAenC,OAAO,CAAC,QAAQ;IAwBhB,OAAO,CAAC,kBAAkB;IA8CpB,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAahE,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA+B/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}

package/dist/document/transformers/json.d.ts

@@ -1,11 +1,11 @@
-import { Document } from '../schema';
+import { Document } from '../schema/index.js';
+import type { JsonChunkOptions } from '../types.js';
 export declare class RecursiveJsonTransformer {
     private maxSize;
     private minSize;
-    constructor({ maxSize, minSize }: {
-        maxSize: number;
-        minSize?: number;
-    });
+    private ensureAscii;
+    private convertLists;
+    constructor({ maxSize, minSize, ensureAscii, convertLists }: JsonChunkOptions);
     private static jsonSize;
     /**
      * Transform JSON data while handling circular references

package/dist/document/transformers/json.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,qBAAa,wBAAwB;IACnC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;gBAEZ,EAAE,OAAc,EAAE,OAAO,EAAE,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE;IAK9E,OAAO,CAAC,MAAM,CAAC,QAAQ;IAoCvB;;OAEG;IACI,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAmChE;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,aAAa;IAS5B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IA0C5B;;;OAGG;IACH,OAAO,CAAC,WAAW;IAMnB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAOzB;;;OAGG;IACH,OAAO,CAAC,WAAW;IA8DnB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAmE1B;;;OAGG;IACH,OAAO,CAAC,eAAe;IAuBvB;;;OAGG;IACH,OAAO,CAAC,SAAS;IAuDjB;;OAEG;IACH,SAAS,CAAC,EACR,QAAQ,EACR,YAAoB,GACrB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;IAYzB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAiBtB;;OAEG;IACH,SAAS,CAAC,EACR,QAAQ,EACR,YAAoB,EACpB,WAAkB,GACnB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,GAAG,MAAM,EAAE;IAoBZ;;OAEG;IACH,eAAe,CAAC,EACd,KAAK,EACL,YAAoB,EACpB,WAAkB,EAClB,SAAS,GACV,EAAE;QACD,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;KACnC,GAAG,QAAQ,EAAE;IAoBd,kBAAkB,CAAC,EACjB,WAAW,EACX,SAAS,EACT,YAAY,GACb,EAAE;QACD,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,SAAS,EAAE,QAAQ,EAAE,CAAC;KACvB,GAAG,QAAQ,EAAE;CAiBf"}
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,qBAAa,wBAAwB;IACnC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,YAAY,CAAU;gBAElB,EAAE,OAAc,EAAE,OAAO,EAAE,WAAmB,EAAE,YAAmB,EAAE,EAAE,gBAAgB;IAOnG,OAAO,CAAC,MAAM,CAAC,QAAQ;IAoCvB;;OAEG;IACI,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAmChE;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,aAAa;IAS5B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IA0C5B;;;OAGG;IACH,OAAO,CAAC,WAAW;IAMnB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAOzB;;;OAGG;IACH,OAAO,CAAC,WAAW;IA8DnB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAmE1B;;;OAGG;IACH,OAAO,CAAC,eAAe;IAuBvB;;;OAGG;IACH,OAAO,CAAC,SAAS;IAuDjB;;OAEG;IACH,SAAS,CAAC,EACR,QAAQ,EACR,YAAoB,GACrB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;IAYzB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAiBtB;;OAEG;IACH,SAAS,CAAC,EACR,QAAQ,EACR,YAAoB,EACpB,WAAkB,GACnB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,GAAG,MAAM,EAAE;IAoBZ;;OAEG;IACH,eAAe,CAAC,EACd,KAAK,EACL,YAAoB,EACpB,WAAkB,EAClB,SAAS,GACV,EAAE;QACD,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;KACnC,GAAG,QAAQ,EAAE;IAoBd,kBAAkB,CAAC,EACjB,WAAW,EACX,SAAS,EACT,YAAY,GACb,EAAE;QACD,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,SAAS,EAAE,QAAQ,EAAE,CAAC;KACvB,GAAG,QAAQ,EAAE;CAiBf"}

package/dist/document/transformers/latex.d.ts

@@ -1,12 +1,6 @@
-import { RecursiveCharacterTransformer } from './character';
+import type { BaseChunkOptions } from '../types.js';
+import { RecursiveCharacterTransformer } from './character.js';
 export declare class LatexTransformer extends RecursiveCharacterTransformer {
-    constructor(options?: {
-        size?: number;
-        overlap?: number;
-        lengthFunction?: (text: string) => number;
-        keepSeparator?: boolean | 'start' | 'end';
-        addStartIndex?: boolean;
-        stripWhitespace?: boolean;
-    });
+    constructor(options?: BaseChunkOptions);
 }
 //# sourceMappingURL=latex.d.ts.map

package/dist/document/transformers/latex.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"latex.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/latex.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAE5D,qBAAa,gBAAiB,SAAQ,6BAA6B;gBAE/D,OAAO,GAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,eAAe,CAAC,EAAE,OAAO,CAAC;KACtB;CAKT"}
+{"version":3,"file":"latex.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/latex.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAE5D,qBAAa,gBAAiB,SAAQ,6BAA6B;gBACrD,OAAO,GAAE,gBAAqB;CAI3C"}

package/dist/document/transformers/markdown.d.ts

@@ -1,14 +1,8 @@
-import { Document } from '../schema';
-import { RecursiveCharacterTransformer } from './character';
+import { Document } from '../schema/index.js';
+import type { BaseChunkOptions } from '../types.js';
+import { RecursiveCharacterTransformer } from './character.js';
 export declare class MarkdownTransformer extends RecursiveCharacterTransformer {
-    constructor(options?: {
-        chunkSize?: number;
-        chunkOverlap?: number;
-        lengthFunction?: (text: string) => number;
-        keepSeparator?: boolean | 'start' | 'end';
-        addStartIndex?: boolean;
-        stripWhitespace?: boolean;
-    });
+    constructor(options?: BaseChunkOptions);
 }
 export declare class MarkdownHeaderTransformer {
     private headersToSplitOn;

package/dist/document/transformers/markdown.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAIrC,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAa5D,qBAAa,mBAAoB,SAAQ,6BAA6B;gBAElE,OAAO,GAAE;QACP,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,eAAe,CAAC,EAAE,OAAO,CAAC;KACtB;CAKT;AAED,qBAAa,yBAAyB;IACpC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,YAAY,CAAU;gBAElB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,cAAc,GAAE,OAAe,EAAE,YAAY,GAAE,OAAc;IAM/G,OAAO,CAAC,sBAAsB;IAuD9B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAqHjD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IAmB/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAa5D,qBAAa,mBAAoB,SAAQ,6BAA6B;gBACxD,OAAO,GAAE,gBAAqB;CAI3C;AAED,qBAAa,yBAAyB;IACpC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,YAAY,CAAU;gBAElB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,cAAc,GAAE,OAAe,EAAE,YAAY,GAAE,OAAc;IAM/G,OAAO,CAAC,sBAAsB;IAuD9B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAqHjD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IAmB/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}

package/dist/document/transformers/sentence.d.ts

@@ -0,0 +1,31 @@
+import type { SentenceChunkOptions } from '../types.js';
+import { TextTransformer } from './text.js';
+export declare class SentenceTransformer extends TextTransformer {
+    protected minSize: number;
+    protected maxSize: number;
+    protected targetSize: number;
+    protected sentenceEnders: string[];
+    protected fallbackToWords: boolean;
+    protected fallbackToCharacters: boolean;
+    protected keepSeparator: boolean | 'start' | 'end';
+    constructor(options: SentenceChunkOptions);
+    private detectSentenceBoundaries;
+    private isRealSentenceBoundary;
+    private isCommonAbbreviation;
+    /**
+     * Group sentences into chunks with integrated overlap processing
+     */
+    private groupSentencesIntoChunks;
+    /**
+     * Handle oversized sentences with fallback strategies
+     */
+    private handleOversizedSentence;
+    private splitSentenceIntoWords;
+    private splitSentenceIntoCharacters;
+    private calculateSentenceOverlap;
+    private calculateChunkSize;
+    splitText({ text }: {
+        text: string;
+    }): string[];
+}
+//# sourceMappingURL=sentence.d.ts.map

package/dist/document/transformers/sentence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/sentence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAEzC,qBAAa,mBAAoB,SAAQ,eAAe;IACtD,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;IACnC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,oBAAoB,EAAE,OAAO,CAAC;IACxC,SAAS,CAAC,aAAa,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;gBAEvC,OAAO,EAAE,oBAAoB;IAuBzC,OAAO,CAAC,wBAAwB;IA+BhC,OAAO,CAAC,sBAAsB;IAqB9B,OAAO,CAAC,oBAAoB;IA8B5B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAsDhC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IAmC9B,OAAO,CAAC,2BAA2B;IAsBnC,OAAO,CAAC,wBAAwB;IA4BhC,OAAO,CAAC,kBAAkB;IAqB1B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;CAShD"}

package/dist/document/transformers/text.d.ts

@@ -1,14 +1,14 @@
-import { Document } from '../schema';
-import type { ChunkOptions } from '../types';
-import type { Transformer } from './transformer';
+import { Document } from '../schema/index.js';
+import type { BaseChunkOptions } from '../types.js';
+import type { Transformer } from './transformer.js';
 export declare abstract class TextTransformer implements Transformer {
-    protected size: number;
+    protected maxSize: number;
     protected overlap: number;
     protected lengthFunction: (text: string) => number;
     protected keepSeparator: boolean | 'start' | 'end';
     protected addStartIndex: boolean;
     protected stripWhitespace: boolean;
-    constructor({ size, overlap, lengthFunction, keepSeparator, addStartIndex, stripWhitespace, }: ChunkOptions);
+    constructor({ maxSize, overlap, lengthFunction, keepSeparator, addStartIndex, stripWhitespace, }: BaseChunkOptions);
     setAddStartIndex(value: boolean): void;
     abstract splitText({ text }: {
         text: string;

package/dist/document/transformers/text.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,8BAAsB,eAAgB,YAAW,WAAW;IAC1D,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IACnD,SAAS,CAAC,aAAa,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;IACnD,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC;IACjC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;gBAEvB,EACV,IAAW,EACX,OAAa,EACb,cAA8C,EAC9C,aAAqB,EACrB,aAAqB,EACrB,eAAsB,GACvB,EAAE,YAAY;IAYf,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAItC,QAAQ,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAExD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA4B/E,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;IAUjD,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;IAYrD,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAQpE,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;CA4DrE"}
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,8BAAsB,eAAgB,YAAW,WAAW;IAC1D,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IACnD,SAAS,CAAC,aAAa,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;IACnD,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC;IACjC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;gBAEvB,EACV,OAAc,EACd,OAAa,EACb,cAA8C,EAC9C,aAAqB,EACrB,aAAqB,EACrB,eAAsB,GACvB,EAAE,gBAAgB;IAYnB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAItC,QAAQ,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAExD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA4B/E,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;IAUjD,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;IAYrD,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAQpE,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;CA4DrE"}

package/dist/document/transformers/token.d.ts

@@ -1,5 +1,6 @@
 import type { TiktokenModel, TiktokenEncoding } from 'js-tiktoken';
-import { TextTransformer } from './text';
+import type { TokenChunkOptions } from '../types.js';
+import { TextTransformer } from './text.js';
 interface Tokenizer {
     overlap: number;
     tokensPerChunk: number;
@@ -15,18 +16,11 @@ export declare class TokenTransformer extends TextTransformer {
     private allowedSpecial;
     private disallowedSpecial;
     constructor({ encodingName, modelName, allowedSpecial, disallowedSpecial, options, }: {
-        encodingName: TiktokenEncoding;
+        encodingName?: TiktokenEncoding;
         modelName?: TiktokenModel;
         allowedSpecial?: Set<string> | 'all';
         disallowedSpecial?: Set<string> | 'all';
-        options: {
-            size?: number;
-            overlap?: number;
-            lengthFunction?: (text: string) => number;
-            keepSeparator?: boolean | 'start' | 'end';
-            addStartIndex?: boolean;
-            stripWhitespace?: boolean;
-        };
+        options: TokenChunkOptions;
     });
     splitText({ text }: {
         text: string;
@@ -34,12 +28,7 @@ export declare class TokenTransformer extends TextTransformer {
     static fromTikToken({ encodingName, modelName, options, }: {
         encodingName?: TiktokenEncoding;
         modelName?: TiktokenModel;
-        options?: {
-            size?: number;
-            overlap?: number;
-            allowedSpecial?: Set<string> | 'all';
-            disallowedSpecial?: Set<string> | 'all';
-        };
+        options?: TokenChunkOptions;
     }): TokenTransformer;
 }
 export {};

package/dist/document/transformers/token.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/token.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAY,MAAM,aAAa,CAAC;AAG7E,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAEzC,UAAU,SAAS;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC;IACrC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CACpC;AAED,wBAAgB,iBAAiB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,GAAG,MAAM,EAAE,CAkBvG;AAED,qBAAa,gBAAiB,SAAQ,eAAe;IACnD,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,cAAc,CAAsB;IAC5C,OAAO,CAAC,iBAAiB,CAAsB;gBAEnC,EACV,YAA4B,EAC5B,SAAS,EACT,cAA0B,EAC1B,iBAAyB,EACzB,OAAY,GACb,EAAE;QACD,YAAY,EAAE,gBAAgB,CAAC;QAC/B,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;QACrC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;QACxC,OAAO,EAAE;YACP,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;YAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;YAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;YACxB,eAAe,CAAC,EAAE,OAAO,CAAC;SAC3B,CAAC;KACH;IAaD,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IA0B/C,MAAM,CAAC,YAAY,CAAC,EAClB,YAA4B,EAC5B,SAAS,EACT,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YACrC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;SACzC,CAAC;KACH,GAAG,gBAAgB;CAuCrB"}
+{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/token.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAY,MAAM,aAAa,CAAC;AAE7E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAEzC,UAAU,SAAS;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC;IACrC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CACpC;AAED,wBAAgB,iBAAiB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,GAAG,MAAM,EAAE,CAkBvG;AAED,qBAAa,gBAAiB,SAAQ,eAAe;IACnD,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,cAAc,CAAsB;IAC5C,OAAO,CAAC,iBAAiB,CAAsB;gBAEnC,EACV,YAA4B,EAC5B,SAAS,EACT,cAA0B,EAC1B,iBAAyB,EACzB,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;QACrC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;QACxC,OAAO,EAAE,iBAAiB,CAAC;KAC5B;IAaD,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IA0B/C,MAAM,CAAC,YAAY,CAAC,EAClB,YAA4B,EAC5B,SAAS,EACT,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,OAAO,CAAC,EAAE,iBAAiB,CAAC;KAC7B,GAAG,gBAAgB;CAuCrB"}

package/dist/document/transformers/transformer.d.ts

@@ -1,4 +1,4 @@
-import type { Document } from '../schema';
+import type { Document } from '../schema/index.js';
 export interface Transformer {
     transformDocuments(documents: Document[]): Document[];
 }