@dakshp1234/langchain-textsplitters 1.0.1

package/dist/text_splitter.cjs

@@ -0,0 +1,536 @@
+let _langchain_core_documents = require("@langchain/core/documents");
+let _langchain_core_utils_tiktoken = require("@langchain/core/utils/tiktoken");
+//#region src/text_splitter.ts
+var TextSplitter = class extends _langchain_core_documents.BaseDocumentTransformer {
+	lc_namespace = [
+		"langchain",
+		"document_transformers",
+		"text_splitters"
+	];
+	chunkSize = 1e3;
+	chunkOverlap = 200;
+	keepSeparator = false;
+	lengthFunction;
+	constructor(fields) {
+		super(fields);
+		this.chunkSize = fields?.chunkSize ?? this.chunkSize;
+		this.chunkOverlap = fields?.chunkOverlap ?? this.chunkOverlap;
+		this.keepSeparator = fields?.keepSeparator ?? this.keepSeparator;
+		this.lengthFunction = fields?.lengthFunction ?? ((text) => text.length);
+		if (this.chunkOverlap >= this.chunkSize) throw new Error("Cannot have chunkOverlap >= chunkSize");
+	}
+	async transformDocuments(documents, chunkHeaderOptions = {}) {
+		return this.splitDocuments(documents, chunkHeaderOptions);
+	}
+	splitOnSeparator(text, separator) {
+		let splits;
+		if (separator) if (this.keepSeparator) {
+			const regexEscapedSeparator = separator.replace(/[/\-\\^$*+?.()|[\]{}]/g, "\\$&");
+			splits = text.split(new RegExp(`(?=${regexEscapedSeparator})`));
+		} else splits = text.split(separator);
+		else splits = text.split("");
+		return splits.filter((s) => s !== "");
+	}
+	async createDocuments(texts, metadatas = [], chunkHeaderOptions = {}) {
+		const _metadatas = metadatas.length > 0 ? metadatas : [...Array(texts.length)].map(() => ({}));
+		const { chunkHeader = "", chunkOverlapHeader = "(cont'd) ", appendChunkOverlapHeader = false } = chunkHeaderOptions;
+		const documents = new Array();
+		for (let i = 0; i < texts.length; i += 1) {
+			const text = texts[i];
+			let lineCounterIndex = 1;
+			let prevChunk = null;
+			let indexPrevChunk = -1;
+			for (const chunk of await this.splitText(text)) {
+				let pageContent = chunkHeader;
+				const indexChunk = text.indexOf(chunk, indexPrevChunk + 1);
+				if (prevChunk === null) {
+					const newLinesBeforeFirstChunk = this.numberOfNewLines(text, 0, indexChunk);
+					lineCounterIndex += newLinesBeforeFirstChunk;
+				} else {
+					const indexEndPrevChunk = indexPrevChunk + await this.lengthFunction(prevChunk);
+					if (indexEndPrevChunk < indexChunk) {
+						const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexEndPrevChunk, indexChunk);
+						lineCounterIndex += numberOfIntermediateNewLines;
+					} else if (indexEndPrevChunk > indexChunk) {
+						const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexChunk, indexEndPrevChunk);
+						lineCounterIndex -= numberOfIntermediateNewLines;
+					}
+					if (appendChunkOverlapHeader) pageContent += chunkOverlapHeader;
+				}
+				const newLinesCount = this.numberOfNewLines(chunk);
+				const loc = _metadatas[i].loc && typeof _metadatas[i].loc === "object" ? { ..._metadatas[i].loc } : {};
+				loc.lines = {
+					from: lineCounterIndex,
+					to: lineCounterIndex + newLinesCount
+				};
+				const metadataWithLinesNumber = {
+					..._metadatas[i],
+					loc
+				};
+				pageContent += chunk;
+				documents.push(new _langchain_core_documents.Document({
+					pageContent,
+					metadata: metadataWithLinesNumber
+				}));
+				lineCounterIndex += newLinesCount;
+				prevChunk = chunk;
+				indexPrevChunk = indexChunk;
+			}
+		}
+		return documents;
+	}
+	numberOfNewLines(text, start, end) {
+		return (text.slice(start, end).match(/\n/g) || []).length;
+	}
+	async splitDocuments(documents, chunkHeaderOptions = {}) {
+		const selectedDocuments = documents.filter((doc) => doc.pageContent !== void 0);
+		const texts = selectedDocuments.map((doc) => doc.pageContent);
+		const metadatas = selectedDocuments.map((doc) => doc.metadata);
+		return this.createDocuments(texts, metadatas, chunkHeaderOptions);
+	}
+	joinDocs(docs, separator) {
+		const text = docs.join(separator).trim();
+		return text === "" ? null : text;
+	}
+	async mergeSplits(splits, separator) {
+		const docs = [];
+		const currentDoc = [];
+		let total = 0;
+		for (const d of splits) {
+			const _len = await this.lengthFunction(d);
+			if (total + _len + currentDoc.length * separator.length > this.chunkSize) {
+				if (total > this.chunkSize) console.warn(`Created a chunk of size ${total}, +
+which is longer than the specified ${this.chunkSize}`);
+				if (currentDoc.length > 0) {
+					const doc = this.joinDocs(currentDoc, separator);
+					if (doc !== null) docs.push(doc);
+					while (total > this.chunkOverlap || total + _len + currentDoc.length * separator.length > this.chunkSize && total > 0) {
+						total -= await this.lengthFunction(currentDoc[0]);
+						currentDoc.shift();
+					}
+				}
+			}
+			currentDoc.push(d);
+			total += _len;
+		}
+		const doc = this.joinDocs(currentDoc, separator);
+		if (doc !== null) docs.push(doc);
+		return docs;
+	}
+};
+var CharacterTextSplitter = class extends TextSplitter {
+	static lc_name() {
+		return "CharacterTextSplitter";
+	}
+	separator = "\n\n";
+	constructor(fields) {
+		super(fields);
+		this.separator = fields?.separator ?? this.separator;
+	}
+	async splitText(text) {
+		const splits = this.splitOnSeparator(text, this.separator);
+		return this.mergeSplits(splits, this.keepSeparator ? "" : this.separator);
+	}
+};
+const SupportedTextSplitterLanguages = [
+	"cpp",
+	"go",
+	"java",
+	"js",
+	"php",
+	"proto",
+	"python",
+	"rst",
+	"ruby",
+	"rust",
+	"scala",
+	"swift",
+	"markdown",
+	"latex",
+	"html",
+	"sol"
+];
+var RecursiveCharacterTextSplitter = class RecursiveCharacterTextSplitter extends TextSplitter {
+	static lc_name() {
+		return "RecursiveCharacterTextSplitter";
+	}
+	separators = [
+		"\n\n",
+		"\n",
+		" ",
+		""
+	];
+	constructor(fields) {
+		super(fields);
+		this.separators = fields?.separators ?? this.separators;
+		this.keepSeparator = fields?.keepSeparator ?? true;
+	}
+	async _splitText(text, separators) {
+		const finalChunks = [];
+		let separator = separators[separators.length - 1];
+		let newSeparators;
+		for (let i = 0; i < separators.length; i += 1) {
+			const s = separators[i];
+			if (s === "") {
+				separator = s;
+				break;
+			}
+			if (text.includes(s)) {
+				separator = s;
+				newSeparators = separators.slice(i + 1);
+				break;
+			}
+		}
+		const splits = this.splitOnSeparator(text, separator);
+		let goodSplits = [];
+		const _separator = this.keepSeparator ? "" : separator;
+		for (const s of splits) if (await this.lengthFunction(s) < this.chunkSize) goodSplits.push(s);
+		else {
+			if (goodSplits.length) {
+				const mergedText = await this.mergeSplits(goodSplits, _separator);
+				finalChunks.push(...mergedText);
+				goodSplits = [];
+			}
+			if (!newSeparators) finalChunks.push(s);
+			else {
+				const otherInfo = await this._splitText(s, newSeparators);
+				finalChunks.push(...otherInfo);
+			}
+		}
+		if (goodSplits.length) {
+			const mergedText = await this.mergeSplits(goodSplits, _separator);
+			finalChunks.push(...mergedText);
+		}
+		return finalChunks;
+	}
+	async splitText(text) {
+		return this._splitText(text, this.separators);
+	}
+	static fromLanguage(language, options) {
+		return new RecursiveCharacterTextSplitter({
+			...options,
+			separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage(language)
+		});
+	}
+	static getSeparatorsForLanguage(language) {
+		if (language === "cpp") return [
+			"\nclass ",
+			"\nvoid ",
+			"\nint ",
+			"\nfloat ",
+			"\ndouble ",
+			"\nif ",
+			"\nfor ",
+			"\nwhile ",
+			"\nswitch ",
+			"\ncase ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "go") return [
+			"\nfunc ",
+			"\nvar ",
+			"\nconst ",
+			"\ntype ",
+			"\nif ",
+			"\nfor ",
+			"\nswitch ",
+			"\ncase ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "java") return [
+			"\nclass ",
+			"\npublic ",
+			"\nprotected ",
+			"\nprivate ",
+			"\nstatic ",
+			"\nif ",
+			"\nfor ",
+			"\nwhile ",
+			"\nswitch ",
+			"\ncase ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "js") return [
+			"\nfunction ",
+			"\nconst ",
+			"\nlet ",
+			"\nvar ",
+			"\nclass ",
+			"\nif ",
+			"\nfor ",
+			"\nwhile ",
+			"\nswitch ",
+			"\ncase ",
+			"\ndefault ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "php") return [
+			"\nfunction ",
+			"\nclass ",
+			"\nif ",
+			"\nforeach ",
+			"\nwhile ",
+			"\ndo ",
+			"\nswitch ",
+			"\ncase ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "proto") return [
+			"\nmessage ",
+			"\nservice ",
+			"\nenum ",
+			"\noption ",
+			"\nimport ",
+			"\nsyntax ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "python") return [
+			"\nclass ",
+			"\ndef ",
+			"\n	def ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "rst") return [
+			"\n===\n",
+			"\n---\n",
+			"\n***\n",
+			"\n.. ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "ruby") return [
+			"\ndef ",
+			"\nclass ",
+			"\nif ",
+			"\nunless ",
+			"\nwhile ",
+			"\nfor ",
+			"\ndo ",
+			"\nbegin ",
+			"\nrescue ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "rust") return [
+			"\nfn ",
+			"\nconst ",
+			"\nlet ",
+			"\nif ",
+			"\nwhile ",
+			"\nfor ",
+			"\nloop ",
+			"\nmatch ",
+			"\nconst ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "scala") return [
+			"\nclass ",
+			"\nobject ",
+			"\ndef ",
+			"\nval ",
+			"\nvar ",
+			"\nif ",
+			"\nfor ",
+			"\nwhile ",
+			"\nmatch ",
+			"\ncase ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "swift") return [
+			"\nfunc ",
+			"\nclass ",
+			"\nstruct ",
+			"\nenum ",
+			"\nif ",
+			"\nfor ",
+			"\nwhile ",
+			"\ndo ",
+			"\nswitch ",
+			"\ncase ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "markdown") return [
+			"\n## ",
+			"\n### ",
+			"\n#### ",
+			"\n##### ",
+			"\n###### ",
+			"```\n\n",
+			"\n\n***\n\n",
+			"\n\n---\n\n",
+			"\n\n___\n\n",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "latex") return [
+			"\n\\chapter{",
+			"\n\\section{",
+			"\n\\subsection{",
+			"\n\\subsubsection{",
+			"\n\\begin{enumerate}",
+			"\n\\begin{itemize}",
+			"\n\\begin{description}",
+			"\n\\begin{list}",
+			"\n\\begin{quote}",
+			"\n\\begin{quotation}",
+			"\n\\begin{verse}",
+			"\n\\begin{verbatim}",
+			"\n\\begin{align}",
+			"$$",
+			"$",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else if (language === "html") return [
+			"<body>",
+			"<div>",
+			"<p>",
+			"<br>",
+			"<li>",
+			"<h1>",
+			"<h2>",
+			"<h3>",
+			"<h4>",
+			"<h5>",
+			"<h6>",
+			"<span>",
+			"<table>",
+			"<tr>",
+			"<td>",
+			"<th>",
+			"<ul>",
+			"<ol>",
+			"<header>",
+			"<footer>",
+			"<nav>",
+			"<head>",
+			"<style>",
+			"<script>",
+			"<meta>",
+			"<title>",
+			" ",
+			""
+		];
+		else if (language === "sol") return [
+			"\npragma ",
+			"\nusing ",
+			"\ncontract ",
+			"\ninterface ",
+			"\nlibrary ",
+			"\nconstructor ",
+			"\ntype ",
+			"\nfunction ",
+			"\nevent ",
+			"\nmodifier ",
+			"\nerror ",
+			"\nstruct ",
+			"\nenum ",
+			"\nif ",
+			"\nfor ",
+			"\nwhile ",
+			"\ndo while ",
+			"\nassembly ",
+			"\n\n",
+			"\n",
+			" ",
+			""
+		];
+		else throw new Error(`Language ${language} is not supported.`);
+	}
+};
+/**
+* Implementation of splitter which looks at tokens.
+*/
+var TokenTextSplitter = class extends TextSplitter {
+	static lc_name() {
+		return "TokenTextSplitter";
+	}
+	encodingName;
+	allowedSpecial;
+	disallowedSpecial;
+	tokenizer;
+	constructor(fields) {
+		super(fields);
+		this.encodingName = fields?.encodingName ?? "gpt2";
+		this.allowedSpecial = fields?.allowedSpecial ?? [];
+		this.disallowedSpecial = fields?.disallowedSpecial ?? "all";
+	}
+	async splitText(text) {
+		if (!this.tokenizer) this.tokenizer = await (0, _langchain_core_utils_tiktoken.getEncoding)(this.encodingName);
+		const splits = [];
+		const input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);
+		let start_idx = 0;
+		while (start_idx < input_ids.length) {
+			if (start_idx > 0) start_idx -= this.chunkOverlap;
+			const end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
+			const chunk_ids = input_ids.slice(start_idx, end_idx);
+			splits.push(this.tokenizer.decode(chunk_ids));
+			start_idx = end_idx;
+		}
+		return splits;
+	}
+};
+var MarkdownTextSplitter = class extends RecursiveCharacterTextSplitter {
+	constructor(fields) {
+		super({
+			...fields,
+			separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage("markdown")
+		});
+	}
+};
+var LatexTextSplitter = class extends RecursiveCharacterTextSplitter {
+	constructor(fields) {
+		super({
+			...fields,
+			separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage("latex")
+		});
+	}
+};
+//#endregion
+exports.CharacterTextSplitter = CharacterTextSplitter;
+exports.LatexTextSplitter = LatexTextSplitter;
+exports.MarkdownTextSplitter = MarkdownTextSplitter;
+exports.RecursiveCharacterTextSplitter = RecursiveCharacterTextSplitter;
+exports.SupportedTextSplitterLanguages = SupportedTextSplitterLanguages;
+exports.TextSplitter = TextSplitter;
+exports.TokenTextSplitter = TokenTextSplitter;
+
+//# sourceMappingURL=text_splitter.cjs.map

package/dist/text_splitter.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"text_splitter.cjs","names":["BaseDocumentTransformer","Document"],"sources":["../src/text_splitter.ts"],"sourcesContent":["import type * as tiktoken from \"js-tiktoken\";\nimport { Document, BaseDocumentTransformer } from \"@langchain/core/documents\";\nimport { getEncoding } from \"@langchain/core/utils/tiktoken\";\n\nexport interface TextSplitterParams {\n  chunkSize: number;\n  chunkOverlap: number;\n  keepSeparator: boolean;\n  lengthFunction?:\n    | ((text: string) => number)\n    | ((text: string) => Promise<number>);\n}\n\nexport type TextSplitterChunkHeaderOptions = {\n  chunkHeader?: string;\n  chunkOverlapHeader?: string;\n  appendChunkOverlapHeader?: boolean;\n};\n\nexport abstract class TextSplitter\n  extends BaseDocumentTransformer\n  implements TextSplitterParams\n{\n  lc_namespace = [\"langchain\", \"document_transformers\", \"text_splitters\"];\n\n  chunkSize = 1000;\n\n  chunkOverlap = 200;\n\n  keepSeparator = false;\n\n  lengthFunction:\n    | ((text: string) => number)\n    | ((text: string) => Promise<number>);\n\n  constructor(fields?: Partial<TextSplitterParams>) {\n    super(fields);\n    this.chunkSize = fields?.chunkSize ?? this.chunkSize;\n    this.chunkOverlap = fields?.chunkOverlap ?? this.chunkOverlap;\n    this.keepSeparator = fields?.keepSeparator ?? this.keepSeparator;\n    this.lengthFunction =\n      fields?.lengthFunction ?? ((text: string) => text.length);\n    if (this.chunkOverlap >= this.chunkSize) {\n      throw new Error(\"Cannot have chunkOverlap >= chunkSize\");\n    }\n  }\n\n  async transformDocuments(\n    documents: Document[],\n    chunkHeaderOptions: TextSplitterChunkHeaderOptions = {}\n  ): Promise<Document[]> {\n    return this.splitDocuments(documents, chunkHeaderOptions);\n  }\n\n  abstract splitText(text: string): Promise<string[]>;\n\n  protected splitOnSeparator(text: string, separator: string): string[] {\n    let splits;\n    if (separator) {\n      if (this.keepSeparator) {\n        const regexEscapedSeparator = separator.replace(\n          /[/\\-\\\\^$*+?.()|[\\]{}]/g,\n          \"\\\\$&\"\n        );\n        splits = text.split(new RegExp(`(?=${regexEscapedSeparator})`));\n      } else {\n        splits = text.split(separator);\n      }\n    } else {\n      splits = text.split(\"\");\n    }\n    return splits.filter((s) => s !== \"\");\n  }\n\n  async createDocuments(\n    texts: string[],\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    metadatas: Record<string, any>[] = [],\n    chunkHeaderOptions: TextSplitterChunkHeaderOptions = {}\n  ): Promise<Document[]> {\n    // if no metadata is provided, we create an empty one for each text\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    const _metadatas: Record<string, any>[] =\n      metadatas.length > 0\n        ? metadatas\n        : [...Array(texts.length)].map(() => ({}));\n    const {\n      chunkHeader = \"\",\n      chunkOverlapHeader = \"(cont'd) \",\n      appendChunkOverlapHeader = false,\n    } = chunkHeaderOptions;\n    const documents = new Array<Document>();\n    for (let i = 0; i < texts.length; i += 1) {\n      const text = texts[i];\n      let lineCounterIndex = 1;\n      let prevChunk = null;\n      let indexPrevChunk = -1;\n      for (const chunk of await this.splitText(text)) {\n        let pageContent = chunkHeader;\n\n        // we need to count the \\n that are in the text before getting removed by the splitting\n        const indexChunk = text.indexOf(chunk, indexPrevChunk + 1);\n        if (prevChunk === null) {\n          const newLinesBeforeFirstChunk = this.numberOfNewLines(\n            text,\n            0,\n            indexChunk\n          );\n          lineCounterIndex += newLinesBeforeFirstChunk;\n        } else {\n          const indexEndPrevChunk =\n            indexPrevChunk + (await this.lengthFunction(prevChunk));\n          if (indexEndPrevChunk < indexChunk) {\n            const numberOfIntermediateNewLines = this.numberOfNewLines(\n              text,\n              indexEndPrevChunk,\n              indexChunk\n            );\n            lineCounterIndex += numberOfIntermediateNewLines;\n          } else if (indexEndPrevChunk > indexChunk) {\n            const numberOfIntermediateNewLines = this.numberOfNewLines(\n              text,\n              indexChunk,\n              indexEndPrevChunk\n            );\n            lineCounterIndex -= numberOfIntermediateNewLines;\n          }\n          if (appendChunkOverlapHeader) {\n            pageContent += chunkOverlapHeader;\n          }\n        }\n        const newLinesCount = this.numberOfNewLines(chunk);\n\n        const loc =\n          _metadatas[i].loc && typeof _metadatas[i].loc === \"object\"\n            ? { ..._metadatas[i].loc }\n            : {};\n        loc.lines = {\n          from: lineCounterIndex,\n          to: lineCounterIndex + newLinesCount,\n        };\n        const metadataWithLinesNumber = {\n          ..._metadatas[i],\n          loc,\n        };\n\n        pageContent += chunk;\n        documents.push(\n          new Document({\n            pageContent,\n            metadata: metadataWithLinesNumber,\n          })\n        );\n        lineCounterIndex += newLinesCount;\n        prevChunk = chunk;\n        indexPrevChunk = indexChunk;\n      }\n    }\n    return documents;\n  }\n\n  private numberOfNewLines(text: string, start?: number, end?: number) {\n    const textSection = text.slice(start, end);\n    return (textSection.match(/\\n/g) || []).length;\n  }\n\n  async splitDocuments(\n    documents: Document[],\n    chunkHeaderOptions: TextSplitterChunkHeaderOptions = {}\n  ): Promise<Document[]> {\n    const selectedDocuments = documents.filter(\n      (doc) => doc.pageContent !== undefined\n    );\n    const texts = selectedDocuments.map((doc) => doc.pageContent);\n    const metadatas = selectedDocuments.map((doc) => doc.metadata);\n    return this.createDocuments(texts, metadatas, chunkHeaderOptions);\n  }\n\n  private joinDocs(docs: string[], separator: string): string | null {\n    const text = docs.join(separator).trim();\n    return text === \"\" ? null : text;\n  }\n\n  async mergeSplits(splits: string[], separator: string): Promise<string[]> {\n    const docs: string[] = [];\n    const currentDoc: string[] = [];\n    let total = 0;\n    for (const d of splits) {\n      const _len = await this.lengthFunction(d);\n      if (\n        total + _len + currentDoc.length * separator.length >\n        this.chunkSize\n      ) {\n        if (total > this.chunkSize) {\n          console.warn(\n            `Created a chunk of size ${total}, +\nwhich is longer than the specified ${this.chunkSize}`\n          );\n        }\n        if (currentDoc.length > 0) {\n          const doc = this.joinDocs(currentDoc, separator);\n          if (doc !== null) {\n            docs.push(doc);\n          }\n          // Keep on popping if:\n          // - we have a larger chunk than in the chunk overlap\n          // - or if we still have any chunks and the length is long\n          while (\n            total > this.chunkOverlap ||\n            (total + _len + currentDoc.length * separator.length >\n              this.chunkSize &&\n              total > 0)\n          ) {\n            total -= await this.lengthFunction(currentDoc[0]);\n            currentDoc.shift();\n          }\n        }\n      }\n      currentDoc.push(d);\n      total += _len;\n    }\n    const doc = this.joinDocs(currentDoc, separator);\n    if (doc !== null) {\n      docs.push(doc);\n    }\n    return docs;\n  }\n}\n\nexport interface CharacterTextSplitterParams extends TextSplitterParams {\n  separator: string;\n}\n\nexport class CharacterTextSplitter\n  extends TextSplitter\n  implements CharacterTextSplitterParams\n{\n  static lc_name() {\n    return \"CharacterTextSplitter\";\n  }\n\n  separator = \"\\n\\n\";\n\n  constructor(fields?: Partial<CharacterTextSplitterParams>) {\n    super(fields);\n    this.separator = fields?.separator ?? this.separator;\n  }\n\n  async splitText(text: string): Promise<string[]> {\n    // First we naively split the large input into a bunch of smaller ones.\n    const splits = this.splitOnSeparator(text, this.separator);\n    return this.mergeSplits(splits, this.keepSeparator ? \"\" : this.separator);\n  }\n}\n\nexport interface RecursiveCharacterTextSplitterParams extends TextSplitterParams {\n  separators: string[];\n}\n\nexport const SupportedTextSplitterLanguages = [\n  \"cpp\",\n  \"go\",\n  \"java\",\n  \"js\",\n  \"php\",\n  \"proto\",\n  \"python\",\n  \"rst\",\n  \"ruby\",\n  \"rust\",\n  \"scala\",\n  \"swift\",\n  \"markdown\",\n  \"latex\",\n  \"html\",\n  \"sol\",\n] as const;\n\nexport type SupportedTextSplitterLanguage =\n  (typeof SupportedTextSplitterLanguages)[number];\n\nexport class RecursiveCharacterTextSplitter\n  extends TextSplitter\n  implements RecursiveCharacterTextSplitterParams\n{\n  static lc_name() {\n    return \"RecursiveCharacterTextSplitter\";\n  }\n\n  separators: string[] = [\"\\n\\n\", \"\\n\", \" \", \"\"];\n\n  constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>) {\n    super(fields);\n    this.separators = fields?.separators ?? this.separators;\n    this.keepSeparator = fields?.keepSeparator ?? true;\n  }\n\n  private async _splitText(text: string, separators: string[]) {\n    const finalChunks: string[] = [];\n\n    // Get appropriate separator to use\n    let separator: string = separators[separators.length - 1];\n    let newSeparators;\n    for (let i = 0; i < separators.length; i += 1) {\n      const s = separators[i];\n      if (s === \"\") {\n        separator = s;\n        break;\n      }\n      if (text.includes(s)) {\n        separator = s;\n        newSeparators = separators.slice(i + 1);\n        break;\n      }\n    }\n\n    // Now that we have the separator, split the text\n    const splits = this.splitOnSeparator(text, separator);\n\n    // Now go merging things, recursively splitting longer texts.\n    let goodSplits: string[] = [];\n    const _separator = this.keepSeparator ? \"\" : separator;\n    for (const s of splits) {\n      if ((await this.lengthFunction(s)) < this.chunkSize) {\n        goodSplits.push(s);\n      } else {\n        if (goodSplits.length) {\n          const mergedText = await this.mergeSplits(goodSplits, _separator);\n          finalChunks.push(...mergedText);\n          goodSplits = [];\n        }\n        if (!newSeparators) {\n          finalChunks.push(s);\n        } else {\n          const otherInfo = await this._splitText(s, newSeparators);\n          finalChunks.push(...otherInfo);\n        }\n      }\n    }\n    if (goodSplits.length) {\n      const mergedText = await this.mergeSplits(goodSplits, _separator);\n      finalChunks.push(...mergedText);\n    }\n    return finalChunks;\n  }\n\n  async splitText(text: string): Promise<string[]> {\n    return this._splitText(text, this.separators);\n  }\n\n  static fromLanguage(\n    language: SupportedTextSplitterLanguage,\n    options?: Partial<RecursiveCharacterTextSplitterParams>\n  ) {\n    return new RecursiveCharacterTextSplitter({\n      ...options,\n      separators:\n        RecursiveCharacterTextSplitter.getSeparatorsForLanguage(language),\n    });\n  }\n\n  static getSeparatorsForLanguage(language: SupportedTextSplitterLanguage) {\n    if (language === \"cpp\") {\n      return [\n        // Split along class definitions\n        \"\\nclass \",\n        // Split along function definitions\n        \"\\nvoid \",\n        \"\\nint \",\n        \"\\nfloat \",\n        \"\\ndouble \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"go\") {\n      return [\n        // Split along function definitions\n        \"\\nfunc \",\n        \"\\nvar \",\n        \"\\nconst \",\n        \"\\ntype \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"java\") {\n      return [\n        // Split along class definitions\n        \"\\nclass \",\n        // Split along method definitions\n        \"\\npublic \",\n        \"\\nprotected \",\n        \"\\nprivate \",\n        \"\\nstatic \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"js\") {\n      return [\n        // Split along function definitions\n        \"\\nfunction \",\n        \"\\nconst \",\n        \"\\nlet \",\n        \"\\nvar \",\n        \"\\nclass \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        \"\\ndefault \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"php\") {\n      return [\n        // Split along function definitions\n        \"\\nfunction \",\n        // Split along class definitions\n        \"\\nclass \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nforeach \",\n        \"\\nwhile \",\n        \"\\ndo \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"proto\") {\n      return [\n        // Split along message definitions\n        \"\\nmessage \",\n        // Split along service definitions\n        \"\\nservice \",\n        // Split along enum definitions\n        \"\\nenum \",\n        // Split along option definitions\n        \"\\noption \",\n        // Split along import statements\n        \"\\nimport \",\n        // Split along syntax declarations\n        \"\\nsyntax \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"python\") {\n      return [\n        // First, try to split along class definitions\n        \"\\nclass \",\n        \"\\ndef \",\n        \"\\n\\tdef \",\n        // Now split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"rst\") {\n      return [\n        // Split along section titles\n        \"\\n===\\n\",\n        \"\\n---\\n\",\n        \"\\n***\\n\",\n        // Split along directive markers\n        \"\\n.. \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"ruby\") {\n      return [\n        // Split along method definitions\n        \"\\ndef \",\n        \"\\nclass \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nunless \",\n        \"\\nwhile \",\n        \"\\nfor \",\n        \"\\ndo \",\n        \"\\nbegin \",\n        \"\\nrescue \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"rust\") {\n      return [\n        // Split along function definitions\n        \"\\nfn \",\n        \"\\nconst \",\n        \"\\nlet \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nwhile \",\n        \"\\nfor \",\n        \"\\nloop \",\n        \"\\nmatch \",\n        \"\\nconst \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"scala\") {\n      return [\n        // Split along class definitions\n        \"\\nclass \",\n        \"\\nobject \",\n        // Split along method definitions\n        \"\\ndef \",\n        \"\\nval \",\n        \"\\nvar \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nmatch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"swift\") {\n      return [\n        // Split along function definitions\n        \"\\nfunc \",\n        // Split along class definitions\n        \"\\nclass \",\n        \"\\nstruct \",\n        \"\\nenum \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\ndo \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"markdown\") {\n      return [\n        // First, try to split along Markdown headings (starting with level 2)\n        \"\\n## \",\n        \"\\n### \",\n        \"\\n#### \",\n        \"\\n##### \",\n        \"\\n###### \",\n        // Note the alternative syntax for headings (below) is not handled here\n        // Heading level 2\n        // ---------------\n        // End of code block\n        \"```\\n\\n\",\n        // Horizontal lines\n        \"\\n\\n***\\n\\n\",\n        \"\\n\\n---\\n\\n\",\n        \"\\n\\n___\\n\\n\",\n        // Note that this splitter doesn't handle horizontal lines defined\n        // by *three or more* of ***, ---, or ___, but this is not handled\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"latex\") {\n      return [\n        // First, try to split along Latex sections\n        \"\\n\\\\chapter{\",\n        \"\\n\\\\section{\",\n        \"\\n\\\\subsection{\",\n        \"\\n\\\\subsubsection{\",\n\n        // Now split by environments\n        \"\\n\\\\begin{enumerate}\",\n        \"\\n\\\\begin{itemize}\",\n        \"\\n\\\\begin{description}\",\n        \"\\n\\\\begin{list}\",\n        \"\\n\\\\begin{quote}\",\n        \"\\n\\\\begin{quotation}\",\n        \"\\n\\\\begin{verse}\",\n        \"\\n\\\\begin{verbatim}\",\n\n        // Now split by math environments\n        \"\\n\\\\begin{align}\",\n        \"$$\",\n        \"$\",\n\n        // Now split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"html\") {\n      return [\n        // First, try to split along HTML tags\n        \"<body>\",\n        \"<div>\",\n        \"<p>\",\n        \"<br>\",\n        \"<li>\",\n        \"<h1>\",\n        \"<h2>\",\n        \"<h3>\",\n        \"<h4>\",\n        \"<h5>\",\n        \"<h6>\",\n        \"<span>\",\n        \"<table>\",\n        \"<tr>\",\n        \"<td>\",\n        \"<th>\",\n        \"<ul>\",\n        \"<ol>\",\n        \"<header>\",\n        \"<footer>\",\n        \"<nav>\",\n        // Head\n        \"<head>\",\n        \"<style>\",\n        \"<script>\",\n        \"<meta>\",\n        \"<title>\",\n        // Normal type of lines\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"sol\") {\n      return [\n        // Split along compiler informations definitions\n        \"\\npragma \",\n        \"\\nusing \",\n        // Split along contract definitions\n        \"\\ncontract \",\n        \"\\ninterface \",\n        \"\\nlibrary \",\n        // Split along method definitions\n        \"\\nconstructor \",\n        \"\\ntype \",\n        \"\\nfunction \",\n        \"\\nevent \",\n        \"\\nmodifier \",\n        \"\\nerror \",\n        \"\\nstruct \",\n        \"\\nenum \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\ndo while \",\n        \"\\nassembly \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else {\n      throw new Error(`Language ${language} is not supported.`);\n    }\n  }\n}\n\nexport interface TokenTextSplitterParams extends TextSplitterParams {\n  encodingName: tiktoken.TiktokenEncoding;\n  allowedSpecial: \"all\" | Array<string>;\n  disallowedSpecial: \"all\" | Array<string>;\n}\n\n/**\n * Implementation of splitter which looks at tokens.\n */\nexport class TokenTextSplitter\n  extends TextSplitter\n  implements TokenTextSplitterParams\n{\n  static lc_name() {\n    return \"TokenTextSplitter\";\n  }\n\n  encodingName: tiktoken.TiktokenEncoding;\n\n  allowedSpecial: \"all\" | Array<string>;\n\n  disallowedSpecial: \"all\" | Array<string>;\n\n  private tokenizer: tiktoken.Tiktoken;\n\n  constructor(fields?: Partial<TokenTextSplitterParams>) {\n    super(fields);\n\n    this.encodingName = fields?.encodingName ?? \"gpt2\";\n    this.allowedSpecial = fields?.allowedSpecial ?? [];\n    this.disallowedSpecial = fields?.disallowedSpecial ?? \"all\";\n  }\n\n  async splitText(text: string): Promise<string[]> {\n    if (!this.tokenizer) {\n      this.tokenizer = await getEncoding(this.encodingName);\n    }\n\n    const splits: string[] = [];\n\n    const input_ids = this.tokenizer.encode(\n      text,\n      this.allowedSpecial,\n      this.disallowedSpecial\n    );\n\n    let start_idx = 0;\n\n    while (start_idx < input_ids.length) {\n      if (start_idx > 0) {\n        start_idx -= this.chunkOverlap;\n      }\n      const end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);\n      const chunk_ids = input_ids.slice(start_idx, end_idx);\n      splits.push(this.tokenizer.decode(chunk_ids));\n      start_idx = end_idx;\n    }\n\n    return splits;\n  }\n}\n\nexport type MarkdownTextSplitterParams = TextSplitterParams;\n\nexport class MarkdownTextSplitter\n  extends RecursiveCharacterTextSplitter\n  implements MarkdownTextSplitterParams\n{\n  constructor(fields?: Partial<MarkdownTextSplitterParams>) {\n    super({\n      ...fields,\n      separators:\n        RecursiveCharacterTextSplitter.getSeparatorsForLanguage(\"markdown\"),\n    });\n  }\n}\n\nexport type LatexTextSplitterParams = TextSplitterParams;\n\nexport class LatexTextSplitter\n  extends RecursiveCharacterTextSplitter\n  implements LatexTextSplitterParams\n{\n  constructor(fields?: Partial<LatexTextSplitterParams>) {\n    super({\n      ...fields,\n      separators:\n        RecursiveCharacterTextSplitter.getSeparatorsForLanguage(\"latex\"),\n    });\n  }\n}\n"],"mappings":";;;AAmBA,IAAsB,eAAtB,cACUA,0BAAAA,wBAEV;CACE,eAAe;EAAC;EAAa;EAAyB;EAAiB;CAEvE,YAAY;CAEZ,eAAe;CAEf,gBAAgB;CAEhB;CAIA,YAAY,QAAsC;AAChD,QAAM,OAAO;AACb,OAAK,YAAY,QAAQ,aAAa,KAAK;AAC3C,OAAK,eAAe,QAAQ,gBAAgB,KAAK;AACjD,OAAK,gBAAgB,QAAQ,iBAAiB,KAAK;AACnD,OAAK,iBACH,QAAQ,oBAAoB,SAAiB,KAAK;AACpD,MAAI,KAAK,gBAAgB,KAAK,UAC5B,OAAM,IAAI,MAAM,wCAAwC;;CAI5D,MAAM,mBACJ,WACA,qBAAqD,EAAE,EAClC;AACrB,SAAO,KAAK,eAAe,WAAW,mBAAmB;;CAK3D,iBAA2B,MAAc,WAA6B;EACpE,IAAI;AACJ,MAAI,UACF,KAAI,KAAK,eAAe;GACtB,MAAM,wBAAwB,UAAU,QACtC,0BACA,OACD;AACD,YAAS,KAAK,MAAM,IAAI,OAAO,MAAM,sBAAsB,GAAG,CAAC;QAE/D,UAAS,KAAK,MAAM,UAAU;MAGhC,UAAS,KAAK,MAAM,GAAG;AAEzB,SAAO,OAAO,QAAQ,MAAM,MAAM,GAAG;;CAGvC,MAAM,gBACJ,OAEA,YAAmC,EAAE,EACrC,qBAAqD,EAAE,EAClC;EAGrB,MAAM,aACJ,UAAU,SAAS,IACf,YACA,CAAC,GAAG,MAAM,MAAM,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE;EAC9C,MAAM,EACJ,cAAc,IACd,qBAAqB,aACrB,2BAA2B,UACzB;EACJ,MAAM,YAAY,IAAI,OAAiB;AACvC,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAM,OAAO,MAAM;GACnB,IAAI,mBAAmB;GACvB,IAAI,YAAY;GAChB,IAAI,iBAAiB;AACrB,QAAK,MAAM,SAAS,MAAM,KAAK,UAAU,KAAK,EAAE;IAC9C,IAAI,cAAc;IAGlB,MAAM,aAAa,KAAK,QAAQ,OAAO,iBAAiB,EAAE;AAC1D,QAAI,cAAc,MAAM;KACtB,MAAM,2BAA2B,KAAK,iBACpC,MACA,GACA,WACD;AACD,yBAAoB;WACf;KACL,MAAM,oBACJ,iBAAkB,MAAM,KAAK,eAAe,UAAU;AACxD,SAAI,oBAAoB,YAAY;MAClC,MAAM,+BAA+B,KAAK,iBACxC,MACA,mBACA,WACD;AACD,0BAAoB;gBACX,oBAAoB,YAAY;MACzC,MAAM,+BAA+B,KAAK,iBACxC,MACA,YACA,kBACD;AACD,0BAAoB;;AAEtB,SAAI,yBACF,gBAAe;;IAGnB,MAAM,gBAAgB,KAAK,iBAAiB,MAAM;IAElD,MAAM,MACJ,WAAW,GAAG,OAAO,OAAO,WAAW,GAAG,QAAQ,WAC9C,EAAE,GAAG,WAAW,GAAG,KAAK,GACxB,EAAE;AACR,QAAI,QAAQ;KACV,MAAM;KACN,IAAI,mBAAmB;KACxB;IACD,MAAM,0BAA0B;KAC9B,GAAG,WAAW;KACd;KACD;AAED,mBAAe;AACf,cAAU,KACR,IAAIC,0BAAAA,SAAS;KACX;KACA,UAAU;KACX,CAAC,CACH;AACD,wBAAoB;AACpB,gBAAY;AACZ,qBAAiB;;;AAGrB,SAAO;;CAGT,iBAAyB,MAAc,OAAgB,KAAc;AAEnE,UADoB,KAAK,MAAM,OAAO,IAAI,CACtB,MAAM,MAAM,IAAI,EAAE,EAAE;;CAG1C,MAAM,eACJ,WACA,qBAAqD,EAAE,EAClC;EACrB,MAAM,oBAAoB,UAAU,QACjC,QAAQ,IAAI,gBAAgB,KAAA,EAC9B;EACD,MAAM,QAAQ,kBAAkB,KAAK,QAAQ,IAAI,YAAY;EAC7D,MAAM,YAAY,kBAAkB,KAAK,QAAQ,IAAI,SAAS;AAC9D,SAAO,KAAK,gBAAgB,OAAO,WAAW,mBAAmB;;CAGnE,SAAiB,MAAgB,WAAkC;EACjE,MAAM,OAAO,KAAK,KAAK,UAAU,CAAC,MAAM;AACxC,SAAO,SAAS,KAAK,OAAO;;CAG9B,MAAM,YAAY,QAAkB,WAAsC;EACxE,MAAM,OAAiB,EAAE;EACzB,MAAM,aAAuB,EAAE;EAC/B,IAAI,QAAQ;AACZ,OAAK,MAAM,KAAK,QAAQ;GACtB,MAAM,OAAO,MAAM,KAAK,eAAe,EAAE;AACzC,OACE,QAAQ,OAAO,WAAW,SAAS,UAAU,SAC7C,KAAK,WACL;AACA,QAAI,QAAQ,KAAK,UACf,SAAQ,KACN,2BAA2B,MAAM;qCACR,KAAK,YAC/B;AAEH,QAAI,WAAW,SAAS,GAAG;KACzB,MAAM,MAAM,KAAK,SAAS,YAAY,UAAU;AAChD,SAAI,QAAQ,KACV,MAAK,KAAK,IAAI;AAKhB,YACE,QAAQ,KAAK,gBACZ,QAAQ,OAAO,WAAW,SAAS,UAAU,SAC5C,KAAK,aACL,QAAQ,GACV;AACA,eAAS,MAAM,KAAK,eAAe,WAAW,GAAG;AACjD,iBAAW,OAAO;;;;AAIxB,cAAW,KAAK,EAAE;AAClB,YAAS;;EAEX,MAAM,MAAM,KAAK,SAAS,YAAY,UAAU;AAChD,MAAI,QAAQ,KACV,MAAK,KAAK,IAAI;AAEhB,SAAO;;;AAQX,IAAa,wBAAb,cACU,aAEV;CACE,OAAO,UAAU;AACf,SAAO;;CAGT,YAAY;CAEZ,YAAY,QAA+C;AACzD,QAAM,OAAO;AACb,OAAK,YAAY,QAAQ,aAAa,KAAK;;CAG7C,MAAM,UAAU,MAAiC;EAE/C,MAAM,SAAS,KAAK,iBAAiB,MAAM,KAAK,UAAU;AAC1D,SAAO,KAAK,YAAY,QAAQ,KAAK,gBAAgB,KAAK,KAAK,UAAU;;;AAQ7E,MAAa,iCAAiC;CAC5C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAKD,IAAa,iCAAb,MAAa,uCACH,aAEV;CACE,OAAO,UAAU;AACf,SAAO;;CAGT,aAAuB;EAAC;EAAQ;EAAM;EAAK;EAAG;CAE9C,YAAY,QAAwD;AAClE,QAAM,OAAO;AACb,OAAK,aAAa,QAAQ,cAAc,KAAK;AAC7C,OAAK,gBAAgB,QAAQ,iBAAiB;;CAGhD,MAAc,WAAW,MAAc,YAAsB;EAC3D,MAAM,cAAwB,EAAE;EAGhC,IAAI,YAAoB,WAAW,WAAW,SAAS;EACvD,IAAI;AACJ,OAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK,GAAG;GAC7C,MAAM,IAAI,WAAW;AACrB,OAAI,MAAM,IAAI;AACZ,gBAAY;AACZ;;AAEF,OAAI,KAAK,SAAS,EAAE,EAAE;AACpB,gBAAY;AACZ,oBAAgB,WAAW,MAAM,IAAI,EAAE;AACvC;;;EAKJ,MAAM,SAAS,KAAK,iBAAiB,MAAM,UAAU;EAGrD,IAAI,aAAuB,EAAE;EAC7B,MAAM,aAAa,KAAK,gBAAgB,KAAK;AAC7C,OAAK,MAAM,KAAK,OACd,KAAK,MAAM,KAAK,eAAe,EAAE,GAAI,KAAK,UACxC,YAAW,KAAK,EAAE;OACb;AACL,OAAI,WAAW,QAAQ;IACrB,MAAM,aAAa,MAAM,KAAK,YAAY,YAAY,WAAW;AACjE,gBAAY,KAAK,GAAG,WAAW;AAC/B,iBAAa,EAAE;;AAEjB,OAAI,CAAC,cACH,aAAY,KAAK,EAAE;QACd;IACL,MAAM,YAAY,MAAM,KAAK,WAAW,GAAG,cAAc;AACzD,gBAAY,KAAK,GAAG,UAAU;;;AAIpC,MAAI,WAAW,QAAQ;GACrB,MAAM,aAAa,MAAM,KAAK,YAAY,YAAY,WAAW;AACjE,eAAY,KAAK,GAAG,WAAW;;AAEjC,SAAO;;CAGT,MAAM,UAAU,MAAiC;AAC/C,SAAO,KAAK,WAAW,MAAM,KAAK,WAAW;;CAG/C,OAAO,aACL,UACA,SACA;AACA,SAAO,IAAI,+BAA+B;GACxC,GAAG;GACH,YACE,+BAA+B,yBAAyB,SAAS;GACpE,CAAC;;CAGJ,OAAO,yBAAyB,UAAyC;AACvE,MAAI,aAAa,MACf,QAAO;GAEL;GAEA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,KACtB,QAAO;GAEL;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,OACtB,QAAO;GAEL;GAEA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,KACtB,QAAO;GAEL;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,MACtB,QAAO;GAEL;GAEA;GAEA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,QACtB,QAAO;GAEL;GAEA;GAEA;GAEA;GAEA;GAEA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,SACtB,QAAO;GAEL;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,MACtB,QAAO;GAEL;GACA;GACA;GAEA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,OACtB,QAAO;GAEL;GACA;GAEA;GACA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,OACtB,QAAO;GAEL;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,QACtB,QAAO;GAEL;GACA;GAEA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,QACtB,QAAO;GAEL;GAEA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;WACQ,aAAa,WACtB,QAAO;GAEL;GACA;GACA;GACA;GACA;GAKA;GAEA;GACA;GACA;GAGA;GACA;GACA;GACA;GACD;WACQ,aAAa,QACtB,QAAO;GAEL;GACA;GACA;GACA;GAGA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GAGA;GACA;GACA;GAGA;GACA;GACA;GACA;GACD;WACQ,aAAa,OACtB,QAAO;GAEL;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GAEA;GACA;GACD;WACQ,aAAa,MACtB,QAAO;GAEL;GACA;GAEA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACA;GAEA;GACA;GACA;GACA;GACD;MAED,OAAM,IAAI,MAAM,YAAY,SAAS,oBAAoB;;;;;;AAc/D,IAAa,oBAAb,cACU,aAEV;CACE,OAAO,UAAU;AACf,SAAO;;CAGT;CAEA;CAEA;CAEA;CAEA,YAAY,QAA2C;AACrD,QAAM,OAAO;AAEb,OAAK,eAAe,QAAQ,gBAAgB;AAC5C,OAAK,iBAAiB,QAAQ,kBAAkB,EAAE;AAClD,OAAK,oBAAoB,QAAQ,qBAAqB;;CAGxD,MAAM,UAAU,MAAiC;AAC/C,MAAI,CAAC,KAAK,UACR,MAAK,YAAY,OAAA,GAAA,+BAAA,aAAkB,KAAK,aAAa;EAGvD,MAAM,SAAmB,EAAE;EAE3B,MAAM,YAAY,KAAK,UAAU,OAC/B,MACA,KAAK,gBACL,KAAK,kBACN;EAED,IAAI,YAAY;AAEhB,SAAO,YAAY,UAAU,QAAQ;AACnC,OAAI,YAAY,EACd,cAAa,KAAK;GAEpB,MAAM,UAAU,KAAK,IAAI,YAAY,KAAK,WAAW,UAAU,OAAO;GACtE,MAAM,YAAY,UAAU,MAAM,WAAW,QAAQ;AACrD,UAAO,KAAK,KAAK,UAAU,OAAO,UAAU,CAAC;AAC7C,eAAY;;AAGd,SAAO;;;AAMX,IAAa,uBAAb,cACU,+BAEV;CACE,YAAY,QAA8C;AACxD,QAAM;GACJ,GAAG;GACH,YACE,+BAA+B,yBAAyB,WAAW;GACtE,CAAC;;;AAMN,IAAa,oBAAb,cACU,+BAEV;CACE,YAAY,QAA2C;AACrD,QAAM;GACJ,GAAG;GACH,YACE,+BAA+B,yBAAyB,QAAQ;GACnE,CAAC"}

package/dist/text_splitter.d.cts

@@ -0,0 +1,82 @@
+import * as tiktoken from "js-tiktoken";
+import { BaseDocumentTransformer, Document } from "@langchain/core/documents";
+
+//#region src/text_splitter.d.ts
+interface TextSplitterParams {
+  chunkSize: number;
+  chunkOverlap: number;
+  keepSeparator: boolean;
+  lengthFunction?: ((text: string) => number) | ((text: string) => Promise<number>);
+}
+type TextSplitterChunkHeaderOptions = {
+  chunkHeader?: string;
+  chunkOverlapHeader?: string;
+  appendChunkOverlapHeader?: boolean;
+};
+declare abstract class TextSplitter extends BaseDocumentTransformer implements TextSplitterParams {
+  lc_namespace: string[];
+  chunkSize: number;
+  chunkOverlap: number;
+  keepSeparator: boolean;
+  lengthFunction: ((text: string) => number) | ((text: string) => Promise<number>);
+  constructor(fields?: Partial<TextSplitterParams>);
+  transformDocuments(documents: Document[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
+  abstract splitText(text: string): Promise<string[]>;
+  protected splitOnSeparator(text: string, separator: string): string[];
+  createDocuments(texts: string[], metadatas?: Record<string, any>[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
+  private numberOfNewLines;
+  splitDocuments(documents: Document[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
+  private joinDocs;
+  mergeSplits(splits: string[], separator: string): Promise<string[]>;
+}
+interface CharacterTextSplitterParams extends TextSplitterParams {
+  separator: string;
+}
+declare class CharacterTextSplitter extends TextSplitter implements CharacterTextSplitterParams {
+  static lc_name(): string;
+  separator: string;
+  constructor(fields?: Partial<CharacterTextSplitterParams>);
+  splitText(text: string): Promise<string[]>;
+}
+interface RecursiveCharacterTextSplitterParams extends TextSplitterParams {
+  separators: string[];
+}
+declare const SupportedTextSplitterLanguages: readonly ["cpp", "go", "java", "js", "php", "proto", "python", "rst", "ruby", "rust", "scala", "swift", "markdown", "latex", "html", "sol"];
+type SupportedTextSplitterLanguage = (typeof SupportedTextSplitterLanguages)[number];
+declare class RecursiveCharacterTextSplitter extends TextSplitter implements RecursiveCharacterTextSplitterParams {
+  static lc_name(): string;
+  separators: string[];
+  constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>);
+  private _splitText;
+  splitText(text: string): Promise<string[]>;
+  static fromLanguage(language: SupportedTextSplitterLanguage, options?: Partial<RecursiveCharacterTextSplitterParams>): RecursiveCharacterTextSplitter;
+  static getSeparatorsForLanguage(language: SupportedTextSplitterLanguage): string[];
+}
+interface TokenTextSplitterParams extends TextSplitterParams {
+  encodingName: tiktoken.TiktokenEncoding;
+  allowedSpecial: "all" | Array<string>;
+  disallowedSpecial: "all" | Array<string>;
+}
+/**
+ * Implementation of splitter which looks at tokens.
+ */
+declare class TokenTextSplitter extends TextSplitter implements TokenTextSplitterParams {
+  static lc_name(): string;
+  encodingName: tiktoken.TiktokenEncoding;
+  allowedSpecial: "all" | Array<string>;
+  disallowedSpecial: "all" | Array<string>;
+  private tokenizer;
+  constructor(fields?: Partial<TokenTextSplitterParams>);
+  splitText(text: string): Promise<string[]>;
+}
+type MarkdownTextSplitterParams = TextSplitterParams;
+declare class MarkdownTextSplitter extends RecursiveCharacterTextSplitter implements MarkdownTextSplitterParams {
+  constructor(fields?: Partial<MarkdownTextSplitterParams>);
+}
+type LatexTextSplitterParams = TextSplitterParams;
+declare class LatexTextSplitter extends RecursiveCharacterTextSplitter implements LatexTextSplitterParams {
+  constructor(fields?: Partial<LatexTextSplitterParams>);
+}
+//#endregion
+export { CharacterTextSplitter, CharacterTextSplitterParams, LatexTextSplitter, LatexTextSplitterParams, MarkdownTextSplitter, MarkdownTextSplitterParams, RecursiveCharacterTextSplitter, RecursiveCharacterTextSplitterParams, SupportedTextSplitterLanguage, SupportedTextSplitterLanguages, TextSplitter, TextSplitterChunkHeaderOptions, TextSplitterParams, TokenTextSplitter, TokenTextSplitterParams };
+//# sourceMappingURL=text_splitter.d.cts.map