@dakshp1234/langchain-textsplitters 1.0.1

package/CHANGELOG.md

@@ -0,0 +1,11 @@
+# @langchain/textsplitters
+
+## 1.0.1
+
+### Patch Changes
+
+- [#9416](https://github.com/langchain-ai/langchainjs/pull/9416) [`0fe9beb`](https://github.com/langchain-ai/langchainjs/commit/0fe9bebee6710f719e47f913eec1ec4f638e4de4) Thanks [@hntrl](https://github.com/hntrl)! - fix 'moduleResultion: "node"' compatibility
+
+## 1.0.0
+
+This release updates the package for compatibility with LangChain v1.0. See the v1.0 [release notes](https://docs.langchain.com/oss/javascript/releases/langchain-v1) for details on what's new.

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,53 @@
+# 🦜✂️ @langchain/textsplitters
+
+This package contains various implementations of LangChain.js text splitters, most commonly used as part of retrieval-augmented generation (RAG) pipelines.
+
+## Installation
+
+```bash npm2yarn
+npm install @langchain/textsplitters @langchain/core
+```
+
+## Development
+
+To develop the `@langchain/textsplitters` package, you'll need to follow these instructions:
+
+### Install dependencies
+
+```bash
+pnpm install
+```
+
+### Build the package
+
+```bash
+pnpm build
+```
+
+Or from the repo root:
+
+```bash
+pnpm build --filter @langchain/textsplitters
+```
+
+### Run tests
+
+Test files should live within a `tests/` file in the `src/` folder. Unit tests should end in `.test.ts` and integration tests should
+end in `.int.test.ts`:
+
+```bash
+$ pnpm test
+$ pnpm test:int
+```
+
+### Lint & Format
+
+Run the linter & formatter to ensure your code is up to standard:
+
+```bash
+pnpm lint && pnpm format
+```
+
+### Adding new entrypoints
+
+If you add a new file to be exported, either import & re-export from `src/index.ts`, or add it to the `exports` field in the `package.json` file and run `pnpm build` to generate the new entrypoint.

package/dist/index.cjs

@@ -0,0 +1,12 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_text_splitter = require("./text_splitter.cjs");
+const require_semantic_text_splitter = require("./semantic_text_splitter.cjs");
+exports.CharacterTextSplitter = require_text_splitter.CharacterTextSplitter;
+exports.LatexTextSplitter = require_text_splitter.LatexTextSplitter;
+exports.MarkdownTextSplitter = require_text_splitter.MarkdownTextSplitter;
+exports.RecursiveCharacterTextSplitter = require_text_splitter.RecursiveCharacterTextSplitter;
+exports.SemanticTextSplitter = require_semantic_text_splitter.SemanticTextSplitter;
+exports.SupportedTextSplitterLanguages = require_text_splitter.SupportedTextSplitterLanguages;
+exports.TextSplitter = require_text_splitter.TextSplitter;
+exports.TokenTextSplitter = require_text_splitter.TokenTextSplitter;
+exports.calculateCosineDistances = require_semantic_text_splitter.calculateCosineDistances;

package/dist/index.d.cts

@@ -0,0 +1,3 @@
+import { CharacterTextSplitter, CharacterTextSplitterParams, LatexTextSplitter, LatexTextSplitterParams, MarkdownTextSplitter, MarkdownTextSplitterParams, RecursiveCharacterTextSplitter, RecursiveCharacterTextSplitterParams, SupportedTextSplitterLanguage, SupportedTextSplitterLanguages, TextSplitter, TextSplitterChunkHeaderOptions, TextSplitterParams, TokenTextSplitter, TokenTextSplitterParams } from "./text_splitter.cjs";
+import { SemanticTextSplitter, SemanticTextSplitterParams, calculateCosineDistances } from "./semantic_text_splitter.cjs";
+export { CharacterTextSplitter, CharacterTextSplitterParams, LatexTextSplitter, LatexTextSplitterParams, MarkdownTextSplitter, MarkdownTextSplitterParams, RecursiveCharacterTextSplitter, RecursiveCharacterTextSplitterParams, SemanticTextSplitter, SemanticTextSplitterParams, SupportedTextSplitterLanguage, SupportedTextSplitterLanguages, TextSplitter, TextSplitterChunkHeaderOptions, TextSplitterParams, TokenTextSplitter, TokenTextSplitterParams, calculateCosineDistances };

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+import { CharacterTextSplitter, CharacterTextSplitterParams, LatexTextSplitter, LatexTextSplitterParams, MarkdownTextSplitter, MarkdownTextSplitterParams, RecursiveCharacterTextSplitter, RecursiveCharacterTextSplitterParams, SupportedTextSplitterLanguage, SupportedTextSplitterLanguages, TextSplitter, TextSplitterChunkHeaderOptions, TextSplitterParams, TokenTextSplitter, TokenTextSplitterParams } from "./text_splitter.js";
+import { SemanticTextSplitter, SemanticTextSplitterParams, calculateCosineDistances } from "./semantic_text_splitter.js";
+export { CharacterTextSplitter, CharacterTextSplitterParams, LatexTextSplitter, LatexTextSplitterParams, MarkdownTextSplitter, MarkdownTextSplitterParams, RecursiveCharacterTextSplitter, RecursiveCharacterTextSplitterParams, SemanticTextSplitter, SemanticTextSplitterParams, SupportedTextSplitterLanguage, SupportedTextSplitterLanguages, TextSplitter, TextSplitterChunkHeaderOptions, TextSplitterParams, TokenTextSplitter, TokenTextSplitterParams, calculateCosineDistances };

package/dist/index.js

@@ -0,0 +1,3 @@
+import { CharacterTextSplitter, LatexTextSplitter, MarkdownTextSplitter, RecursiveCharacterTextSplitter, SupportedTextSplitterLanguages, TextSplitter, TokenTextSplitter } from "./text_splitter.js";
+import { SemanticTextSplitter, calculateCosineDistances } from "./semantic_text_splitter.js";
+export { CharacterTextSplitter, LatexTextSplitter, MarkdownTextSplitter, RecursiveCharacterTextSplitter, SemanticTextSplitter, SupportedTextSplitterLanguages, TextSplitter, TokenTextSplitter, calculateCosineDistances };

package/dist/semantic_text_splitter.cjs

@@ -0,0 +1,179 @@
+let _langchain_core_documents = require("@langchain/core/documents");
+//#region src/semantic_text_splitter.ts
+/**
+* Experimental semantic text splitter based on semantic similarity.
+* 
+* Inspired by Greg Kamradt's semantic chunking approach:
+* https://github.com/FullStackRetrieval-com/RetrievalTutorials/blob/main/tutorials/LevelsOfTextSplitting/5_Levels_Of_Text_Splitting.ipynb
+*/
+function combineSentences(sentences, bufferSize = 1) {
+	for (let i = 0; i < sentences.length; i++) {
+		const sentence = sentences[i];
+		let combinedSentence = "";
+		for (let j = Math.max(0, i - bufferSize); j < i; j++) combinedSentence += sentences[j].sentence + " ";
+		combinedSentence += sentence.sentence;
+		for (let j = i + 1; j <= Math.min(sentences.length - 1, i + bufferSize); j++) combinedSentence += " " + sentences[j].sentence;
+		sentence.combinedSentence = combinedSentence.trim();
+	}
+	return sentences;
+}
+function cosineSimilarity(a, b) {
+	const dotProduct = a.reduce((sum, ai, i) => sum + ai * b[i], 0);
+	const magnitudeA = Math.sqrt(a.reduce((sum, ai) => sum + ai * ai, 0));
+	const magnitudeB = Math.sqrt(b.reduce((sum, bi) => sum + bi * bi, 0));
+	if (magnitudeA === 0 || magnitudeB === 0) return 0;
+	return dotProduct / (magnitudeA * magnitudeB);
+}
+function calculateCosineDistances(sentences) {
+	const distances = [];
+	for (let i = 0; i < sentences.length - 1; i++) {
+		const embeddingCurrent = sentences[i].combinedSentenceEmbedding;
+		const embeddingNext = sentences[i + 1].combinedSentenceEmbedding;
+		const distance = 1 - cosineSimilarity(embeddingCurrent, embeddingNext);
+		distances.push(distance);
+	}
+	return [distances, sentences];
+}
+const BREAKPOINT_DEFAULTS = {
+	percentile: 95,
+	standard_deviation: 3,
+	interquartile: 1.5,
+	gradient: 95
+};
+var SemanticTextSplitter = class extends _langchain_core_documents.BaseDocumentTransformer {
+	embeddings;
+	bufferSize;
+	addStartIndex;
+	breakpointThresholdType;
+	breakpointThresholdAmount;
+	numberOfChunks;
+	sentenceSplitRegex;
+	minChunkSize;
+	constructor(params) {
+		super();
+		this.embeddings = params.embeddings;
+		this.bufferSize = params.bufferSize ?? 1;
+		this.addStartIndex = params.addStartIndex ?? false;
+		this.breakpointThresholdType = params.breakpointThresholdType ?? "percentile";
+		this.numberOfChunks = params.numberOfChunks;
+		this.sentenceSplitRegex = params.sentenceSplitRegex ?? "(?<=[.?!])\\s+";
+		this.minChunkSize = params.minChunkSize;
+		if (params.breakpointThresholdAmount === void 0) this.breakpointThresholdAmount = BREAKPOINT_DEFAULTS[this.breakpointThresholdType];
+		else this.breakpointThresholdAmount = params.breakpointThresholdAmount;
+	}
+	calculateBreakpointThreshold(distances) {
+		if (this.breakpointThresholdType === "percentile") {
+			const sorted = [...distances].sort((a, b) => a - b);
+			const index = Math.ceil(this.breakpointThresholdAmount / 100 * sorted.length) - 1;
+			return [sorted[Math.max(0, index)], distances];
+		} else if (this.breakpointThresholdType === "standard_deviation") {
+			const mean = distances.reduce((sum, d) => sum + d, 0) / distances.length;
+			const variance = distances.reduce((sum, d) => sum + Math.pow(d - mean, 2), 0) / distances.length;
+			const stdDev = Math.sqrt(variance);
+			return [mean + this.breakpointThresholdAmount * stdDev, distances];
+		} else if (this.breakpointThresholdType === "interquartile") {
+			const sorted = [...distances].sort((a, b) => a - b);
+			const q1Index = Math.floor(.25 * sorted.length);
+			const q3Index = Math.floor(.75 * sorted.length);
+			const q1 = sorted[q1Index];
+			const iqr = sorted[q3Index] - q1;
+			return [distances.reduce((sum, d) => sum + d, 0) / distances.length + this.breakpointThresholdAmount * iqr, distances];
+		} else if (this.breakpointThresholdType === "gradient") {
+			const distanceGradient = [];
+			for (let i = 0; i < distances.length - 1; i++) distanceGradient.push(distances[i + 1] - distances[i]);
+			const sortedGradient = [...distanceGradient].sort((a, b) => a - b);
+			const index = Math.ceil(this.breakpointThresholdAmount / 100 * sortedGradient.length) - 1;
+			return [sortedGradient[Math.max(0, index)], distanceGradient];
+		} else throw new Error(`Got unexpected breakpointThresholdType: ${this.breakpointThresholdType}`);
+	}
+	thresholdFromClusters(distances) {
+		if (this.numberOfChunks === void 0) throw new Error("This should never be called if numberOfChunks is undefined.");
+		const x1 = distances.length;
+		const y1 = 0;
+		const x2 = 1;
+		const y2 = 100;
+		const x = Math.max(Math.min(this.numberOfChunks, x1), x2);
+		let y;
+		if (x2 === x1) y = y2;
+		else y = y1 + (y2 - y1) / (x2 - x1) * (x - x1);
+		y = Math.min(Math.max(y, 0), 100);
+		const sorted = [...distances].sort((a, b) => a - b);
+		const index = Math.ceil(y / 100 * sorted.length) - 1;
+		return sorted[Math.max(0, index)];
+	}
+	async calculateSentenceDistances(singleSentencesList) {
+		const sentencesWithCombined = combineSentences(singleSentencesList.map((sentence, index) => ({
+			sentence,
+			index
+		})), this.bufferSize);
+		const combinedSentences = sentencesWithCombined.map((s) => s.combinedSentence);
+		const embeddings = await this.embeddings.embedDocuments(combinedSentences);
+		for (let i = 0; i < sentencesWithCombined.length; i++) sentencesWithCombined[i].combinedSentenceEmbedding = embeddings[i];
+		return calculateCosineDistances(sentencesWithCombined);
+	}
+	getSingleSentencesList(text) {
+		return text.split(new RegExp(this.sentenceSplitRegex)).filter((sentence) => sentence.trim().length > 0);
+	}
+	async splitText(text) {
+		const singleSentencesList = this.getSingleSentencesList(text);
+		if (singleSentencesList.length === 1) return singleSentencesList;
+		if (this.breakpointThresholdType === "gradient" && singleSentencesList.length === 2) return singleSentencesList;
+		const [distances, sentences] = await this.calculateSentenceDistances(singleSentencesList);
+		let breakpointDistanceThreshold;
+		let breakpointArray;
+		if (this.numberOfChunks !== void 0) {
+			breakpointDistanceThreshold = this.thresholdFromClusters(distances);
+			breakpointArray = distances;
+		} else [breakpointDistanceThreshold, breakpointArray] = this.calculateBreakpointThreshold(distances);
+		const indicesAboveThresh = breakpointArray.map((x, i) => ({
+			value: x,
+			index: i
+		})).filter(({ value }) => value > breakpointDistanceThreshold).map(({ index }) => index);
+		const chunks = [];
+		let startIndex = 0;
+		for (const index of indicesAboveThresh) {
+			const endIndex = index;
+			const combinedText = sentences.slice(startIndex, endIndex + 1).map((d) => d.sentence).join(" ");
+			if (this.minChunkSize !== void 0 && combinedText.length < this.minChunkSize) continue;
+			chunks.push(combinedText);
+			startIndex = index + 1;
+		}
+		if (startIndex < sentences.length) {
+			const combinedText = sentences.slice(startIndex).map((d) => d.sentence).join(" ");
+			chunks.push(combinedText);
+		}
+		return chunks;
+	}
+	async createDocuments(texts, metadatas = []) {
+		const _metadatas = metadatas.length > 0 ? metadatas : texts.map(() => ({}));
+		const documents = [];
+		for (let i = 0; i < texts.length; i++) {
+			const text = texts[i];
+			let startIndex = 0;
+			for (const chunk of await this.splitText(text)) {
+				const metadata = { ..._metadatas[i] };
+				if (this.addStartIndex) metadata.start_index = startIndex;
+				const newDoc = new _langchain_core_documents.Document({
+					pageContent: chunk,
+					metadata
+				});
+				documents.push(newDoc);
+				startIndex += chunk.length;
+			}
+		}
+		return documents;
+	}
+	async splitDocuments(documents) {
+		const texts = documents.map((doc) => doc.pageContent);
+		const metadatas = documents.map((doc) => doc.metadata);
+		return this.createDocuments(texts, metadatas);
+	}
+	async transformDocuments(documents) {
+		return this.splitDocuments(documents);
+	}
+};
+//#endregion
+exports.SemanticTextSplitter = SemanticTextSplitter;
+exports.calculateCosineDistances = calculateCosineDistances;
+
+//# sourceMappingURL=semantic_text_splitter.cjs.map

package/dist/semantic_text_splitter.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic_text_splitter.cjs","names":["BaseDocumentTransformer","Document"],"sources":["../src/semantic_text_splitter.ts"],"sourcesContent":["/**\r\n * Experimental semantic text splitter based on semantic similarity.\r\n * \r\n * Inspired by Greg Kamradt's semantic chunking approach:\r\n * https://github.com/FullStackRetrieval-com/RetrievalTutorials/blob/main/tutorials/LevelsOfTextSplitting/5_Levels_Of_Text_Splitting.ipynb\r\n */\r\nimport { Document, BaseDocumentTransformer } from \"@langchain/core/documents\";\r\nimport { Embeddings } from \"@langchain/core/embeddings\";\r\n\r\ninterface SentenceDict {\r\n  sentence: string;\r\n  index: number;\r\n  combinedSentence?: string;\r\n  combinedSentenceEmbedding?: number[];\r\n}\r\n\r\nfunction combineSentences(sentences: SentenceDict[], bufferSize: number = 1): SentenceDict[] {\r\n  // Go through each sentence dict\r\n  for (let i = 0; i < sentences.length; i++) {\r\n    const sentence = sentences[i];\r\n    \r\n    // Create the combined sentence by combining the sentences within the buffer\r\n    let combinedSentence = \"\";\r\n    \r\n    // Add sentences before the current one\r\n    for (let j = Math.max(0, i - bufferSize); j < i; j++) {\r\n      combinedSentence += sentences[j].sentence + \" \";\r\n    }\r\n    \r\n    // Add the current sentence\r\n    combinedSentence += sentence.sentence;\r\n    \r\n    // Add sentences after the current one\r\n    for (let j = i + 1; j <= Math.min(sentences.length - 1, i + bufferSize); j++) {\r\n      combinedSentence += \" \" + sentences[j].sentence;\r\n    }\r\n    \r\n    // Assign the combined sentence to the dict\r\n    sentence.combinedSentence = combinedSentence.trim();\r\n  }\r\n  \r\n  return sentences;\r\n}\r\n\r\nfunction cosineSimilarity(a: number[], b: number[]): number {\r\n  const dotProduct = a.reduce((sum, ai, i) => sum + ai * b[i], 0);\r\n  const magnitudeA = Math.sqrt(a.reduce((sum, ai) => sum + ai * ai, 0));\r\n  const magnitudeB = Math.sqrt(b.reduce((sum, bi) => sum + bi * bi, 0));\r\n  \r\n  if (magnitudeA === 0 || magnitudeB === 0) {\r\n    return 0;\r\n  }\r\n  \r\n  return dotProduct / (magnitudeA * magnitudeB);\r\n}\r\n\r\nexport function calculateCosineDistances(sentences: SentenceDict[]): [number[], SentenceDict[]] {\r\n  const distances: number[] = [];\r\n  \r\n  for (let i = 0; i < sentences.length - 1; i++) {\r\n    const embeddingCurrent = sentences[i].combinedSentenceEmbedding!;\r\n    const embeddingNext = sentences[i + 1].combinedSentenceEmbedding!;\r\n    \r\n    // Calculate cosine similarity\r\n    const similarity = cosineSimilarity(embeddingCurrent, embeddingNext);\r\n    \r\n    // Convert to cosine distance (1 - cosine similarity)\r\n    const distance = 1 - similarity;\r\n    distances.push(distance);\r\n  }\r\n  \r\n  return [distances, sentences];\r\n}\r\n\r\ntype BreakpointThresholdType = \"percentile\" | \"standard_deviation\" | \"interquartile\" | \"gradient\";\r\n\r\nconst BREAKPOINT_DEFAULTS: Record<BreakpointThresholdType, number> = {\r\n  percentile: 95,\r\n  standard_deviation: 3,\r\n  interquartile: 1.5,\r\n  gradient: 95,\r\n};\r\n\r\nexport interface SemanticTextSplitterParams {\r\n  embeddings: Embeddings;\r\n  bufferSize?: number;\r\n  addStartIndex?: boolean;\r\n  breakpointThresholdType?: BreakpointThresholdType;\r\n  breakpointThresholdAmount?: number;\r\n  numberOfChunks?: number;\r\n  sentenceSplitRegex?: string;\r\n  minChunkSize?: number;\r\n}\r\n\r\nexport class SemanticTextSplitter extends BaseDocumentTransformer {\r\n  private embeddings: Embeddings;\r\n  private bufferSize: number;\r\n  private addStartIndex: boolean;\r\n  private breakpointThresholdType: BreakpointThresholdType;\r\n  private breakpointThresholdAmount: number;\r\n  private numberOfChunks?: number;\r\n  private sentenceSplitRegex: string;\r\n  private minChunkSize?: number;\r\n\r\n  constructor(params: SemanticTextSplitterParams) {\r\n    super();\r\n    this.embeddings = params.embeddings;\r\n    this.bufferSize = params.bufferSize ?? 1;\r\n    this.addStartIndex = params.addStartIndex ?? false;\r\n    this.breakpointThresholdType = params.breakpointThresholdType ?? \"percentile\";\r\n    this.numberOfChunks = params.numberOfChunks;\r\n    this.sentenceSplitRegex = params.sentenceSplitRegex ?? \"(?<=[.?!])\\\\s+\";\r\n    this.minChunkSize = params.minChunkSize;\r\n    \r\n    if (params.breakpointThresholdAmount === undefined) {\r\n      this.breakpointThresholdAmount = BREAKPOINT_DEFAULTS[this.breakpointThresholdType];\r\n    } else {\r\n      this.breakpointThresholdAmount = params.breakpointThresholdAmount;\r\n    }\r\n  }\r\n\r\n  private calculateBreakpointThreshold(distances: number[]): [number, number[]] {\r\n    if (this.breakpointThresholdType === \"percentile\") {\r\n      const sorted = [...distances].sort((a, b) => a - b);\r\n      const index = Math.ceil((this.breakpointThresholdAmount / 100) * sorted.length) - 1;\r\n      return [sorted[Math.max(0, index)], distances];\r\n    } else if (this.breakpointThresholdType === \"standard_deviation\") {\r\n      const mean = distances.reduce((sum, d) => sum + d, 0) / distances.length;\r\n      const variance = distances.reduce((sum, d) => sum + Math.pow(d - mean, 2), 0) / distances.length;\r\n      const stdDev = Math.sqrt(variance);\r\n      return [mean + this.breakpointThresholdAmount * stdDev, distances];\r\n    } else if (this.breakpointThresholdType === \"interquartile\") {\r\n      const sorted = [...distances].sort((a, b) => a - b);\r\n      const q1Index = Math.floor(0.25 * sorted.length);\r\n      const q3Index = Math.floor(0.75 * sorted.length);\r\n      const q1 = sorted[q1Index];\r\n      const q3 = sorted[q3Index];\r\n      const iqr = q3 - q1;\r\n      const mean = distances.reduce((sum, d) => sum + d, 0) / distances.length;\r\n      return [mean + this.breakpointThresholdAmount * iqr, distances];\r\n    } else if (this.breakpointThresholdType === \"gradient\") {\r\n      const distanceGradient: number[] = [];\r\n      for (let i = 0; i < distances.length - 1; i++) {\r\n        distanceGradient.push(distances[i + 1] - distances[i]);\r\n      }\r\n      const sortedGradient = [...distanceGradient].sort((a, b) => a - b);\r\n      const index = Math.ceil((this.breakpointThresholdAmount / 100) * sortedGradient.length) - 1;\r\n      return [sortedGradient[Math.max(0, index)], distanceGradient];\r\n    } else {\r\n      throw new Error(`Got unexpected breakpointThresholdType: ${this.breakpointThresholdType}`);\r\n    }\r\n  }\r\n\r\n  private thresholdFromClusters(distances: number[]): number {\r\n    if (this.numberOfChunks === undefined) {\r\n      throw new Error(\"This should never be called if numberOfChunks is undefined.\");\r\n    }\r\n    \r\n    const x1 = distances.length;\r\n    const y1 = 0.0;\r\n    const x2 = 1.0;\r\n    const y2 = 100.0;\r\n    \r\n    const x = Math.max(Math.min(this.numberOfChunks, x1), x2);\r\n    \r\n    let y: number;\r\n    if (x2 === x1) {\r\n      y = y2;\r\n    } else {\r\n      y = y1 + ((y2 - y1) / (x2 - x1)) * (x - x1);\r\n    }\r\n    \r\n    y = Math.min(Math.max(y, 0), 100);\r\n    \r\n    const sorted = [...distances].sort((a, b) => a - b);\r\n    const index = Math.ceil((y / 100) * sorted.length) - 1;\r\n    return sorted[Math.max(0, index)];\r\n  }\r\n\r\n  private async calculateSentenceDistances(singleSentencesList: string[]): Promise<[number[], SentenceDict[]]> {\r\n    const sentences: SentenceDict[] = singleSentencesList.map((sentence, index) => ({\r\n      sentence,\r\n      index,\r\n    }));\r\n\r\n    const sentencesWithCombined = combineSentences(sentences, this.bufferSize);\r\n    \r\n    const combinedSentences = sentencesWithCombined.map(s => s.combinedSentence!);\r\n    const embeddings = await this.embeddings.embedDocuments(combinedSentences);\r\n    \r\n    for (let i = 0; i < sentencesWithCombined.length; i++) {\r\n      sentencesWithCombined[i].combinedSentenceEmbedding = embeddings[i];\r\n    }\r\n\r\n    return calculateCosineDistances(sentencesWithCombined);\r\n  }\r\n\r\n  private getSingleSentencesList(text: string): string[] {\r\n    return text.split(new RegExp(this.sentenceSplitRegex)).filter(sentence => sentence.trim().length > 0);\r\n  }\r\n\r\n  async splitText(text: string): Promise<string[]> {\r\n    const singleSentencesList = this.getSingleSentencesList(text);\r\n\r\n    if (singleSentencesList.length === 1) {\r\n      return singleSentencesList;\r\n    }\r\n\r\n    if (this.breakpointThresholdType === \"gradient\" && singleSentencesList.length === 2) {\r\n      return singleSentencesList;\r\n    }\r\n\r\n    const [distances, sentences] = await this.calculateSentenceDistances(singleSentencesList);\r\n\r\n    let breakpointDistanceThreshold: number;\r\n    let breakpointArray: number[];\r\n\r\n    if (this.numberOfChunks !== undefined) {\r\n      breakpointDistanceThreshold = this.thresholdFromClusters(distances);\r\n      breakpointArray = distances;\r\n    } else {\r\n      [breakpointDistanceThreshold, breakpointArray] = this.calculateBreakpointThreshold(distances);\r\n    }\r\n\r\n    const indicesAboveThresh = breakpointArray\r\n      .map((x, i) => ({ value: x, index: i }))\r\n      .filter(({ value }) => value > breakpointDistanceThreshold)\r\n      .map(({ index }) => index);\r\n\r\n    const chunks: string[] = [];\r\n    let startIndex = 0;\r\n\r\n    for (const index of indicesAboveThresh) {\r\n      const endIndex = index;\r\n      const group = sentences.slice(startIndex, endIndex + 1);\r\n      const combinedText = group.map((d: SentenceDict) => d.sentence).join(\" \");\r\n\r\n      if (this.minChunkSize !== undefined && combinedText.length < this.minChunkSize) {\r\n        continue;\r\n      }\r\n      \r\n      chunks.push(combinedText);\r\n      startIndex = index + 1;\r\n    }\r\n\r\n    if (startIndex < sentences.length) {\r\n      const combinedText = sentences.slice(startIndex).map((d: SentenceDict) => d.sentence).join(\" \");\r\n      chunks.push(combinedText);\r\n    }\r\n\r\n    return chunks;\r\n  }\r\n\r\n  async createDocuments(\r\n    texts: string[],\r\n    metadatas: Record<string, any>[] = []\r\n  ): Promise<Document[]> {\r\n    const _metadatas = metadatas.length > 0 ? metadatas : texts.map(() => (<Record<string, any>>{}));\r\n    const documents: Document[] = [];\r\n    \r\n    for (let i = 0; i < texts.length; i++) {\r\n      const text = texts[i];\r\n      let startIndex = 0;\r\n      \r\n      for (const chunk of await this.splitText(text)) {\r\n        const metadata = { ..._metadatas[i] };\r\n        if (this.addStartIndex) {\r\n          metadata.start_index = startIndex;\r\n        }\r\n        \r\n        const newDoc = new Document({\r\n          pageContent: chunk,\r\n          metadata,\r\n        });\r\n        \r\n        documents.push(newDoc);\r\n        startIndex += chunk.length;\r\n      }\r\n    }\r\n    \r\n    return documents;\r\n  }\r\n\r\n  async splitDocuments(documents: Document[]): Promise<Document[]> {\r\n    const texts = documents.map(doc => doc.pageContent);\r\n    const metadatas = documents.map(doc => doc.metadata);\r\n    return this.createDocuments(texts, metadatas);\r\n  }\r\n\r\n  async transformDocuments(documents: Document[]): Promise<Document[]> {\r\n    return this.splitDocuments(documents);\r\n  }\r\n}"],"mappings":";;;;;;;;AAgBA,SAAS,iBAAiB,WAA2B,aAAqB,GAAmB;AAE3F,MAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;EACzC,MAAM,WAAW,UAAU;EAG3B,IAAI,mBAAmB;AAGvB,OAAK,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,WAAW,EAAE,IAAI,GAAG,IAC/C,qBAAoB,UAAU,GAAG,WAAW;AAI9C,sBAAoB,SAAS;AAG7B,OAAK,IAAI,IAAI,IAAI,GAAG,KAAK,KAAK,IAAI,UAAU,SAAS,GAAG,IAAI,WAAW,EAAE,IACvE,qBAAoB,MAAM,UAAU,GAAG;AAIzC,WAAS,mBAAmB,iBAAiB,MAAM;;AAGrD,QAAO;;AAGT,SAAS,iBAAiB,GAAa,GAAqB;CAC1D,MAAM,aAAa,EAAE,QAAQ,KAAK,IAAI,MAAM,MAAM,KAAK,EAAE,IAAI,EAAE;CAC/D,MAAM,aAAa,KAAK,KAAK,EAAE,QAAQ,KAAK,OAAO,MAAM,KAAK,IAAI,EAAE,CAAC;CACrE,MAAM,aAAa,KAAK,KAAK,EAAE,QAAQ,KAAK,OAAO,MAAM,KAAK,IAAI,EAAE,CAAC;AAErE,KAAI,eAAe,KAAK,eAAe,EACrC,QAAO;AAGT,QAAO,cAAc,aAAa;;AAGpC,SAAgB,yBAAyB,WAAuD;CAC9F,MAAM,YAAsB,EAAE;AAE9B,MAAK,IAAI,IAAI,GAAG,IAAI,UAAU,SAAS,GAAG,KAAK;EAC7C,MAAM,mBAAmB,UAAU,GAAG;EACtC,MAAM,gBAAgB,UAAU,IAAI,GAAG;EAMvC,MAAM,WAAW,IAHE,iBAAiB,kBAAkB,cAAc;AAIpE,YAAU,KAAK,SAAS;;AAG1B,QAAO,CAAC,WAAW,UAAU;;AAK/B,MAAM,sBAA+D;CACnE,YAAY;CACZ,oBAAoB;CACpB,eAAe;CACf,UAAU;CACX;AAaD,IAAa,uBAAb,cAA0CA,0BAAAA,wBAAwB;CAChE;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,QAAoC;AAC9C,SAAO;AACP,OAAK,aAAa,OAAO;AACzB,OAAK,aAAa,OAAO,cAAc;AACvC,OAAK,gBAAgB,OAAO,iBAAiB;AAC7C,OAAK,0BAA0B,OAAO,2BAA2B;AACjE,OAAK,iBAAiB,OAAO;AAC7B,OAAK,qBAAqB,OAAO,sBAAsB;AACvD,OAAK,eAAe,OAAO;AAE3B,MAAI,OAAO,8BAA8B,KAAA,EACvC,MAAK,4BAA4B,oBAAoB,KAAK;MAE1D,MAAK,4BAA4B,OAAO;;CAI5C,6BAAqC,WAAyC;AAC5E,MAAI,KAAK,4BAA4B,cAAc;GACjD,MAAM,SAAS,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,IAAI,EAAE;GACnD,MAAM,QAAQ,KAAK,KAAM,KAAK,4BAA4B,MAAO,OAAO,OAAO,GAAG;AAClF,UAAO,CAAC,OAAO,KAAK,IAAI,GAAG,MAAM,GAAG,UAAU;aACrC,KAAK,4BAA4B,sBAAsB;GAChE,MAAM,OAAO,UAAU,QAAQ,KAAK,MAAM,MAAM,GAAG,EAAE,GAAG,UAAU;GAClE,MAAM,WAAW,UAAU,QAAQ,KAAK,MAAM,MAAM,KAAK,IAAI,IAAI,MAAM,EAAE,EAAE,EAAE,GAAG,UAAU;GAC1F,MAAM,SAAS,KAAK,KAAK,SAAS;AAClC,UAAO,CAAC,OAAO,KAAK,4BAA4B,QAAQ,UAAU;aACzD,KAAK,4BAA4B,iBAAiB;GAC3D,MAAM,SAAS,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,IAAI,EAAE;GACnD,MAAM,UAAU,KAAK,MAAM,MAAO,OAAO,OAAO;GAChD,MAAM,UAAU,KAAK,MAAM,MAAO,OAAO,OAAO;GAChD,MAAM,KAAK,OAAO;GAElB,MAAM,MADK,OAAO,WACD;AAEjB,UAAO,CADM,UAAU,QAAQ,KAAK,MAAM,MAAM,GAAG,EAAE,GAAG,UAAU,SACnD,KAAK,4BAA4B,KAAK,UAAU;aACtD,KAAK,4BAA4B,YAAY;GACtD,MAAM,mBAA6B,EAAE;AACrC,QAAK,IAAI,IAAI,GAAG,IAAI,UAAU,SAAS,GAAG,IACxC,kBAAiB,KAAK,UAAU,IAAI,KAAK,UAAU,GAAG;GAExD,MAAM,iBAAiB,CAAC,GAAG,iBAAiB,CAAC,MAAM,GAAG,MAAM,IAAI,EAAE;GAClE,MAAM,QAAQ,KAAK,KAAM,KAAK,4BAA4B,MAAO,eAAe,OAAO,GAAG;AAC1F,UAAO,CAAC,eAAe,KAAK,IAAI,GAAG,MAAM,GAAG,iBAAiB;QAE7D,OAAM,IAAI,MAAM,2CAA2C,KAAK,0BAA0B;;CAI9F,sBAA8B,WAA6B;AACzD,MAAI,KAAK,mBAAmB,KAAA,EAC1B,OAAM,IAAI,MAAM,8DAA8D;EAGhF,MAAM,KAAK,UAAU;EACrB,MAAM,KAAK;EACX,MAAM,KAAK;EACX,MAAM,KAAK;EAEX,MAAM,IAAI,KAAK,IAAI,KAAK,IAAI,KAAK,gBAAgB,GAAG,EAAE,GAAG;EAEzD,IAAI;AACJ,MAAI,OAAO,GACT,KAAI;MAEJ,KAAI,MAAO,KAAK,OAAO,KAAK,OAAQ,IAAI;AAG1C,MAAI,KAAK,IAAI,KAAK,IAAI,GAAG,EAAE,EAAE,IAAI;EAEjC,MAAM,SAAS,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,IAAI,EAAE;EACnD,MAAM,QAAQ,KAAK,KAAM,IAAI,MAAO,OAAO,OAAO,GAAG;AACrD,SAAO,OAAO,KAAK,IAAI,GAAG,MAAM;;CAGlC,MAAc,2BAA2B,qBAAoE;EAM3G,MAAM,wBAAwB,iBALI,oBAAoB,KAAK,UAAU,WAAW;GAC9E;GACA;GACD,EAAE,EAEuD,KAAK,WAAW;EAE1E,MAAM,oBAAoB,sBAAsB,KAAI,MAAK,EAAE,iBAAkB;EAC7E,MAAM,aAAa,MAAM,KAAK,WAAW,eAAe,kBAAkB;AAE1E,OAAK,IAAI,IAAI,GAAG,IAAI,sBAAsB,QAAQ,IAChD,uBAAsB,GAAG,4BAA4B,WAAW;AAGlE,SAAO,yBAAyB,sBAAsB;;CAGxD,uBAA+B,MAAwB;AACrD,SAAO,KAAK,MAAM,IAAI,OAAO,KAAK,mBAAmB,CAAC,CAAC,QAAO,aAAY,SAAS,MAAM,CAAC,SAAS,EAAE;;CAGvG,MAAM,UAAU,MAAiC;EAC/C,MAAM,sBAAsB,KAAK,uBAAuB,KAAK;AAE7D,MAAI,oBAAoB,WAAW,EACjC,QAAO;AAGT,MAAI,KAAK,4BAA4B,cAAc,oBAAoB,WAAW,EAChF,QAAO;EAGT,MAAM,CAAC,WAAW,aAAa,MAAM,KAAK,2BAA2B,oBAAoB;EAEzF,IAAI;EACJ,IAAI;AAEJ,MAAI,KAAK,mBAAmB,KAAA,GAAW;AACrC,iCAA8B,KAAK,sBAAsB,UAAU;AACnE,qBAAkB;QAElB,EAAC,6BAA6B,mBAAmB,KAAK,6BAA6B,UAAU;EAG/F,MAAM,qBAAqB,gBACxB,KAAK,GAAG,OAAO;GAAE,OAAO;GAAG,OAAO;GAAG,EAAE,CACvC,QAAQ,EAAE,YAAY,QAAQ,4BAA4B,CAC1D,KAAK,EAAE,YAAY,MAAM;EAE5B,MAAM,SAAmB,EAAE;EAC3B,IAAI,aAAa;AAEjB,OAAK,MAAM,SAAS,oBAAoB;GACtC,MAAM,WAAW;GAEjB,MAAM,eADQ,UAAU,MAAM,YAAY,WAAW,EAAE,CAC5B,KAAK,MAAoB,EAAE,SAAS,CAAC,KAAK,IAAI;AAEzE,OAAI,KAAK,iBAAiB,KAAA,KAAa,aAAa,SAAS,KAAK,aAChE;AAGF,UAAO,KAAK,aAAa;AACzB,gBAAa,QAAQ;;AAGvB,MAAI,aAAa,UAAU,QAAQ;GACjC,MAAM,eAAe,UAAU,MAAM,WAAW,CAAC,KAAK,MAAoB,EAAE,SAAS,CAAC,KAAK,IAAI;AAC/F,UAAO,KAAK,aAAa;;AAG3B,SAAO;;CAGT,MAAM,gBACJ,OACA,YAAmC,EAAE,EAChB;EACrB,MAAM,aAAa,UAAU,SAAS,IAAI,YAAY,MAAM,WAAgC,EAAE,EAAE;EAChG,MAAM,YAAwB,EAAE;AAEhC,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;GACrC,MAAM,OAAO,MAAM;GACnB,IAAI,aAAa;AAEjB,QAAK,MAAM,SAAS,MAAM,KAAK,UAAU,KAAK,EAAE;IAC9C,MAAM,WAAW,EAAE,GAAG,WAAW,IAAI;AACrC,QAAI,KAAK,cACP,UAAS,cAAc;IAGzB,MAAM,SAAS,IAAIC,0BAAAA,SAAS;KAC1B,aAAa;KACb;KACD,CAAC;AAEF,cAAU,KAAK,OAAO;AACtB,kBAAc,MAAM;;;AAIxB,SAAO;;CAGT,MAAM,eAAe,WAA4C;EAC/D,MAAM,QAAQ,UAAU,KAAI,QAAO,IAAI,YAAY;EACnD,MAAM,YAAY,UAAU,KAAI,QAAO,IAAI,SAAS;AACpD,SAAO,KAAK,gBAAgB,OAAO,UAAU;;CAG/C,MAAM,mBAAmB,WAA4C;AACnE,SAAO,KAAK,eAAe,UAAU"}

package/dist/semantic_text_splitter.d.cts

@@ -0,0 +1,44 @@
+import { BaseDocumentTransformer, Document } from "@langchain/core/documents";
+import { Embeddings } from "@langchain/core/embeddings";
+
+//#region src/semantic_text_splitter.d.ts
+interface SentenceDict {
+  sentence: string;
+  index: number;
+  combinedSentence?: string;
+  combinedSentenceEmbedding?: number[];
+}
+declare function calculateCosineDistances(sentences: SentenceDict[]): [number[], SentenceDict[]];
+type BreakpointThresholdType = "percentile" | "standard_deviation" | "interquartile" | "gradient";
+interface SemanticTextSplitterParams {
+  embeddings: Embeddings;
+  bufferSize?: number;
+  addStartIndex?: boolean;
+  breakpointThresholdType?: BreakpointThresholdType;
+  breakpointThresholdAmount?: number;
+  numberOfChunks?: number;
+  sentenceSplitRegex?: string;
+  minChunkSize?: number;
+}
+declare class SemanticTextSplitter extends BaseDocumentTransformer {
+  private embeddings;
+  private bufferSize;
+  private addStartIndex;
+  private breakpointThresholdType;
+  private breakpointThresholdAmount;
+  private numberOfChunks?;
+  private sentenceSplitRegex;
+  private minChunkSize?;
+  constructor(params: SemanticTextSplitterParams);
+  private calculateBreakpointThreshold;
+  private thresholdFromClusters;
+  private calculateSentenceDistances;
+  private getSingleSentencesList;
+  splitText(text: string): Promise<string[]>;
+  createDocuments(texts: string[], metadatas?: Record<string, any>[]): Promise<Document[]>;
+  splitDocuments(documents: Document[]): Promise<Document[]>;
+  transformDocuments(documents: Document[]): Promise<Document[]>;
+}
+//#endregion
+export { SemanticTextSplitter, SemanticTextSplitterParams, calculateCosineDistances };
+//# sourceMappingURL=semantic_text_splitter.d.cts.map

package/dist/semantic_text_splitter.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic_text_splitter.d.cts","names":[],"sources":["../src/semantic_text_splitter.ts"],"mappings":";;;;UASU,YAAA;EACR,QAAA;EACA,KAAA;EACA,gBAAA;EACA,yBAAA;AAAA;AAAA,iBA2Cc,wBAAA,CAAyB,SAAA,EAAW,YAAA,gBAA4B,YAAA;AAAA,KAkB3E,uBAAA;AAAA,UASY,0BAAA;EACf,UAAA,EAAY,UAAA;EACZ,UAAA;EACA,aAAA;EACA,uBAAA,GAA0B,uBAAA;EAC1B,yBAAA;EACA,cAAA;EACA,kBAAA;EACA,YAAA;AAAA;AAAA,cAGW,oBAAA,SAA6B,uBAAA;EAAA,QAChC,UAAA;EAAA,QACA,UAAA;EAAA,QACA,aAAA;EAAA,QACA,uBAAA;EAAA,QACA,yBAAA;EAAA,QACA,cAAA;EAAA,QACA,kBAAA;EAAA,QACA,YAAA;EAER,WAAA,CAAY,MAAA,EAAQ,0BAAA;EAAA,QAiBZ,4BAAA;EAAA,QAgCA,qBAAA;EAAA,QA0BM,0BAAA;EAAA,QAkBN,sBAAA;EAIF,SAAA,CAAU,IAAA,WAAe,OAAA;EAoDzB,eAAA,CACJ,KAAA,YACA,SAAA,GAAW,MAAA,kBACV,OAAA,CAAQ,QAAA;EA2BL,cAAA,CAAe,SAAA,EAAW,QAAA,KAAa,OAAA,CAAQ,QAAA;EAM/C,kBAAA,CAAmB,SAAA,EAAW,QAAA,KAAa,OAAA,CAAQ,QAAA;AAAA"}

package/dist/semantic_text_splitter.d.ts

@@ -0,0 +1,44 @@
+import { BaseDocumentTransformer, Document } from "@langchain/core/documents";
+import { Embeddings } from "@langchain/core/embeddings";
+
+//#region src/semantic_text_splitter.d.ts
+interface SentenceDict {
+  sentence: string;
+  index: number;
+  combinedSentence?: string;
+  combinedSentenceEmbedding?: number[];
+}
+declare function calculateCosineDistances(sentences: SentenceDict[]): [number[], SentenceDict[]];
+type BreakpointThresholdType = "percentile" | "standard_deviation" | "interquartile" | "gradient";
+interface SemanticTextSplitterParams {
+  embeddings: Embeddings;
+  bufferSize?: number;
+  addStartIndex?: boolean;
+  breakpointThresholdType?: BreakpointThresholdType;
+  breakpointThresholdAmount?: number;
+  numberOfChunks?: number;
+  sentenceSplitRegex?: string;
+  minChunkSize?: number;
+}
+declare class SemanticTextSplitter extends BaseDocumentTransformer {
+  private embeddings;
+  private bufferSize;
+  private addStartIndex;
+  private breakpointThresholdType;
+  private breakpointThresholdAmount;
+  private numberOfChunks?;
+  private sentenceSplitRegex;
+  private minChunkSize?;
+  constructor(params: SemanticTextSplitterParams);
+  private calculateBreakpointThreshold;
+  private thresholdFromClusters;
+  private calculateSentenceDistances;
+  private getSingleSentencesList;
+  splitText(text: string): Promise<string[]>;
+  createDocuments(texts: string[], metadatas?: Record<string, any>[]): Promise<Document[]>;
+  splitDocuments(documents: Document[]): Promise<Document[]>;
+  transformDocuments(documents: Document[]): Promise<Document[]>;
+}
+//#endregion
+export { SemanticTextSplitter, SemanticTextSplitterParams, calculateCosineDistances };
+//# sourceMappingURL=semantic_text_splitter.d.ts.map

package/dist/semantic_text_splitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic_text_splitter.d.ts","names":[],"sources":["../src/semantic_text_splitter.ts"],"mappings":";;;;UASU,YAAA;EACR,QAAA;EACA,KAAA;EACA,gBAAA;EACA,yBAAA;AAAA;AAAA,iBA2Cc,wBAAA,CAAyB,SAAA,EAAW,YAAA,gBAA4B,YAAA;AAAA,KAkB3E,uBAAA;AAAA,UASY,0BAAA;EACf,UAAA,EAAY,UAAA;EACZ,UAAA;EACA,aAAA;EACA,uBAAA,GAA0B,uBAAA;EAC1B,yBAAA;EACA,cAAA;EACA,kBAAA;EACA,YAAA;AAAA;AAAA,cAGW,oBAAA,SAA6B,uBAAA;EAAA,QAChC,UAAA;EAAA,QACA,UAAA;EAAA,QACA,aAAA;EAAA,QACA,uBAAA;EAAA,QACA,yBAAA;EAAA,QACA,cAAA;EAAA,QACA,kBAAA;EAAA,QACA,YAAA;EAER,WAAA,CAAY,MAAA,EAAQ,0BAAA;EAAA,QAiBZ,4BAAA;EAAA,QAgCA,qBAAA;EAAA,QA0BM,0BAAA;EAAA,QAkBN,sBAAA;EAIF,SAAA,CAAU,IAAA,WAAe,OAAA;EAoDzB,eAAA,CACJ,KAAA,YACA,SAAA,GAAW,MAAA,kBACV,OAAA,CAAQ,QAAA;EA2BL,cAAA,CAAe,SAAA,EAAW,QAAA,KAAa,OAAA,CAAQ,QAAA;EAM/C,kBAAA,CAAmB,SAAA,EAAW,QAAA,KAAa,OAAA,CAAQ,QAAA;AAAA"}

package/dist/semantic_text_splitter.js

@@ -0,0 +1,178 @@
+import { BaseDocumentTransformer, Document } from "@langchain/core/documents";
+//#region src/semantic_text_splitter.ts
+/**
+* Experimental semantic text splitter based on semantic similarity.
+* 
+* Inspired by Greg Kamradt's semantic chunking approach:
+* https://github.com/FullStackRetrieval-com/RetrievalTutorials/blob/main/tutorials/LevelsOfTextSplitting/5_Levels_Of_Text_Splitting.ipynb
+*/
+function combineSentences(sentences, bufferSize = 1) {
+	for (let i = 0; i < sentences.length; i++) {
+		const sentence = sentences[i];
+		let combinedSentence = "";
+		for (let j = Math.max(0, i - bufferSize); j < i; j++) combinedSentence += sentences[j].sentence + " ";
+		combinedSentence += sentence.sentence;
+		for (let j = i + 1; j <= Math.min(sentences.length - 1, i + bufferSize); j++) combinedSentence += " " + sentences[j].sentence;
+		sentence.combinedSentence = combinedSentence.trim();
+	}
+	return sentences;
+}
+function cosineSimilarity(a, b) {
+	const dotProduct = a.reduce((sum, ai, i) => sum + ai * b[i], 0);
+	const magnitudeA = Math.sqrt(a.reduce((sum, ai) => sum + ai * ai, 0));
+	const magnitudeB = Math.sqrt(b.reduce((sum, bi) => sum + bi * bi, 0));
+	if (magnitudeA === 0 || magnitudeB === 0) return 0;
+	return dotProduct / (magnitudeA * magnitudeB);
+}
+function calculateCosineDistances(sentences) {
+	const distances = [];
+	for (let i = 0; i < sentences.length - 1; i++) {
+		const embeddingCurrent = sentences[i].combinedSentenceEmbedding;
+		const embeddingNext = sentences[i + 1].combinedSentenceEmbedding;
+		const distance = 1 - cosineSimilarity(embeddingCurrent, embeddingNext);
+		distances.push(distance);
+	}
+	return [distances, sentences];
+}
+const BREAKPOINT_DEFAULTS = {
+	percentile: 95,
+	standard_deviation: 3,
+	interquartile: 1.5,
+	gradient: 95
+};
+var SemanticTextSplitter = class extends BaseDocumentTransformer {
+	embeddings;
+	bufferSize;
+	addStartIndex;
+	breakpointThresholdType;
+	breakpointThresholdAmount;
+	numberOfChunks;
+	sentenceSplitRegex;
+	minChunkSize;
+	constructor(params) {
+		super();
+		this.embeddings = params.embeddings;
+		this.bufferSize = params.bufferSize ?? 1;
+		this.addStartIndex = params.addStartIndex ?? false;
+		this.breakpointThresholdType = params.breakpointThresholdType ?? "percentile";
+		this.numberOfChunks = params.numberOfChunks;
+		this.sentenceSplitRegex = params.sentenceSplitRegex ?? "(?<=[.?!])\\s+";
+		this.minChunkSize = params.minChunkSize;
+		if (params.breakpointThresholdAmount === void 0) this.breakpointThresholdAmount = BREAKPOINT_DEFAULTS[this.breakpointThresholdType];
+		else this.breakpointThresholdAmount = params.breakpointThresholdAmount;
+	}
+	calculateBreakpointThreshold(distances) {
+		if (this.breakpointThresholdType === "percentile") {
+			const sorted = [...distances].sort((a, b) => a - b);
+			const index = Math.ceil(this.breakpointThresholdAmount / 100 * sorted.length) - 1;
+			return [sorted[Math.max(0, index)], distances];
+		} else if (this.breakpointThresholdType === "standard_deviation") {
+			const mean = distances.reduce((sum, d) => sum + d, 0) / distances.length;
+			const variance = distances.reduce((sum, d) => sum + Math.pow(d - mean, 2), 0) / distances.length;
+			const stdDev = Math.sqrt(variance);
+			return [mean + this.breakpointThresholdAmount * stdDev, distances];
+		} else if (this.breakpointThresholdType === "interquartile") {
+			const sorted = [...distances].sort((a, b) => a - b);
+			const q1Index = Math.floor(.25 * sorted.length);
+			const q3Index = Math.floor(.75 * sorted.length);
+			const q1 = sorted[q1Index];
+			const iqr = sorted[q3Index] - q1;
+			return [distances.reduce((sum, d) => sum + d, 0) / distances.length + this.breakpointThresholdAmount * iqr, distances];
+		} else if (this.breakpointThresholdType === "gradient") {
+			const distanceGradient = [];
+			for (let i = 0; i < distances.length - 1; i++) distanceGradient.push(distances[i + 1] - distances[i]);
+			const sortedGradient = [...distanceGradient].sort((a, b) => a - b);
+			const index = Math.ceil(this.breakpointThresholdAmount / 100 * sortedGradient.length) - 1;
+			return [sortedGradient[Math.max(0, index)], distanceGradient];
+		} else throw new Error(`Got unexpected breakpointThresholdType: ${this.breakpointThresholdType}`);
+	}
+	thresholdFromClusters(distances) {
+		if (this.numberOfChunks === void 0) throw new Error("This should never be called if numberOfChunks is undefined.");
+		const x1 = distances.length;
+		const y1 = 0;
+		const x2 = 1;
+		const y2 = 100;
+		const x = Math.max(Math.min(this.numberOfChunks, x1), x2);
+		let y;
+		if (x2 === x1) y = y2;
+		else y = y1 + (y2 - y1) / (x2 - x1) * (x - x1);
+		y = Math.min(Math.max(y, 0), 100);
+		const sorted = [...distances].sort((a, b) => a - b);
+		const index = Math.ceil(y / 100 * sorted.length) - 1;
+		return sorted[Math.max(0, index)];
+	}
+	async calculateSentenceDistances(singleSentencesList) {
+		const sentencesWithCombined = combineSentences(singleSentencesList.map((sentence, index) => ({
+			sentence,
+			index
+		})), this.bufferSize);
+		const combinedSentences = sentencesWithCombined.map((s) => s.combinedSentence);
+		const embeddings = await this.embeddings.embedDocuments(combinedSentences);
+		for (let i = 0; i < sentencesWithCombined.length; i++) sentencesWithCombined[i].combinedSentenceEmbedding = embeddings[i];
+		return calculateCosineDistances(sentencesWithCombined);
+	}
+	getSingleSentencesList(text) {
+		return text.split(new RegExp(this.sentenceSplitRegex)).filter((sentence) => sentence.trim().length > 0);
+	}
+	async splitText(text) {
+		const singleSentencesList = this.getSingleSentencesList(text);
+		if (singleSentencesList.length === 1) return singleSentencesList;
+		if (this.breakpointThresholdType === "gradient" && singleSentencesList.length === 2) return singleSentencesList;
+		const [distances, sentences] = await this.calculateSentenceDistances(singleSentencesList);
+		let breakpointDistanceThreshold;
+		let breakpointArray;
+		if (this.numberOfChunks !== void 0) {
+			breakpointDistanceThreshold = this.thresholdFromClusters(distances);
+			breakpointArray = distances;
+		} else [breakpointDistanceThreshold, breakpointArray] = this.calculateBreakpointThreshold(distances);
+		const indicesAboveThresh = breakpointArray.map((x, i) => ({
+			value: x,
+			index: i
+		})).filter(({ value }) => value > breakpointDistanceThreshold).map(({ index }) => index);
+		const chunks = [];
+		let startIndex = 0;
+		for (const index of indicesAboveThresh) {
+			const endIndex = index;
+			const combinedText = sentences.slice(startIndex, endIndex + 1).map((d) => d.sentence).join(" ");
+			if (this.minChunkSize !== void 0 && combinedText.length < this.minChunkSize) continue;
+			chunks.push(combinedText);
+			startIndex = index + 1;
+		}
+		if (startIndex < sentences.length) {
+			const combinedText = sentences.slice(startIndex).map((d) => d.sentence).join(" ");
+			chunks.push(combinedText);
+		}
+		return chunks;
+	}
+	async createDocuments(texts, metadatas = []) {
+		const _metadatas = metadatas.length > 0 ? metadatas : texts.map(() => ({}));
+		const documents = [];
+		for (let i = 0; i < texts.length; i++) {
+			const text = texts[i];
+			let startIndex = 0;
+			for (const chunk of await this.splitText(text)) {
+				const metadata = { ..._metadatas[i] };
+				if (this.addStartIndex) metadata.start_index = startIndex;
+				const newDoc = new Document({
+					pageContent: chunk,
+					metadata
+				});
+				documents.push(newDoc);
+				startIndex += chunk.length;
+			}
+		}
+		return documents;
+	}
+	async splitDocuments(documents) {
+		const texts = documents.map((doc) => doc.pageContent);
+		const metadatas = documents.map((doc) => doc.metadata);
+		return this.createDocuments(texts, metadatas);
+	}
+	async transformDocuments(documents) {
+		return this.splitDocuments(documents);
+	}
+};
+//#endregion
+export { SemanticTextSplitter, calculateCosineDistances };
+
+//# sourceMappingURL=semantic_text_splitter.js.map

package/dist/semantic_text_splitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic_text_splitter.js","names":[],"sources":["../src/semantic_text_splitter.ts"],"sourcesContent":["/**\r\n * Experimental semantic text splitter based on semantic similarity.\r\n * \r\n * Inspired by Greg Kamradt's semantic chunking approach:\r\n * https://github.com/FullStackRetrieval-com/RetrievalTutorials/blob/main/tutorials/LevelsOfTextSplitting/5_Levels_Of_Text_Splitting.ipynb\r\n */\r\nimport { Document, BaseDocumentTransformer } from \"@langchain/core/documents\";\r\nimport { Embeddings } from \"@langchain/core/embeddings\";\r\n\r\ninterface SentenceDict {\r\n  sentence: string;\r\n  index: number;\r\n  combinedSentence?: string;\r\n  combinedSentenceEmbedding?: number[];\r\n}\r\n\r\nfunction combineSentences(sentences: SentenceDict[], bufferSize: number = 1): SentenceDict[] {\r\n  // Go through each sentence dict\r\n  for (let i = 0; i < sentences.length; i++) {\r\n    const sentence = sentences[i];\r\n    \r\n    // Create the combined sentence by combining the sentences within the buffer\r\n    let combinedSentence = \"\";\r\n    \r\n    // Add sentences before the current one\r\n    for (let j = Math.max(0, i - bufferSize); j < i; j++) {\r\n      combinedSentence += sentences[j].sentence + \" \";\r\n    }\r\n    \r\n    // Add the current sentence\r\n    combinedSentence += sentence.sentence;\r\n    \r\n    // Add sentences after the current one\r\n    for (let j = i + 1; j <= Math.min(sentences.length - 1, i + bufferSize); j++) {\r\n      combinedSentence += \" \" + sentences[j].sentence;\r\n    }\r\n    \r\n    // Assign the combined sentence to the dict\r\n    sentence.combinedSentence = combinedSentence.trim();\r\n  }\r\n  \r\n  return sentences;\r\n}\r\n\r\nfunction cosineSimilarity(a: number[], b: number[]): number {\r\n  const dotProduct = a.reduce((sum, ai, i) => sum + ai * b[i], 0);\r\n  const magnitudeA = Math.sqrt(a.reduce((sum, ai) => sum + ai * ai, 0));\r\n  const magnitudeB = Math.sqrt(b.reduce((sum, bi) => sum + bi * bi, 0));\r\n  \r\n  if (magnitudeA === 0 || magnitudeB === 0) {\r\n    return 0;\r\n  }\r\n  \r\n  return dotProduct / (magnitudeA * magnitudeB);\r\n}\r\n\r\nexport function calculateCosineDistances(sentences: SentenceDict[]): [number[], SentenceDict[]] {\r\n  const distances: number[] = [];\r\n  \r\n  for (let i = 0; i < sentences.length - 1; i++) {\r\n    const embeddingCurrent = sentences[i].combinedSentenceEmbedding!;\r\n    const embeddingNext = sentences[i + 1].combinedSentenceEmbedding!;\r\n    \r\n    // Calculate cosine similarity\r\n    const similarity = cosineSimilarity(embeddingCurrent, embeddingNext);\r\n    \r\n    // Convert to cosine distance (1 - cosine similarity)\r\n    const distance = 1 - similarity;\r\n    distances.push(distance);\r\n  }\r\n  \r\n  return [distances, sentences];\r\n}\r\n\r\ntype BreakpointThresholdType = \"percentile\" | \"standard_deviation\" | \"interquartile\" | \"gradient\";\r\n\r\nconst BREAKPOINT_DEFAULTS: Record<BreakpointThresholdType, number> = {\r\n  percentile: 95,\r\n  standard_deviation: 3,\r\n  interquartile: 1.5,\r\n  gradient: 95,\r\n};\r\n\r\nexport interface SemanticTextSplitterParams {\r\n  embeddings: Embeddings;\r\n  bufferSize?: number;\r\n  addStartIndex?: boolean;\r\n  breakpointThresholdType?: BreakpointThresholdType;\r\n  breakpointThresholdAmount?: number;\r\n  numberOfChunks?: number;\r\n  sentenceSplitRegex?: string;\r\n  minChunkSize?: number;\r\n}\r\n\r\nexport class SemanticTextSplitter extends BaseDocumentTransformer {\r\n  private embeddings: Embeddings;\r\n  private bufferSize: number;\r\n  private addStartIndex: boolean;\r\n  private breakpointThresholdType: BreakpointThresholdType;\r\n  private breakpointThresholdAmount: number;\r\n  private numberOfChunks?: number;\r\n  private sentenceSplitRegex: string;\r\n  private minChunkSize?: number;\r\n\r\n  constructor(params: SemanticTextSplitterParams) {\r\n    super();\r\n    this.embeddings = params.embeddings;\r\n    this.bufferSize = params.bufferSize ?? 1;\r\n    this.addStartIndex = params.addStartIndex ?? false;\r\n    this.breakpointThresholdType = params.breakpointThresholdType ?? \"percentile\";\r\n    this.numberOfChunks = params.numberOfChunks;\r\n    this.sentenceSplitRegex = params.sentenceSplitRegex ?? \"(?<=[.?!])\\\\s+\";\r\n    this.minChunkSize = params.minChunkSize;\r\n    \r\n    if (params.breakpointThresholdAmount === undefined) {\r\n      this.breakpointThresholdAmount = BREAKPOINT_DEFAULTS[this.breakpointThresholdType];\r\n    } else {\r\n      this.breakpointThresholdAmount = params.breakpointThresholdAmount;\r\n    }\r\n  }\r\n\r\n  private calculateBreakpointThreshold(distances: number[]): [number, number[]] {\r\n    if (this.breakpointThresholdType === \"percentile\") {\r\n      const sorted = [...distances].sort((a, b) => a - b);\r\n      const index = Math.ceil((this.breakpointThresholdAmount / 100) * sorted.length) - 1;\r\n      return [sorted[Math.max(0, index)], distances];\r\n    } else if (this.breakpointThresholdType === \"standard_deviation\") {\r\n      const mean = distances.reduce((sum, d) => sum + d, 0) / distances.length;\r\n      const variance = distances.reduce((sum, d) => sum + Math.pow(d - mean, 2), 0) / distances.length;\r\n      const stdDev = Math.sqrt(variance);\r\n      return [mean + this.breakpointThresholdAmount * stdDev, distances];\r\n    } else if (this.breakpointThresholdType === \"interquartile\") {\r\n      const sorted = [...distances].sort((a, b) => a - b);\r\n      const q1Index = Math.floor(0.25 * sorted.length);\r\n      const q3Index = Math.floor(0.75 * sorted.length);\r\n      const q1 = sorted[q1Index];\r\n      const q3 = sorted[q3Index];\r\n      const iqr = q3 - q1;\r\n      const mean = distances.reduce((sum, d) => sum + d, 0) / distances.length;\r\n      return [mean + this.breakpointThresholdAmount * iqr, distances];\r\n    } else if (this.breakpointThresholdType === \"gradient\") {\r\n      const distanceGradient: number[] = [];\r\n      for (let i = 0; i < distances.length - 1; i++) {\r\n        distanceGradient.push(distances[i + 1] - distances[i]);\r\n      }\r\n      const sortedGradient = [...distanceGradient].sort((a, b) => a - b);\r\n      const index = Math.ceil((this.breakpointThresholdAmount / 100) * sortedGradient.length) - 1;\r\n      return [sortedGradient[Math.max(0, index)], distanceGradient];\r\n    } else {\r\n      throw new Error(`Got unexpected breakpointThresholdType: ${this.breakpointThresholdType}`);\r\n    }\r\n  }\r\n\r\n  private thresholdFromClusters(distances: number[]): number {\r\n    if (this.numberOfChunks === undefined) {\r\n      throw new Error(\"This should never be called if numberOfChunks is undefined.\");\r\n    }\r\n    \r\n    const x1 = distances.length;\r\n    const y1 = 0.0;\r\n    const x2 = 1.0;\r\n    const y2 = 100.0;\r\n    \r\n    const x = Math.max(Math.min(this.numberOfChunks, x1), x2);\r\n    \r\n    let y: number;\r\n    if (x2 === x1) {\r\n      y = y2;\r\n    } else {\r\n      y = y1 + ((y2 - y1) / (x2 - x1)) * (x - x1);\r\n    }\r\n    \r\n    y = Math.min(Math.max(y, 0), 100);\r\n    \r\n    const sorted = [...distances].sort((a, b) => a - b);\r\n    const index = Math.ceil((y / 100) * sorted.length) - 1;\r\n    return sorted[Math.max(0, index)];\r\n  }\r\n\r\n  private async calculateSentenceDistances(singleSentencesList: string[]): Promise<[number[], SentenceDict[]]> {\r\n    const sentences: SentenceDict[] = singleSentencesList.map((sentence, index) => ({\r\n      sentence,\r\n      index,\r\n    }));\r\n\r\n    const sentencesWithCombined = combineSentences(sentences, this.bufferSize);\r\n    \r\n    const combinedSentences = sentencesWithCombined.map(s => s.combinedSentence!);\r\n    const embeddings = await this.embeddings.embedDocuments(combinedSentences);\r\n    \r\n    for (let i = 0; i < sentencesWithCombined.length; i++) {\r\n      sentencesWithCombined[i].combinedSentenceEmbedding = embeddings[i];\r\n    }\r\n\r\n    return calculateCosineDistances(sentencesWithCombined);\r\n  }\r\n\r\n  private getSingleSentencesList(text: string): string[] {\r\n    return text.split(new RegExp(this.sentenceSplitRegex)).filter(sentence => sentence.trim().length > 0);\r\n  }\r\n\r\n  async splitText(text: string): Promise<string[]> {\r\n    const singleSentencesList = this.getSingleSentencesList(text);\r\n\r\n    if (singleSentencesList.length === 1) {\r\n      return singleSentencesList;\r\n    }\r\n\r\n    if (this.breakpointThresholdType === \"gradient\" && singleSentencesList.length === 2) {\r\n      return singleSentencesList;\r\n    }\r\n\r\n    const [distances, sentences] = await this.calculateSentenceDistances(singleSentencesList);\r\n\r\n    let breakpointDistanceThreshold: number;\r\n    let breakpointArray: number[];\r\n\r\n    if (this.numberOfChunks !== undefined) {\r\n      breakpointDistanceThreshold = this.thresholdFromClusters(distances);\r\n      breakpointArray = distances;\r\n    } else {\r\n      [breakpointDistanceThreshold, breakpointArray] = this.calculateBreakpointThreshold(distances);\r\n    }\r\n\r\n    const indicesAboveThresh = breakpointArray\r\n      .map((x, i) => ({ value: x, index: i }))\r\n      .filter(({ value }) => value > breakpointDistanceThreshold)\r\n      .map(({ index }) => index);\r\n\r\n    const chunks: string[] = [];\r\n    let startIndex = 0;\r\n\r\n    for (const index of indicesAboveThresh) {\r\n      const endIndex = index;\r\n      const group = sentences.slice(startIndex, endIndex + 1);\r\n      const combinedText = group.map((d: SentenceDict) => d.sentence).join(\" \");\r\n\r\n      if (this.minChunkSize !== undefined && combinedText.length < this.minChunkSize) {\r\n        continue;\r\n      }\r\n      \r\n      chunks.push(combinedText);\r\n      startIndex = index + 1;\r\n    }\r\n\r\n    if (startIndex < sentences.length) {\r\n      const combinedText = sentences.slice(startIndex).map((d: SentenceDict) => d.sentence).join(\" \");\r\n      chunks.push(combinedText);\r\n    }\r\n\r\n    return chunks;\r\n  }\r\n\r\n  async createDocuments(\r\n    texts: string[],\r\n    metadatas: Record<string, any>[] = []\r\n  ): Promise<Document[]> {\r\n    const _metadatas = metadatas.length > 0 ? metadatas : texts.map(() => (<Record<string, any>>{}));\r\n    const documents: Document[] = [];\r\n    \r\n    for (let i = 0; i < texts.length; i++) {\r\n      const text = texts[i];\r\n      let startIndex = 0;\r\n      \r\n      for (const chunk of await this.splitText(text)) {\r\n        const metadata = { ..._metadatas[i] };\r\n        if (this.addStartIndex) {\r\n          metadata.start_index = startIndex;\r\n        }\r\n        \r\n        const newDoc = new Document({\r\n          pageContent: chunk,\r\n          metadata,\r\n        });\r\n        \r\n        documents.push(newDoc);\r\n        startIndex += chunk.length;\r\n      }\r\n    }\r\n    \r\n    return documents;\r\n  }\r\n\r\n  async splitDocuments(documents: Document[]): Promise<Document[]> {\r\n    const texts = documents.map(doc => doc.pageContent);\r\n    const metadatas = documents.map(doc => doc.metadata);\r\n    return this.createDocuments(texts, metadatas);\r\n  }\r\n\r\n  async transformDocuments(documents: Document[]): Promise<Document[]> {\r\n    return this.splitDocuments(documents);\r\n  }\r\n}"],"mappings":";;;;;;;;AAgBA,SAAS,iBAAiB,WAA2B,aAAqB,GAAmB;AAE3F,MAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;EACzC,MAAM,WAAW,UAAU;EAG3B,IAAI,mBAAmB;AAGvB,OAAK,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,WAAW,EAAE,IAAI,GAAG,IAC/C,qBAAoB,UAAU,GAAG,WAAW;AAI9C,sBAAoB,SAAS;AAG7B,OAAK,IAAI,IAAI,IAAI,GAAG,KAAK,KAAK,IAAI,UAAU,SAAS,GAAG,IAAI,WAAW,EAAE,IACvE,qBAAoB,MAAM,UAAU,GAAG;AAIzC,WAAS,mBAAmB,iBAAiB,MAAM;;AAGrD,QAAO;;AAGT,SAAS,iBAAiB,GAAa,GAAqB;CAC1D,MAAM,aAAa,EAAE,QAAQ,KAAK,IAAI,MAAM,MAAM,KAAK,EAAE,IAAI,EAAE;CAC/D,MAAM,aAAa,KAAK,KAAK,EAAE,QAAQ,KAAK,OAAO,MAAM,KAAK,IAAI,EAAE,CAAC;CACrE,MAAM,aAAa,KAAK,KAAK,EAAE,QAAQ,KAAK,OAAO,MAAM,KAAK,IAAI,EAAE,CAAC;AAErE,KAAI,eAAe,KAAK,eAAe,EACrC,QAAO;AAGT,QAAO,cAAc,aAAa;;AAGpC,SAAgB,yBAAyB,WAAuD;CAC9F,MAAM,YAAsB,EAAE;AAE9B,MAAK,IAAI,IAAI,GAAG,IAAI,UAAU,SAAS,GAAG,KAAK;EAC7C,MAAM,mBAAmB,UAAU,GAAG;EACtC,MAAM,gBAAgB,UAAU,IAAI,GAAG;EAMvC,MAAM,WAAW,IAHE,iBAAiB,kBAAkB,cAAc;AAIpE,YAAU,KAAK,SAAS;;AAG1B,QAAO,CAAC,WAAW,UAAU;;AAK/B,MAAM,sBAA+D;CACnE,YAAY;CACZ,oBAAoB;CACpB,eAAe;CACf,UAAU;CACX;AAaD,IAAa,uBAAb,cAA0C,wBAAwB;CAChE;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,QAAoC;AAC9C,SAAO;AACP,OAAK,aAAa,OAAO;AACzB,OAAK,aAAa,OAAO,cAAc;AACvC,OAAK,gBAAgB,OAAO,iBAAiB;AAC7C,OAAK,0BAA0B,OAAO,2BAA2B;AACjE,OAAK,iBAAiB,OAAO;AAC7B,OAAK,qBAAqB,OAAO,sBAAsB;AACvD,OAAK,eAAe,OAAO;AAE3B,MAAI,OAAO,8BAA8B,KAAA,EACvC,MAAK,4BAA4B,oBAAoB,KAAK;MAE1D,MAAK,4BAA4B,OAAO;;CAI5C,6BAAqC,WAAyC;AAC5E,MAAI,KAAK,4BAA4B,cAAc;GACjD,MAAM,SAAS,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,IAAI,EAAE;GACnD,MAAM,QAAQ,KAAK,KAAM,KAAK,4BAA4B,MAAO,OAAO,OAAO,GAAG;AAClF,UAAO,CAAC,OAAO,KAAK,IAAI,GAAG,MAAM,GAAG,UAAU;aACrC,KAAK,4BAA4B,sBAAsB;GAChE,MAAM,OAAO,UAAU,QAAQ,KAAK,MAAM,MAAM,GAAG,EAAE,GAAG,UAAU;GAClE,MAAM,WAAW,UAAU,QAAQ,KAAK,MAAM,MAAM,KAAK,IAAI,IAAI,MAAM,EAAE,EAAE,EAAE,GAAG,UAAU;GAC1F,MAAM,SAAS,KAAK,KAAK,SAAS;AAClC,UAAO,CAAC,OAAO,KAAK,4BAA4B,QAAQ,UAAU;aACzD,KAAK,4BAA4B,iBAAiB;GAC3D,MAAM,SAAS,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,IAAI,EAAE;GACnD,MAAM,UAAU,KAAK,MAAM,MAAO,OAAO,OAAO;GAChD,MAAM,UAAU,KAAK,MAAM,MAAO,OAAO,OAAO;GAChD,MAAM,KAAK,OAAO;GAElB,MAAM,MADK,OAAO,WACD;AAEjB,UAAO,CADM,UAAU,QAAQ,KAAK,MAAM,MAAM,GAAG,EAAE,GAAG,UAAU,SACnD,KAAK,4BAA4B,KAAK,UAAU;aACtD,KAAK,4BAA4B,YAAY;GACtD,MAAM,mBAA6B,EAAE;AACrC,QAAK,IAAI,IAAI,GAAG,IAAI,UAAU,SAAS,GAAG,IACxC,kBAAiB,KAAK,UAAU,IAAI,KAAK,UAAU,GAAG;GAExD,MAAM,iBAAiB,CAAC,GAAG,iBAAiB,CAAC,MAAM,GAAG,MAAM,IAAI,EAAE;GAClE,MAAM,QAAQ,KAAK,KAAM,KAAK,4BAA4B,MAAO,eAAe,OAAO,GAAG;AAC1F,UAAO,CAAC,eAAe,KAAK,IAAI,GAAG,MAAM,GAAG,iBAAiB;QAE7D,OAAM,IAAI,MAAM,2CAA2C,KAAK,0BAA0B;;CAI9F,sBAA8B,WAA6B;AACzD,MAAI,KAAK,mBAAmB,KAAA,EAC1B,OAAM,IAAI,MAAM,8DAA8D;EAGhF,MAAM,KAAK,UAAU;EACrB,MAAM,KAAK;EACX,MAAM,KAAK;EACX,MAAM,KAAK;EAEX,MAAM,IAAI,KAAK,IAAI,KAAK,IAAI,KAAK,gBAAgB,GAAG,EAAE,GAAG;EAEzD,IAAI;AACJ,MAAI,OAAO,GACT,KAAI;MAEJ,KAAI,MAAO,KAAK,OAAO,KAAK,OAAQ,IAAI;AAG1C,MAAI,KAAK,IAAI,KAAK,IAAI,GAAG,EAAE,EAAE,IAAI;EAEjC,MAAM,SAAS,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,MAAM,IAAI,EAAE;EACnD,MAAM,QAAQ,KAAK,KAAM,IAAI,MAAO,OAAO,OAAO,GAAG;AACrD,SAAO,OAAO,KAAK,IAAI,GAAG,MAAM;;CAGlC,MAAc,2BAA2B,qBAAoE;EAM3G,MAAM,wBAAwB,iBALI,oBAAoB,KAAK,UAAU,WAAW;GAC9E;GACA;GACD,EAAE,EAEuD,KAAK,WAAW;EAE1E,MAAM,oBAAoB,sBAAsB,KAAI,MAAK,EAAE,iBAAkB;EAC7E,MAAM,aAAa,MAAM,KAAK,WAAW,eAAe,kBAAkB;AAE1E,OAAK,IAAI,IAAI,GAAG,IAAI,sBAAsB,QAAQ,IAChD,uBAAsB,GAAG,4BAA4B,WAAW;AAGlE,SAAO,yBAAyB,sBAAsB;;CAGxD,uBAA+B,MAAwB;AACrD,SAAO,KAAK,MAAM,IAAI,OAAO,KAAK,mBAAmB,CAAC,CAAC,QAAO,aAAY,SAAS,MAAM,CAAC,SAAS,EAAE;;CAGvG,MAAM,UAAU,MAAiC;EAC/C,MAAM,sBAAsB,KAAK,uBAAuB,KAAK;AAE7D,MAAI,oBAAoB,WAAW,EACjC,QAAO;AAGT,MAAI,KAAK,4BAA4B,cAAc,oBAAoB,WAAW,EAChF,QAAO;EAGT,MAAM,CAAC,WAAW,aAAa,MAAM,KAAK,2BAA2B,oBAAoB;EAEzF,IAAI;EACJ,IAAI;AAEJ,MAAI,KAAK,mBAAmB,KAAA,GAAW;AACrC,iCAA8B,KAAK,sBAAsB,UAAU;AACnE,qBAAkB;QAElB,EAAC,6BAA6B,mBAAmB,KAAK,6BAA6B,UAAU;EAG/F,MAAM,qBAAqB,gBACxB,KAAK,GAAG,OAAO;GAAE,OAAO;GAAG,OAAO;GAAG,EAAE,CACvC,QAAQ,EAAE,YAAY,QAAQ,4BAA4B,CAC1D,KAAK,EAAE,YAAY,MAAM;EAE5B,MAAM,SAAmB,EAAE;EAC3B,IAAI,aAAa;AAEjB,OAAK,MAAM,SAAS,oBAAoB;GACtC,MAAM,WAAW;GAEjB,MAAM,eADQ,UAAU,MAAM,YAAY,WAAW,EAAE,CAC5B,KAAK,MAAoB,EAAE,SAAS,CAAC,KAAK,IAAI;AAEzE,OAAI,KAAK,iBAAiB,KAAA,KAAa,aAAa,SAAS,KAAK,aAChE;AAGF,UAAO,KAAK,aAAa;AACzB,gBAAa,QAAQ;;AAGvB,MAAI,aAAa,UAAU,QAAQ;GACjC,MAAM,eAAe,UAAU,MAAM,WAAW,CAAC,KAAK,MAAoB,EAAE,SAAS,CAAC,KAAK,IAAI;AAC/F,UAAO,KAAK,aAAa;;AAG3B,SAAO;;CAGT,MAAM,gBACJ,OACA,YAAmC,EAAE,EAChB;EACrB,MAAM,aAAa,UAAU,SAAS,IAAI,YAAY,MAAM,WAAgC,EAAE,EAAE;EAChG,MAAM,YAAwB,EAAE;AAEhC,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;GACrC,MAAM,OAAO,MAAM;GACnB,IAAI,aAAa;AAEjB,QAAK,MAAM,SAAS,MAAM,KAAK,UAAU,KAAK,EAAE;IAC9C,MAAM,WAAW,EAAE,GAAG,WAAW,IAAI;AACrC,QAAI,KAAK,cACP,UAAS,cAAc;IAGzB,MAAM,SAAS,IAAI,SAAS;KAC1B,aAAa;KACb;KACD,CAAC;AAEF,cAAU,KAAK,OAAO;AACtB,kBAAc,MAAM;;;AAIxB,SAAO;;CAGT,MAAM,eAAe,WAA4C;EAC/D,MAAM,QAAQ,UAAU,KAAI,QAAO,IAAI,YAAY;EACnD,MAAM,YAAY,UAAU,KAAI,QAAO,IAAI,SAAS;AACpD,SAAO,KAAK,gBAAgB,OAAO,UAAU;;CAG/C,MAAM,mBAAmB,WAA4C;AACnE,SAAO,KAAK,eAAe,UAAU"}