modelfusion 0.5.0 → 0.7.0

package/model-function/generate-text/generateText.js

@@ -1,16 +1,4 @@
 import { executeCall } from "../executeCall.js";
-/**
- * Generates a text using a prompt.
- * The prompt format depends on the model.
- * For example, OpenAI text models expect a string prompt, and OpenAI chat models expect an array of chat messages.
- *
- * @example
- * const model = new OpenAITextGenerationModel(...);
- *
- * const { text } = await model.generateText(
- *   "Write a short story about a robot learning to love:\n\n"
- * );
- */
 export async function generateText(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 model, prompt, options) {
@@ -55,9 +43,11 @@ model, prompt, options) {
             generatedText: output,
         }),
     });
-    return {
-        text: result.output,
-        response: result.response,
-        metadata: result.metadata,
-    };
+    return options?.fullResponse === true
+        ? {
+            text: result.output,
+            response: result.response,
+            metadata: result.metadata,
+        }
+        : result.output;
 }

package/model-function/generate-text/streamText.cjs

@@ -107,9 +107,11 @@ async function streamText(model, prompt, options) {
         });
         throw result.error;
     }
-    return {
-        textStream: result.output,
-        metadata: startMetadata,
-    };
+    return options?.fullResponse === true
+        ? {
+            textStream: result.output,
+            metadata: startMetadata,
+        }
+        : result.output;
 }
 exports.streamText = streamText;

package/model-function/generate-text/streamText.d.ts

@@ -5,7 +5,15 @@ import { TextGenerationModel, TextGenerationModelSettings } from "./TextGenerati
 export declare function streamText<PROMPT, FULL_DELTA, SETTINGS extends TextGenerationModelSettings>(model: TextGenerationModel<PROMPT, unknown, FULL_DELTA, SETTINGS> & {
     generateDeltaStreamResponse: (prompt: PROMPT, options: FunctionOptions<SETTINGS>) => PromiseLike<AsyncIterable<DeltaEvent<FULL_DELTA>>>;
     extractTextDelta: (fullDelta: FULL_DELTA) => string | undefined;
-}, prompt: PROMPT, options?: FunctionOptions<SETTINGS>): Promise<{
+}, prompt: PROMPT, options?: FunctionOptions<SETTINGS> & {
+    fullResponse?: false;
+}): Promise<AsyncIterable<string>>;
+export declare function streamText<PROMPT, FULL_DELTA, SETTINGS extends TextGenerationModelSettings>(model: TextGenerationModel<PROMPT, unknown, FULL_DELTA, SETTINGS> & {
+    generateDeltaStreamResponse: (prompt: PROMPT, options: FunctionOptions<SETTINGS>) => PromiseLike<AsyncIterable<DeltaEvent<FULL_DELTA>>>;
+    extractTextDelta: (fullDelta: FULL_DELTA) => string | undefined;
+}, prompt: PROMPT, options: FunctionOptions<SETTINGS> & {
+    fullResponse: true;
+}): Promise<{
     textStream: AsyncIterable<string>;
     metadata: Omit<CallMetadata<TextGenerationModel<PROMPT, unknown, FULL_DELTA, SETTINGS>>, "durationInMs">;
 }>;

package/model-function/generate-text/streamText.js

@@ -104,8 +104,10 @@ export async function streamText(model, prompt, options) {
         });
         throw result.error;
     }
-    return {
-        textStream: result.output,
-        metadata: startMetadata,
-    };
+    return options?.fullResponse === true
+        ? {
+            textStream: result.output,
+            metadata: startMetadata,
+        }
+        : result.output;
 }

package/model-function/transcribe-audio/transcribe.cjs

@@ -2,20 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.transcribe = void 0;
 const executeCall_js_1 = require("../executeCall.cjs");
-/**
- * Transcribe audio data into text.
- *
- * @example
- * const data = await fs.promises.readFile("data/test.mp3");
- *
- * const { transcription } = await transcribe(
- *   new OpenAITranscriptionModel({ model: "whisper-1" }),
- *   {
- *     type: "mp3",
- *     data,
- *   }
- * );
- */
 async function transcribe(model, data, options) {
     const result = await (0, executeCall_js_1.executeCall)({
         model,
@@ -53,10 +39,12 @@ async function transcribe(model, data, options) {
             transcription: output,
         }),
     });
-    return {
-        transcription: result.output,
-        response: result.response,
-        metadata: result.metadata,
-    };
+    return options?.fullResponse === true
+        ? {
+            transcription: result.output,
+            response: result.response,
+            metadata: result.metadata,
+        }
+        : result.output;
 }
 exports.transcribe = transcribe;

package/model-function/transcribe-audio/transcribe.d.ts

@@ -7,7 +7,7 @@ import { TranscriptionModel, TranscriptionModelSettings } from "./TranscriptionM
  * @example
  * const data = await fs.promises.readFile("data/test.mp3");
  *
- * const { transcription } = await transcribe(
+ * const transcription = await transcribe(
  *   new OpenAITranscriptionModel({ model: "whisper-1" }),
  *   {
  *     type: "mp3",
@@ -15,8 +15,13 @@ import { TranscriptionModel, TranscriptionModelSettings } from "./TranscriptionM
  *   }
  * );
  */
-export declare function transcribe<DATA, RESPONSE, SETTINGS extends TranscriptionModelSettings>(model: TranscriptionModel<DATA, RESPONSE, SETTINGS>, data: DATA, options?: FunctionOptions<SETTINGS>): Promise<{
+export declare function transcribe<DATA, RESPONSE, SETTINGS extends TranscriptionModelSettings>(model: TranscriptionModel<DATA, RESPONSE, SETTINGS>, data: DATA, options: FunctionOptions<SETTINGS> & {
+    fullResponse: true;
+}): Promise<{
     transcription: string;
     response: RESPONSE;
     metadata: CallMetadata<TranscriptionModel<DATA, RESPONSE, SETTINGS>>;
 }>;
+export declare function transcribe<DATA, RESPONSE, SETTINGS extends TranscriptionModelSettings>(model: TranscriptionModel<DATA, RESPONSE, SETTINGS>, data: DATA, options?: FunctionOptions<SETTINGS> & {
+    fullResponse?: false;
+}): Promise<string>;

package/model-function/transcribe-audio/transcribe.js

@@ -1,18 +1,4 @@
 import { executeCall } from "../executeCall.js";
-/**
- * Transcribe audio data into text.
- *
- * @example
- * const data = await fs.promises.readFile("data/test.mp3");
- *
- * const { transcription } = await transcribe(
- *   new OpenAITranscriptionModel({ model: "whisper-1" }),
- *   {
- *     type: "mp3",
- *     data,
- *   }
- * );
- */
 export async function transcribe(model, data, options) {
     const result = await executeCall({
         model,
@@ -50,9 +36,11 @@ export async function transcribe(model, data, options) {
             transcription: output,
         }),
     });
-    return {
-        transcription: result.output,
-        response: result.response,
-        metadata: result.metadata,
-    };
+    return options?.fullResponse === true
+        ? {
+            transcription: result.output,
+            response: result.response,
+            metadata: result.metadata,
+        }
+        : result.output;
 }

package/model-provider/cohere/CohereTextEmbeddingModel.cjs

@@ -30,7 +30,7 @@ exports.COHERE_TEXT_EMBEDDING_MODELS = {
  * @see https://docs.cohere.com/reference/embed
  *
  * @example
- * const { embeddings } = await embedTexts(
+ * const embeddings = await embedTexts(
  *   new CohereTextEmbeddingModel({ model: "embed-english-light-v2.0" }),
  *   [
  *     "At first, Nox didn't know what to do with the pup.",

package/model-provider/cohere/CohereTextEmbeddingModel.d.ts

@@ -38,7 +38,7 @@ export interface CohereTextEmbeddingModelSettings extends TextEmbeddingModelSett
  * @see https://docs.cohere.com/reference/embed
  *
  * @example
- * const { embeddings } = await embedTexts(
+ * const embeddings = await embedTexts(
  *   new CohereTextEmbeddingModel({ model: "embed-english-light-v2.0" }),
  *   [
  *     "At first, Nox didn't know what to do with the pup.",

package/model-provider/cohere/CohereTextEmbeddingModel.js

@@ -24,7 +24,7 @@ export const COHERE_TEXT_EMBEDDING_MODELS = {
  * @see https://docs.cohere.com/reference/embed
  *
  * @example
- * const { embeddings } = await embedTexts(
+ * const embeddings = await embedTexts(
  *   new CohereTextEmbeddingModel({ model: "embed-english-light-v2.0" }),
  *   [
  *     "At first, Nox didn't know what to do with the pup.",

package/model-provider/cohere/CohereTextGenerationModel.cjs

@@ -40,7 +40,7 @@ exports.COHERE_TEXT_GENERATION_MODELS = {
  *   maxTokens: 500,
  * });
  *
- * const { text } = await generateText(
+ * const text = await generateText(
  *    model,
  *   "Write a short story about a robot learning to love:\n\n"
  * );

package/model-provider/cohere/CohereTextGenerationModel.d.ts

@@ -59,7 +59,7 @@ export interface CohereTextGenerationModelSettings extends TextGenerationModelSe
  *   maxTokens: 500,
  * });
  *
- * const { text } = await generateText(
+ * const text = await generateText(
  *    model,
  *   "Write a short story about a robot learning to love:\n\n"
  * );

package/model-provider/cohere/CohereTextGenerationModel.js

@@ -34,7 +34,7 @@ export const COHERE_TEXT_GENERATION_MODELS = {
  *   maxTokens: 500,
  * });
  *
- * const { text } = await generateText(
+ * const text = await generateText(
  *    model,
  *   "Write a short story about a robot learning to love:\n\n"
  * );

package/model-provider/huggingface/HuggingFaceTextEmbeddingModel.cjs

@@ -21,7 +21,7 @@ const HuggingFaceError_js_1 = require("./HuggingFaceError.cjs");
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),
  * });
  *
- * const { embeddings } = await embedTexts(
+ * const embeddings = await embedTexts(
  *   model,
  *   [
  *     "At first, Nox didn't know what to do with the pup.",

package/model-provider/huggingface/HuggingFaceTextEmbeddingModel.d.ts

@@ -29,7 +29,7 @@ export interface HuggingFaceTextEmbeddingModelSettings extends TextEmbeddingMode
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),
  * });
  *
- * const { embeddings } = await embedTexts(
+ * const embeddings = await embedTexts(
  *   model,
  *   [
  *     "At first, Nox didn't know what to do with the pup.",

package/model-provider/huggingface/HuggingFaceTextEmbeddingModel.js

@@ -15,7 +15,7 @@ import { failedHuggingFaceCallResponseHandler } from "./HuggingFaceError.js";
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),
  * });
  *
- * const { embeddings } = await embedTexts(
+ * const embeddings = await embedTexts(
  *   model,
  *   [
  *     "At first, Nox didn't know what to do with the pup.",

package/model-provider/huggingface/HuggingFaceTextGenerationModel.cjs

@@ -23,7 +23,7 @@ const PromptMappingTextGenerationModel_js_1 = require("../../prompt/PromptMappin
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),
  * });
  *
- * const { text } = await generateText(
+ * const text = await generateText(
  *   model,
  *   "Write a short story about a robot learning to love:\n\n"
  * );

package/model-provider/huggingface/HuggingFaceTextGenerationModel.d.ts

@@ -38,7 +38,7 @@ export interface HuggingFaceTextGenerationModelSettings extends TextGenerationMo
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),
  * });
  *
- * const { text } = await generateText(
+ * const text = await generateText(
  *   model,
  *   "Write a short story about a robot learning to love:\n\n"
  * );

package/model-provider/huggingface/HuggingFaceTextGenerationModel.js

@@ -17,7 +17,7 @@ import { PromptMappingTextGenerationModel } from "../../prompt/PromptMappingText
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),
  * });
  *
- * const { text } = await generateText(
+ * const text = await generateText(
  *   model,
  *   "Write a short story about a robot learning to love:\n\n"
  * );

package/model-provider/openai/OpenAIImageGenerationModel.cjs

@@ -22,7 +22,7 @@ exports.calculateOpenAIImageGenerationCostInMillicents = calculateOpenAIImageGen
  * @see https://platform.openai.com/docs/api-reference/images/create
  *
  * @example
- * const { image } = await generateImage(
+ * const image = await generateImage(
  *   new OpenAIImageGenerationModel({ size: "512x512" }),
  *   "the wicked witch of the west in the style of early 19th century painting"
  * );

package/model-provider/openai/OpenAIImageGenerationModel.d.ts

@@ -20,7 +20,7 @@ export interface OpenAIImageGenerationSettings extends ImageGenerationModelSetti
  * @see https://platform.openai.com/docs/api-reference/images/create
  *
  * @example
- * const { image } = await generateImage(
+ * const image = await generateImage(
  *   new OpenAIImageGenerationModel({ size: "512x512" }),
  *   "the wicked witch of the west in the style of early 19th century painting"
  * );

package/model-provider/openai/OpenAIImageGenerationModel.js

@@ -18,7 +18,7 @@ export const calculateOpenAIImageGenerationCostInMillicents = ({ settings, }) =>
  * @see https://platform.openai.com/docs/api-reference/images/create
  *
  * @example
- * const { image } = await generateImage(
+ * const image = await generateImage(
  *   new OpenAIImageGenerationModel({ size: "512x512" }),
  *   "the wicked witch of the west in the style of early 19th century painting"
  * );

package/model-provider/openai/OpenAITextEmbeddingModel.cjs

@@ -36,7 +36,7 @@ exports.calculateOpenAIEmbeddingCostInMillicents = calculateOpenAIEmbeddingCostI
  * @see https://platform.openai.com/docs/api-reference/embeddings
  *
  * @example
- * const { embeddings } = await embedTexts(
+ * const embeddings = await embedTexts(
  *   new OpenAITextEmbeddingModel({ model: "text-embedding-ada-002" }),
  *   [
  *     "At first, Nox didn't know what to do with the pup.",

package/model-provider/openai/OpenAITextEmbeddingModel.d.ts

@@ -32,7 +32,7 @@ export interface OpenAITextEmbeddingModelSettings extends TextEmbeddingModelSett
  * @see https://platform.openai.com/docs/api-reference/embeddings
  *
  * @example
- * const { embeddings } = await embedTexts(
+ * const embeddings = await embedTexts(
  *   new OpenAITextEmbeddingModel({ model: "text-embedding-ada-002" }),
  *   [
  *     "At first, Nox didn't know what to do with the pup.",

package/model-provider/openai/OpenAITextEmbeddingModel.js

@@ -28,7 +28,7 @@ export const calculateOpenAIEmbeddingCostInMillicents = ({ model, responses, })
  * @see https://platform.openai.com/docs/api-reference/embeddings
  *
  * @example
- * const { embeddings } = await embedTexts(
+ * const embeddings = await embedTexts(
  *   new OpenAITextEmbeddingModel({ model: "text-embedding-ada-002" }),
  *   [
  *     "At first, Nox didn't know what to do with the pup.",

package/model-provider/openai/OpenAITextGenerationModel.cjs

@@ -79,7 +79,7 @@ exports.calculateOpenAITextGenerationCostInMillicents = calculateOpenAITextGener
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),
  * });
  *
- * const { text } = await generateText(
+ * const text = await generateText(
  *   model,
  *   "Write a short story about a robot learning to love:\n\n"
  * );

package/model-provider/openai/OpenAITextGenerationModel.d.ts

@@ -96,7 +96,7 @@ export interface OpenAITextGenerationModelSettings extends TextGenerationModelSe
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),
  * });
  *
- * const { text } = await generateText(
+ * const text = await generateText(
  *   model,
  *   "Write a short story about a robot learning to love:\n\n"
  * );

package/model-provider/openai/OpenAITextGenerationModel.js

@@ -71,7 +71,7 @@ export const calculateOpenAITextGenerationCostInMillicents = ({ model, response,
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),
  * });
  *
- * const { text } = await generateText(
+ * const text = await generateText(
  *   model,
  *   "Write a short story about a robot learning to love:\n\n"
  * );

package/model-provider/openai/OpenAITranscriptionModel.cjs

@@ -34,7 +34,7 @@ exports.calculateOpenAITranscriptionCostInMillicents = calculateOpenAITranscript
  * @example
  * const data = await fs.promises.readFile("data/test.mp3");
  *
- * const { transcription } = await transcribe(
+ * const transcription = await transcribe(
  *   new OpenAITranscriptionModel({ model: "whisper-1" }),
  *   {
  *     type: "mp3",

package/model-provider/openai/OpenAITranscriptionModel.d.ts

@@ -39,7 +39,7 @@ export type OpenAITranscriptionInput = {
  * @example
  * const data = await fs.promises.readFile("data/test.mp3");
  *
- * const { transcription } = await transcribe(
+ * const transcription = await transcribe(
  *   new OpenAITranscriptionModel({ model: "whisper-1" }),
  *   {
  *     type: "mp3",

package/model-provider/openai/OpenAITranscriptionModel.js

@@ -27,7 +27,7 @@ export const calculateOpenAITranscriptionCostInMillicents = ({ model, response,
  * @example
  * const data = await fs.promises.readFile("data/test.mp3");
  *
- * const { transcription } = await transcribe(
+ * const transcription = await transcribe(
  *   new OpenAITranscriptionModel({ model: "whisper-1" }),
  *   {
  *     type: "mp3",

package/model-provider/openai/chat/OpenAIChatModel.cjs

@@ -95,7 +95,7 @@ exports.calculateOpenAIChatCostInMillicents = calculateOpenAIChatCostInMillicent
  *   maxTokens: 500,
  * });
  *
- * const { text } = await generateText([
+ * const text = await generateText([
  *   model,
  *   OpenAIChatMessage.system(
  *     "Write a short story about a robot learning to love:"

package/model-provider/openai/chat/OpenAIChatModel.d.ts

@@ -109,7 +109,7 @@ export interface OpenAIChatSettings extends TextGenerationModelSettings, OpenAIM
  *   maxTokens: 500,
  * });
  *
- * const { text } = await generateText([
+ * const text = await generateText([
  *   model,
  *   OpenAIChatMessage.system(
  *     "Write a short story about a robot learning to love:"

package/model-provider/openai/chat/OpenAIChatModel.js

@@ -87,7 +87,7 @@ export const calculateOpenAIChatCostInMillicents = ({ model, response, }) => res
  *   maxTokens: 500,
  * });
  *
- * const { text } = await generateText([
+ * const text = await generateText([
  *   model,
  *   OpenAIChatMessage.system(
  *     "Write a short story about a robot learning to love:"

package/model-provider/stability/StabilityImageGenerationModel.cjs

@@ -12,7 +12,7 @@ const StabilityError_js_1 = require("./StabilityError.cjs");
  * @see https://api.stability.ai/docs#tag/v1generation/operation/textToImage
  *
  * @example
- * const { image } = await generateImage(
+ * const image = await generateImage(
  *   new StabilityImageGenerationModel({
  *     model: "stable-diffusion-512-v2-1",
  *     cfgScale: 7,

package/model-provider/stability/StabilityImageGenerationModel.d.ts

@@ -10,7 +10,7 @@ import { ThrottleFunction } from "../../util/api/ThrottleFunction.js";
  * @see https://api.stability.ai/docs#tag/v1generation/operation/textToImage
  *
  * @example
- * const { image } = await generateImage(
+ * const image = await generateImage(
  *   new StabilityImageGenerationModel({
  *     model: "stable-diffusion-512-v2-1",
  *     cfgScale: 7,

package/model-provider/stability/StabilityImageGenerationModel.js

@@ -9,7 +9,7 @@ import { failedStabilityCallResponseHandler } from "./StabilityError.js";
  * @see https://api.stability.ai/docs#tag/v1generation/operation/textToImage
  *
  * @example
- * const { image } = await generateImage(
+ * const image = await generateImage(
  *   new StabilityImageGenerationModel({
  *     model: "stable-diffusion-512-v2-1",
  *     cfgScale: 7,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build AI applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [
@@ -65,7 +65,7 @@
     "@typescript-eslint/parser": "^6.1.0",
     "copyfiles": "2.4.1",
     "eslint": "^8.45.0",
-    "eslint-config-prettier": "8.10.0",
+    "eslint-config-prettier": "9.0.0",
     "husky": "^8.0.3",
     "lint-staged": "13.2.3",
     "prettier": "3.0.1",

package/text-chunk/SimilarTextChunksFromVectorIndexRetriever.cjs

@@ -36,7 +36,7 @@ class SimilarTextChunksFromVectorIndexRetriever {
                 run: options.run,
             });
         }
-        const { embedding } = await (0, embedText_js_1.embedText)(this.embeddingModel, query, {
+        const embedding = await (0, embedText_js_1.embedText)(this.embeddingModel, query, {
             functionId: options?.functionId,
             run: options?.run,
         });

package/text-chunk/SimilarTextChunksFromVectorIndexRetriever.js

@@ -33,7 +33,7 @@ export class SimilarTextChunksFromVectorIndexRetriever {
                 run: options.run,
             });
         }
-        const { embedding } = await embedText(this.embeddingModel, query, {
+        const embedding = await embedText(this.embeddingModel, query, {
             functionId: options?.functionId,
             run: options?.run,
         });

package/text-chunk/split/splitOnSeparator.cjs

@@ -1,12 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitOnSeparatorAsSplitFunction = exports.splitOnSeparator = void 0;
-const splitOnSeparator = async ({ separator, text, }) => {
-    return text.split(separator);
-};
+exports.splitOnSeparator = void 0;
+/**
+ * Splits text on a separator string.
+ */
+function splitOnSeparator({ separator, }) {
+    return async ({ text }) => text.split(separator);
+}
 exports.splitOnSeparator = splitOnSeparator;
-const splitOnSeparatorAsSplitFunction = ({ separator }) => async ({ text }) => (0, exports.splitOnSeparator)({
-    separator,
-    text,
-});
-exports.splitOnSeparatorAsSplitFunction = splitOnSeparatorAsSplitFunction;

package/text-chunk/split/splitOnSeparator.d.ts

@@ -1,8 +1,7 @@
 import { SplitFunction } from "./SplitFunction.js";
-export declare const splitOnSeparator: ({ separator, text, }: {
+/**
+ * Splits text on a separator string.
+ */
+export declare function splitOnSeparator({ separator, }: {
     separator: string;
-    text: string;
-}) => Promise<string[]>;
-export declare const splitOnSeparatorAsSplitFunction: ({ separator }: {
-    separator: string;
-}) => SplitFunction;
+}): SplitFunction;

package/text-chunk/split/splitOnSeparator.js

@@ -1,7 +1,6 @@
-export const splitOnSeparator = async ({ separator, text, }) => {
-    return text.split(separator);
-};
-export const splitOnSeparatorAsSplitFunction = ({ separator }) => async ({ text }) => splitOnSeparator({
-    separator,
-    text,
-});
+/**
+ * Splits text on a separator string.
+ */
+export function splitOnSeparator({ separator, }) {
+    return async ({ text }) => text.split(separator);
+}

package/text-chunk/split/splitRecursively.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitRecursivelyAtToken = exports.splitRecursivelyAtCharacter = void 0;
+exports.splitAtToken = exports.splitAtCharacter = void 0;
 // when segments is a string, it splits by character, otherwise according to the provided segments
 function splitRecursively({ maxChunkSize, segments, }) {
     if (segments.length < maxChunkSize) {
@@ -20,13 +20,22 @@ function splitRecursively({ maxChunkSize, segments, }) {
         }),
     ];
 }
-const splitRecursivelyAtCharacter = ({ maxChunkSize }) => async ({ text }) => splitRecursively({
-    maxChunkSize,
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxCharactersPerChunk`.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+const splitAtCharacter = ({ maxCharactersPerChunk, }) => async ({ text }) => splitRecursively({
+    maxChunkSize: maxCharactersPerChunk,
     segments: text,
 });
-exports.splitRecursivelyAtCharacter = splitRecursivelyAtCharacter;
-const splitRecursivelyAtToken = ({ tokenizer, maxChunkSize, }) => async ({ text }) => splitRecursively({
-    maxChunkSize,
+exports.splitAtCharacter = splitAtCharacter;
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxTokensPerChunk`,
+ * while respecting the token boundaries.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+const splitAtToken = ({ tokenizer, maxTokensPerChunk, }) => async ({ text }) => splitRecursively({
+    maxChunkSize: maxTokensPerChunk,
     segments: (await tokenizer.tokenizeWithTexts(text)).tokenTexts,
 });
-exports.splitRecursivelyAtToken = splitRecursivelyAtToken;
+exports.splitAtToken = splitAtToken;