modelfusion 0.5.0 → 0.6.0

package/README.md

@@ -343,9 +343,9 @@ const { chunks } = await retrieveTextChunks(
   - [Transcribe Audio](https://modelfusion.dev/guide/function/transcribe-audio)
   - [Generate images](https://modelfusion.dev/guide/function/generate-image)
 - Summarize text
-- Split text
 - [Tools](https://modelfusion.dev/guide/tools)
-- [Text Chunks](https://modelfusion.dev/guide/text-chunks)
+- [Text Chunks](https://modelfusion.dev/guide/text-chunk/)
+  - [Split Text](https://modelfusion.dev/guide/text-chunk/split)
 - [Run abstraction](https://modelfusion.dev/guide/run/)
   - [Abort signals](https://modelfusion.dev/guide/run/abort)
   - [Cost calculation](https://modelfusion.dev/guide/run/cost-calculation)
@@ -416,6 +416,12 @@ Examples for the individual functions and objects.
 
 A web chat with an AI assistant, implemented as a Next.js app.
 
+### [Chat with PDF](https://github.com/lgrammel/modelfusion/tree/main/examples/pdf-chat-terminal)
+
+> _terminal app_, _PDF parsing_, _in memory vector indices_, _retrieval augmented generation_, _hypothetical document embedding_
+
+Ask questions about a PDF document and get answers from the document.
+
 ### [Image generator (Next.js)](https://github.com/lgrammel/modelfusion/tree/main/examples/image-generator-next-js)
 
 > _Next.js app_, _Stability AI image generation_
@@ -440,12 +446,6 @@ TypeScript implementation of the classic [BabyAGI](https://github.com/yoheinakaj
 
 Small agent that solves middle school math problems. It uses a calculator tool to solve the problems.
 
-### [Chat with PDF](https://github.com/lgrammel/modelfusion/tree/main/examples/pdf-chat-terminal)
-
-> _terminal app_, _PDF parsing_, _in memory vector indices_, _retrieval augmented generation_, _hypothetical document embedding_
-
-Ask questions about a PDF document and get answers from the document.
-
 ### [PDF to Tweet](https://github.com/lgrammel/modelfusion/tree/main/examples/pdf-to-tweet)
 
 > _terminal app_, _PDF parsing_, _recursive information extraction_, _in memory vector index, \_style example retrieval_, _OpenAI GPT-4_, _cost calculation_

package/composed-function/summarize/summarizeRecursivelyWithTextGenerationAndTokenSplitting.cjs

@@ -13,9 +13,9 @@ async function summarizeRecursivelyWithTextGenerationAndTokenSplitting({ text, m
     (model.maxCompletionTokens ?? model.contextWindowSize / 4), join, }, options) {
     const emptyPromptTokens = await model.countPromptTokens(await prompt({ text: "" }));
     return (0, summarizeRecursively_js_1.summarizeRecursively)({
-        split: (0, splitRecursively_js_1.splitRecursivelyAtToken)({
+        split: (0, splitRecursively_js_1.splitAtToken)({
             tokenizer: model.tokenizer,
-            maxChunkSize: tokenLimit - emptyPromptTokens,
+            maxTokensPerChunk: tokenLimit - emptyPromptTokens,
         }),
         summarize: async (input) => {
             const { text } = await (0, generateText_js_1.generateText)(model, await prompt(input), options);

package/composed-function/summarize/summarizeRecursivelyWithTextGenerationAndTokenSplitting.js

@@ -1,5 +1,5 @@
 import { generateText } from "../../model-function/generate-text/generateText.js";
-import { splitRecursivelyAtToken } from "../../text-chunk/split/splitRecursively.js";
+import { splitAtToken } from "../../text-chunk/split/splitRecursively.js";
 import { summarizeRecursively } from "./summarizeRecursively.js";
 /**
  * Recursively summarizes a text using a text generation model, e.g. for summarization or text extraction.
@@ -10,9 +10,9 @@ export async function summarizeRecursivelyWithTextGenerationAndTokenSplitting({
     (model.maxCompletionTokens ?? model.contextWindowSize / 4), join, }, options) {
     const emptyPromptTokens = await model.countPromptTokens(await prompt({ text: "" }));
     return summarizeRecursively({
-        split: splitRecursivelyAtToken({
+        split: splitAtToken({
             tokenizer: model.tokenizer,
-            maxChunkSize: tokenLimit - emptyPromptTokens,
+            maxTokensPerChunk: tokenLimit - emptyPromptTokens,
         }),
         summarize: async (input) => {
             const { text } = await generateText(model, await prompt(input), options);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build AI applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [
@@ -65,7 +65,7 @@
     "@typescript-eslint/parser": "^6.1.0",
     "copyfiles": "2.4.1",
     "eslint": "^8.45.0",
-    "eslint-config-prettier": "8.10.0",
+    "eslint-config-prettier": "9.0.0",
     "husky": "^8.0.3",
     "lint-staged": "13.2.3",
     "prettier": "3.0.1",

package/text-chunk/split/splitOnSeparator.cjs

@@ -1,12 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitOnSeparatorAsSplitFunction = exports.splitOnSeparator = void 0;
-const splitOnSeparator = async ({ separator, text, }) => {
-    return text.split(separator);
-};
+exports.splitOnSeparator = void 0;
+/**
+ * Splits text on a separator string.
+ */
+function splitOnSeparator({ separator, }) {
+    return async ({ text }) => text.split(separator);
+}
 exports.splitOnSeparator = splitOnSeparator;
-const splitOnSeparatorAsSplitFunction = ({ separator }) => async ({ text }) => (0, exports.splitOnSeparator)({
-    separator,
-    text,
-});
-exports.splitOnSeparatorAsSplitFunction = splitOnSeparatorAsSplitFunction;

package/text-chunk/split/splitOnSeparator.d.ts

@@ -1,8 +1,7 @@
 import { SplitFunction } from "./SplitFunction.js";
-export declare const splitOnSeparator: ({ separator, text, }: {
+/**
+ * Splits text on a separator string.
+ */
+export declare function splitOnSeparator({ separator, }: {
     separator: string;
-    text: string;
-}) => Promise<string[]>;
-export declare const splitOnSeparatorAsSplitFunction: ({ separator }: {
-    separator: string;
-}) => SplitFunction;
+}): SplitFunction;

package/text-chunk/split/splitOnSeparator.js

@@ -1,7 +1,6 @@
-export const splitOnSeparator = async ({ separator, text, }) => {
-    return text.split(separator);
-};
-export const splitOnSeparatorAsSplitFunction = ({ separator }) => async ({ text }) => splitOnSeparator({
-    separator,
-    text,
-});
+/**
+ * Splits text on a separator string.
+ */
+export function splitOnSeparator({ separator, }) {
+    return async ({ text }) => text.split(separator);
+}

package/text-chunk/split/splitRecursively.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitRecursivelyAtToken = exports.splitRecursivelyAtCharacter = void 0;
+exports.splitAtToken = exports.splitAtCharacter = void 0;
 // when segments is a string, it splits by character, otherwise according to the provided segments
 function splitRecursively({ maxChunkSize, segments, }) {
     if (segments.length < maxChunkSize) {
@@ -20,13 +20,22 @@ function splitRecursively({ maxChunkSize, segments, }) {
         }),
     ];
 }
-const splitRecursivelyAtCharacter = ({ maxChunkSize }) => async ({ text }) => splitRecursively({
-    maxChunkSize,
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxCharactersPerChunk`.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+const splitAtCharacter = ({ maxCharactersPerChunk, }) => async ({ text }) => splitRecursively({
+    maxChunkSize: maxCharactersPerChunk,
     segments: text,
 });
-exports.splitRecursivelyAtCharacter = splitRecursivelyAtCharacter;
-const splitRecursivelyAtToken = ({ tokenizer, maxChunkSize, }) => async ({ text }) => splitRecursively({
-    maxChunkSize,
+exports.splitAtCharacter = splitAtCharacter;
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxTokensPerChunk`,
+ * while respecting the token boundaries.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+const splitAtToken = ({ tokenizer, maxTokensPerChunk, }) => async ({ text }) => splitRecursively({
+    maxChunkSize: maxTokensPerChunk,
     segments: (await tokenizer.tokenizeWithTexts(text)).tokenTexts,
 });
-exports.splitRecursivelyAtToken = splitRecursivelyAtToken;
+exports.splitAtToken = splitAtToken;

package/text-chunk/split/splitRecursively.d.ts

@@ -1,9 +1,18 @@
 import { FullTokenizer } from "../../model-function/tokenize-text/Tokenizer.js";
 import { SplitFunction } from "./SplitFunction.js";
-export declare const splitRecursivelyAtCharacter: ({ maxChunkSize }: {
-    maxChunkSize: number;
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxCharactersPerChunk`.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+export declare const splitAtCharacter: ({ maxCharactersPerChunk, }: {
+    maxCharactersPerChunk: number;
 }) => SplitFunction;
-export declare const splitRecursivelyAtToken: ({ tokenizer, maxChunkSize, }: {
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxTokensPerChunk`,
+ * while respecting the token boundaries.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+export declare const splitAtToken: ({ tokenizer, maxTokensPerChunk, }: {
     tokenizer: FullTokenizer;
-    maxChunkSize: number;
+    maxTokensPerChunk: number;
 }) => SplitFunction;

package/text-chunk/split/splitRecursively.js

@@ -17,11 +17,20 @@ function splitRecursively({ maxChunkSize, segments, }) {
         }),
     ];
 }
-export const splitRecursivelyAtCharacter = ({ maxChunkSize }) => async ({ text }) => splitRecursively({
-    maxChunkSize,
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxCharactersPerChunk`.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+export const splitAtCharacter = ({ maxCharactersPerChunk, }) => async ({ text }) => splitRecursively({
+    maxChunkSize: maxCharactersPerChunk,
     segments: text,
 });
-export const splitRecursivelyAtToken = ({ tokenizer, maxChunkSize, }) => async ({ text }) => splitRecursively({
-    maxChunkSize,
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxTokensPerChunk`,
+ * while respecting the token boundaries.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+export const splitAtToken = ({ tokenizer, maxTokensPerChunk, }) => async ({ text }) => splitRecursively({
+    maxChunkSize: maxTokensPerChunk,
     segments: (await tokenizer.tokenizeWithTexts(text)).tokenTexts,
 });

package/text-chunk/split/splitTextChunks.cjs

@@ -1,14 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitTextChunks = void 0;
+exports.splitTextChunk = exports.splitTextChunks = void 0;
 async function splitTextChunks(splitFunction, inputs) {
-    const pageChunks = await Promise.all(inputs.map(async (input) => {
-        const parts = await splitFunction(input);
-        return parts.map((text) => ({
-            ...input,
-            text,
-        }));
-    }));
+    const pageChunks = await Promise.all(inputs.map((input) => splitTextChunk(splitFunction, input)));
     return pageChunks.flat();
 }
 exports.splitTextChunks = splitTextChunks;
+async function splitTextChunk(splitFunction, input) {
+    const parts = await splitFunction(input);
+    return parts.map((text) => ({
+        ...input,
+        text,
+    }));
+}
+exports.splitTextChunk = splitTextChunk;

package/text-chunk/split/splitTextChunks.d.ts

@@ -1,3 +1,4 @@
 import { TextChunk } from "../TextChunk.js";
 import { SplitFunction } from "./SplitFunction.js";
 export declare function splitTextChunks<CHUNK extends TextChunk>(splitFunction: SplitFunction, inputs: CHUNK[]): Promise<CHUNK[]>;
+export declare function splitTextChunk<CHUNK extends TextChunk>(splitFunction: SplitFunction, input: CHUNK): Promise<CHUNK[]>;

package/text-chunk/split/splitTextChunks.js

@@ -1,10 +1,11 @@
 export async function splitTextChunks(splitFunction, inputs) {
-    const pageChunks = await Promise.all(inputs.map(async (input) => {
-        const parts = await splitFunction(input);
-        return parts.map((text) => ({
-            ...input,
-            text,
-        }));
-    }));
+    const pageChunks = await Promise.all(inputs.map((input) => splitTextChunk(splitFunction, input)));
     return pageChunks.flat();
 }
+export async function splitTextChunk(splitFunction, input) {
+    const parts = await splitFunction(input);
+    return parts.map((text) => ({
+        ...input,
+        text,
+    }));
+}