modelfusion 0.4.1 → 0.6.0

package/README.md

@@ -317,12 +317,12 @@ const embeddingModel = new OpenAITextEmbeddingModel({
 await upsertTextChunks({
   vectorIndex,
   embeddingModel,
-  chunks: texts.map((text) => ({ content: text })),
+  chunks: texts.map((text) => ({ text })),
 });
 
 // retrieve text chunks from the vector index - usually done at query time:
 const { chunks } = await retrieveTextChunks(
-  new VectorIndexSimilarTextChunkRetriever({
+  new SimilarTextChunksFromVectorIndexRetriever({
     vectorIndex,
     embeddingModel,
     maxResults: 3,
@@ -343,9 +343,9 @@ const { chunks } = await retrieveTextChunks(
   - [Transcribe Audio](https://modelfusion.dev/guide/function/transcribe-audio)
   - [Generate images](https://modelfusion.dev/guide/function/generate-image)
 - Summarize text
-- Split text
 - [Tools](https://modelfusion.dev/guide/tools)
-- [Text Chunks](https://modelfusion.dev/guide/text-chunks)
+- [Text Chunks](https://modelfusion.dev/guide/text-chunk/)
+  - [Split Text](https://modelfusion.dev/guide/text-chunk/split)
 - [Run abstraction](https://modelfusion.dev/guide/run/)
   - [Abort signals](https://modelfusion.dev/guide/run/abort)
   - [Cost calculation](https://modelfusion.dev/guide/run/cost-calculation)
@@ -416,6 +416,12 @@ Examples for the individual functions and objects.
 
 A web chat with an AI assistant, implemented as a Next.js app.
 
+### [Chat with PDF](https://github.com/lgrammel/modelfusion/tree/main/examples/pdf-chat-terminal)
+
+> _terminal app_, _PDF parsing_, _in memory vector indices_, _retrieval augmented generation_, _hypothetical document embedding_
+
+Ask questions about a PDF document and get answers from the document.
+
 ### [Image generator (Next.js)](https://github.com/lgrammel/modelfusion/tree/main/examples/image-generator-next-js)
 
 > _Next.js app_, _Stability AI image generation_

package/composed-function/summarize/summarizeRecursivelyWithTextGenerationAndTokenSplitting.cjs

@@ -13,9 +13,9 @@ async function summarizeRecursivelyWithTextGenerationAndTokenSplitting({ text, m
     (model.maxCompletionTokens ?? model.contextWindowSize / 4), join, }, options) {
     const emptyPromptTokens = await model.countPromptTokens(await prompt({ text: "" }));
     return (0, summarizeRecursively_js_1.summarizeRecursively)({
-        split: (0, splitRecursively_js_1.splitRecursivelyAtTokenAsSplitFunction)({
+        split: (0, splitRecursively_js_1.splitAtToken)({
             tokenizer: model.tokenizer,
-            maxChunkSize: tokenLimit - emptyPromptTokens,
+            maxTokensPerChunk: tokenLimit - emptyPromptTokens,
         }),
         summarize: async (input) => {
             const { text } = await (0, generateText_js_1.generateText)(model, await prompt(input), options);

package/composed-function/summarize/summarizeRecursivelyWithTextGenerationAndTokenSplitting.d.ts

@@ -1,4 +1,4 @@
-import { TextGenerationModelSettings, TextGenerationModel } from "../../model-function/generate-text/TextGenerationModel.js";
+import { TextGenerationModel, TextGenerationModelSettings } from "../../model-function/generate-text/TextGenerationModel.js";
 import { FullTokenizer } from "../../model-function/tokenize-text/Tokenizer.js";
 import { Run } from "../../run/Run.js";
 /**

package/composed-function/summarize/summarizeRecursivelyWithTextGenerationAndTokenSplitting.js

@@ -1,5 +1,5 @@
 import { generateText } from "../../model-function/generate-text/generateText.js";
-import { splitRecursivelyAtTokenAsSplitFunction } from "../../text-chunk/split/splitRecursively.js";
+import { splitAtToken } from "../../text-chunk/split/splitRecursively.js";
 import { summarizeRecursively } from "./summarizeRecursively.js";
 /**
  * Recursively summarizes a text using a text generation model, e.g. for summarization or text extraction.
@@ -10,9 +10,9 @@ export async function summarizeRecursivelyWithTextGenerationAndTokenSplitting({
     (model.maxCompletionTokens ?? model.contextWindowSize / 4), join, }, options) {
     const emptyPromptTokens = await model.countPromptTokens(await prompt({ text: "" }));
     return summarizeRecursively({
-        split: splitRecursivelyAtTokenAsSplitFunction({
+        split: splitAtToken({
             tokenizer: model.tokenizer,
-            maxChunkSize: tokenLimit - emptyPromptTokens,
+            maxTokensPerChunk: tokenLimit - emptyPromptTokens,
         }),
         summarize: async (input) => {
             const { text } = await generateText(model, await prompt(input), options);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build AI applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.4.1",
+  "version": "0.6.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [
@@ -65,7 +65,7 @@
     "@typescript-eslint/parser": "^6.1.0",
     "copyfiles": "2.4.1",
     "eslint": "^8.45.0",
-    "eslint-config-prettier": "8.10.0",
+    "eslint-config-prettier": "9.0.0",
     "husky": "^8.0.3",
     "lint-staged": "13.2.3",
     "prettier": "3.0.1",

package/{vector-index/VectorIndexSimilarTextChunkRetriever.cjs → text-chunk/SimilarTextChunksFromVectorIndexRetriever.cjs}

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.VectorIndexSimilarTextChunkRetriever = void 0;
+exports.SimilarTextChunksFromVectorIndexRetriever = void 0;
 const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
-class VectorIndexSimilarTextChunkRetriever {
+class SimilarTextChunksFromVectorIndexRetriever {
     constructor({ vectorIndex, embeddingModel, maxResults, similarityThreshold, }) {
         Object.defineProperty(this, "vectorIndex", {
             enumerable: true,
@@ -48,10 +48,10 @@ class VectorIndexSimilarTextChunkRetriever {
         return queryResult.map((item) => item.data);
     }
     withSettings(additionalSettings) {
-        return new VectorIndexSimilarTextChunkRetriever(Object.assign({}, this.settings, additionalSettings, {
+        return new SimilarTextChunksFromVectorIndexRetriever(Object.assign({}, this.settings, additionalSettings, {
             vectorIndex: this.vectorIndex,
             embeddingModel: this.embeddingModel,
         }));
     }
 }
-exports.VectorIndexSimilarTextChunkRetriever = VectorIndexSimilarTextChunkRetriever;
+exports.SimilarTextChunksFromVectorIndexRetriever = SimilarTextChunksFromVectorIndexRetriever;

package/{vector-index/VectorIndexSimilarTextChunkRetriever.d.ts → text-chunk/SimilarTextChunksFromVectorIndexRetriever.d.ts}

@@ -1,20 +1,20 @@
 import { FunctionOptions } from "../model-function/FunctionOptions.js";
 import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
-import { TextChunk } from "../text-chunk/TextChunk.js";
-import { TextChunkRetriever, TextChunkRetrieverSettings } from "../text-chunk/retrieve-text-chunks/TextChunkRetriever.js";
-import { VectorIndex } from "./VectorIndex.js";
-export interface VectorIndexTextChunkRetrieverSettings {
+import { TextChunk } from "./TextChunk.js";
+import { TextChunkRetriever, TextChunkRetrieverSettings } from "./retrieve-text-chunks/TextChunkRetriever.js";
+import { VectorIndex } from "../vector-index/VectorIndex.js";
+export interface SimilarTextChunksFromVectorIndexRetrieverSettings {
     maxResults?: number;
     similarityThreshold?: number;
 }
-export declare class VectorIndexSimilarTextChunkRetriever<CHUNK extends TextChunk, INDEX, SETTINGS extends TextEmbeddingModelSettings> implements TextChunkRetriever<CHUNK, string, VectorIndexTextChunkRetrieverSettings> {
+export declare class SimilarTextChunksFromVectorIndexRetriever<CHUNK extends TextChunk, INDEX, SETTINGS extends TextEmbeddingModelSettings> implements TextChunkRetriever<CHUNK, string, SimilarTextChunksFromVectorIndexRetrieverSettings> {
     private readonly vectorIndex;
     private readonly embeddingModel;
     private readonly settings;
     constructor({ vectorIndex, embeddingModel, maxResults, similarityThreshold, }: {
         vectorIndex: VectorIndex<CHUNK, INDEX>;
         embeddingModel: TextEmbeddingModel<unknown, SETTINGS>;
-    } & VectorIndexTextChunkRetrieverSettings);
+    } & SimilarTextChunksFromVectorIndexRetrieverSettings);
     retrieveTextChunks(query: string, options?: FunctionOptions<TextChunkRetrieverSettings>): Promise<CHUNK[]>;
-    withSettings(additionalSettings: Partial<VectorIndexTextChunkRetrieverSettings>): this;
+    withSettings(additionalSettings: Partial<SimilarTextChunksFromVectorIndexRetrieverSettings>): this;
 }

package/{vector-index/VectorIndexSimilarTextChunkRetriever.js → text-chunk/SimilarTextChunksFromVectorIndexRetriever.js}

@@ -1,5 +1,5 @@
 import { embedText } from "../model-function/embed-text/embedText.js";
-export class VectorIndexSimilarTextChunkRetriever {
+export class SimilarTextChunksFromVectorIndexRetriever {
     constructor({ vectorIndex, embeddingModel, maxResults, similarityThreshold, }) {
         Object.defineProperty(this, "vectorIndex", {
             enumerable: true,
@@ -45,7 +45,7 @@ export class VectorIndexSimilarTextChunkRetriever {
         return queryResult.map((item) => item.data);
     }
     withSettings(additionalSettings) {
-        return new VectorIndexSimilarTextChunkRetriever(Object.assign({}, this.settings, additionalSettings, {
+        return new SimilarTextChunksFromVectorIndexRetriever(Object.assign({}, this.settings, additionalSettings, {
             vectorIndex: this.vectorIndex,
             embeddingModel: this.embeddingModel,
         }));

package/text-chunk/TextChunk.d.ts

@@ -1,3 +1,3 @@
 export type TextChunk = {
-    content: string;
+    text: string;
 };

package/text-chunk/index.cjs

@@ -14,9 +14,12 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./SimilarTextChunksFromVectorIndexRetriever.cjs"), exports);
 __exportStar(require("./TextChunk.cjs"), exports);
 __exportStar(require("./retrieve-text-chunks/TextChunkRetriever.cjs"), exports);
 __exportStar(require("./retrieve-text-chunks/retrieveTextChunks.cjs"), exports);
 __exportStar(require("./split/SplitFunction.cjs"), exports);
 __exportStar(require("./split/splitOnSeparator.cjs"), exports);
 __exportStar(require("./split/splitRecursively.cjs"), exports);
+__exportStar(require("./split/splitTextChunks.cjs"), exports);
+__exportStar(require("./upsertTextChunks.cjs"), exports);

package/text-chunk/index.d.ts

@@ -1,6 +1,9 @@
+export * from "./SimilarTextChunksFromVectorIndexRetriever.js";
 export * from "./TextChunk.js";
 export * from "./retrieve-text-chunks/TextChunkRetriever.js";
 export * from "./retrieve-text-chunks/retrieveTextChunks.js";
 export * from "./split/SplitFunction.js";
 export * from "./split/splitOnSeparator.js";
 export * from "./split/splitRecursively.js";
+export * from "./split/splitTextChunks.js";
+export * from "./upsertTextChunks.js";

package/text-chunk/index.js

@@ -1,6 +1,9 @@
+export * from "./SimilarTextChunksFromVectorIndexRetriever.js";
 export * from "./TextChunk.js";
 export * from "./retrieve-text-chunks/TextChunkRetriever.js";
 export * from "./retrieve-text-chunks/retrieveTextChunks.js";
 export * from "./split/SplitFunction.js";
 export * from "./split/splitOnSeparator.js";
 export * from "./split/splitRecursively.js";
+export * from "./split/splitTextChunks.js";
+export * from "./upsertTextChunks.js";

package/text-chunk/split/splitOnSeparator.cjs

@@ -1,12 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitOnSeparatorAsSplitFunction = exports.splitOnSeparator = void 0;
-const splitOnSeparator = async ({ separator, text, }) => {
-    return text.split(separator);
-};
+exports.splitOnSeparator = void 0;
+/**
+ * Splits text on a separator string.
+ */
+function splitOnSeparator({ separator, }) {
+    return async ({ text }) => text.split(separator);
+}
 exports.splitOnSeparator = splitOnSeparator;
-const splitOnSeparatorAsSplitFunction = ({ separator }) => async ({ text }) => (0, exports.splitOnSeparator)({
-    separator,
-    text,
-});
-exports.splitOnSeparatorAsSplitFunction = splitOnSeparatorAsSplitFunction;

package/text-chunk/split/splitOnSeparator.d.ts

@@ -1,8 +1,7 @@
 import { SplitFunction } from "./SplitFunction.js";
-export declare const splitOnSeparator: ({ separator, text, }: {
+/**
+ * Splits text on a separator string.
+ */
+export declare function splitOnSeparator({ separator, }: {
     separator: string;
-    text: string;
-}) => Promise<string[]>;
-export declare const splitOnSeparatorAsSplitFunction: ({ separator }: {
-    separator: string;
-}) => SplitFunction;
+}): SplitFunction;

package/text-chunk/split/splitOnSeparator.js

@@ -1,7 +1,6 @@
-export const splitOnSeparator = async ({ separator, text, }) => {
-    return text.split(separator);
-};
-export const splitOnSeparatorAsSplitFunction = ({ separator }) => async ({ text }) => splitOnSeparator({
-    separator,
-    text,
-});
+/**
+ * Splits text on a separator string.
+ */
+export function splitOnSeparator({ separator, }) {
+    return async ({ text }) => text.split(separator);
+}

package/text-chunk/split/splitRecursively.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitRecursivelyAtTokenAsSplitFunction = exports.splitRecursivelyAtToken = exports.splitRecursivelyAtCharacterAsSplitFunction = exports.splitRecursivelyAtCharacter = exports.splitRecursively = void 0;
+exports.splitAtToken = exports.splitAtCharacter = void 0;
 // when segments is a string, it splits by character, otherwise according to the provided segments
 function splitRecursively({ maxChunkSize, segments, }) {
     if (segments.length < maxChunkSize) {
@@ -20,22 +20,22 @@ function splitRecursively({ maxChunkSize, segments, }) {
         }),
     ];
 }
-exports.splitRecursively = splitRecursively;
-const splitRecursivelyAtCharacter = async ({ maxChunkSize, text, }) => splitRecursively({
-    maxChunkSize,
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxCharactersPerChunk`.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+const splitAtCharacter = ({ maxCharactersPerChunk, }) => async ({ text }) => splitRecursively({
+    maxChunkSize: maxCharactersPerChunk,
     segments: text,
 });
-exports.splitRecursivelyAtCharacter = splitRecursivelyAtCharacter;
-const splitRecursivelyAtCharacterAsSplitFunction = ({ maxChunkSize }) => async ({ text }) => (0, exports.splitRecursivelyAtCharacter)({ maxChunkSize, text });
-exports.splitRecursivelyAtCharacterAsSplitFunction = splitRecursivelyAtCharacterAsSplitFunction;
-const splitRecursivelyAtToken = async ({ tokenizer, maxChunkSize, text, }) => splitRecursively({
-    maxChunkSize,
+exports.splitAtCharacter = splitAtCharacter;
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxTokensPerChunk`,
+ * while respecting the token boundaries.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+const splitAtToken = ({ tokenizer, maxTokensPerChunk, }) => async ({ text }) => splitRecursively({
+    maxChunkSize: maxTokensPerChunk,
     segments: (await tokenizer.tokenizeWithTexts(text)).tokenTexts,
 });
-exports.splitRecursivelyAtToken = splitRecursivelyAtToken;
-const splitRecursivelyAtTokenAsSplitFunction = ({ tokenizer, maxChunkSize, }) => async ({ text }) => (0, exports.splitRecursivelyAtToken)({
-    tokenizer,
-    maxChunkSize,
-    text,
-});
-exports.splitRecursivelyAtTokenAsSplitFunction = splitRecursivelyAtTokenAsSplitFunction;
+exports.splitAtToken = splitAtToken;

package/text-chunk/split/splitRecursively.d.ts

@@ -1,22 +1,18 @@
 import { FullTokenizer } from "../../model-function/tokenize-text/Tokenizer.js";
 import { SplitFunction } from "./SplitFunction.js";
-export declare function splitRecursively({ maxChunkSize, segments, }: {
-    maxChunkSize: number;
-    segments: string | Array<string>;
-}): Array<string>;
-export declare const splitRecursivelyAtCharacter: ({ maxChunkSize, text, }: {
-    maxChunkSize: number;
-    text: string;
-}) => Promise<string[]>;
-export declare const splitRecursivelyAtCharacterAsSplitFunction: ({ maxChunkSize }: {
-    maxChunkSize: number;
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxCharactersPerChunk`.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+export declare const splitAtCharacter: ({ maxCharactersPerChunk, }: {
+    maxCharactersPerChunk: number;
 }) => SplitFunction;
-export declare const splitRecursivelyAtToken: ({ tokenizer, maxChunkSize, text, }: {
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxTokensPerChunk`,
+ * while respecting the token boundaries.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+export declare const splitAtToken: ({ tokenizer, maxTokensPerChunk, }: {
     tokenizer: FullTokenizer;
-    maxChunkSize: number;
-    text: string;
-}) => Promise<string[]>;
-export declare const splitRecursivelyAtTokenAsSplitFunction: ({ tokenizer, maxChunkSize, }: {
-    tokenizer: FullTokenizer;
-    maxChunkSize: number;
+    maxTokensPerChunk: number;
 }) => SplitFunction;

package/text-chunk/split/splitRecursively.js

@@ -1,5 +1,5 @@
 // when segments is a string, it splits by character, otherwise according to the provided segments
-export function splitRecursively({ maxChunkSize, segments, }) {
+function splitRecursively({ maxChunkSize, segments, }) {
     if (segments.length < maxChunkSize) {
         return Array.isArray(segments) ? [segments.join("")] : [segments];
     }
@@ -17,17 +17,20 @@ export function splitRecursively({ maxChunkSize, segments, }) {
         }),
     ];
 }
-export const splitRecursivelyAtCharacter = async ({ maxChunkSize, text, }) => splitRecursively({
-    maxChunkSize,
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxCharactersPerChunk`.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+export const splitAtCharacter = ({ maxCharactersPerChunk, }) => async ({ text }) => splitRecursively({
+    maxChunkSize: maxCharactersPerChunk,
     segments: text,
 });
-export const splitRecursivelyAtCharacterAsSplitFunction = ({ maxChunkSize }) => async ({ text }) => splitRecursivelyAtCharacter({ maxChunkSize, text });
-export const splitRecursivelyAtToken = async ({ tokenizer, maxChunkSize, text, }) => splitRecursively({
-    maxChunkSize,
+/**
+ * Splits text recursively until the resulting chunks are smaller than the `maxTokensPerChunk`,
+ * while respecting the token boundaries.
+ * The text is recursively split in the middle, so that all chunks are roughtly the same size.
+ */
+export const splitAtToken = ({ tokenizer, maxTokensPerChunk, }) => async ({ text }) => splitRecursively({
+    maxChunkSize: maxTokensPerChunk,
     segments: (await tokenizer.tokenizeWithTexts(text)).tokenTexts,
 });
-export const splitRecursivelyAtTokenAsSplitFunction = ({ tokenizer, maxChunkSize, }) => async ({ text }) => splitRecursivelyAtToken({
-    tokenizer,
-    maxChunkSize,
-    text,
-});

package/text-chunk/split/splitTextChunks.cjs

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitTextChunk = exports.splitTextChunks = void 0;
+async function splitTextChunks(splitFunction, inputs) {
+    const pageChunks = await Promise.all(inputs.map((input) => splitTextChunk(splitFunction, input)));
+    return pageChunks.flat();
+}
+exports.splitTextChunks = splitTextChunks;
+async function splitTextChunk(splitFunction, input) {
+    const parts = await splitFunction(input);
+    return parts.map((text) => ({
+        ...input,
+        text,
+    }));
+}
+exports.splitTextChunk = splitTextChunk;

package/text-chunk/split/splitTextChunks.d.ts

@@ -0,0 +1,4 @@
+import { TextChunk } from "../TextChunk.js";
+import { SplitFunction } from "./SplitFunction.js";
+export declare function splitTextChunks<CHUNK extends TextChunk>(splitFunction: SplitFunction, inputs: CHUNK[]): Promise<CHUNK[]>;
+export declare function splitTextChunk<CHUNK extends TextChunk>(splitFunction: SplitFunction, input: CHUNK): Promise<CHUNK[]>;

package/text-chunk/split/splitTextChunks.js

@@ -0,0 +1,11 @@
+export async function splitTextChunks(splitFunction, inputs) {
+    const pageChunks = await Promise.all(inputs.map((input) => splitTextChunk(splitFunction, input)));
+    return pageChunks.flat();
+}
+export async function splitTextChunk(splitFunction, input) {
+    const parts = await splitFunction(input);
+    return parts.map((text) => ({
+        ...input,
+        text,
+    }));
+}

package/{vector-index → text-chunk}/upsertTextChunks.cjs

@@ -5,7 +5,7 @@ const nanoid_1 = require("nanoid");
 const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
 async function upsertTextChunks({ vectorIndex, embeddingModel, generateId = nanoid_1.nanoid, chunks, ids, }, options) {
     // many embedding models support bulk embedding, so we first embed all texts:
-    const { embeddings } = await (0, embedText_js_1.embedTexts)(embeddingModel, chunks.map((chunk) => chunk.content), options);
+    const { embeddings } = await (0, embedText_js_1.embedTexts)(embeddingModel, chunks.map((chunk) => chunk.text), options);
     await vectorIndex.upsertMany(chunks.map((chunk, i) => ({
         id: ids?.[i] ?? generateId(),
         vector: embeddings[i],

package/{vector-index → text-chunk}/upsertTextChunks.d.ts

@@ -1,7 +1,7 @@
 import { FunctionOptions } from "../model-function/FunctionOptions.js";
 import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
-import { TextChunk } from "../text-chunk/TextChunk.js";
-import { VectorIndex } from "./VectorIndex.js";
+import { TextChunk } from "./TextChunk.js";
+import { VectorIndex } from "../vector-index/VectorIndex.js";
 export declare function upsertTextChunks<CHUNK extends TextChunk, SETTINGS extends TextEmbeddingModelSettings>({ vectorIndex, embeddingModel, generateId, chunks, ids, }: {
     vectorIndex: VectorIndex<CHUNK, unknown>;
     embeddingModel: TextEmbeddingModel<unknown, SETTINGS>;

package/{vector-index → text-chunk}/upsertTextChunks.js

@@ -2,7 +2,7 @@ import { nanoid as createId } from "nanoid";
 import { embedTexts } from "../model-function/embed-text/embedText.js";
 export async function upsertTextChunks({ vectorIndex, embeddingModel, generateId = createId, chunks, ids, }, options) {
     // many embedding models support bulk embedding, so we first embed all texts:
-    const { embeddings } = await embedTexts(embeddingModel, chunks.map((chunk) => chunk.content), options);
+    const { embeddings } = await embedTexts(embeddingModel, chunks.map((chunk) => chunk.text), options);
     await vectorIndex.upsertMany(chunks.map((chunk, i) => ({
         id: ids?.[i] ?? generateId(),
         vector: embeddings[i],

package/vector-index/index.cjs

@@ -15,8 +15,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./VectorIndex.cjs"), exports);
-__exportStar(require("./VectorIndexSimilarTextChunkRetriever.cjs"), exports);
-__exportStar(require("./VectorIndexTextChunkStore.cjs"), exports);
 __exportStar(require("./memory/MemoryVectorIndex.cjs"), exports);
 __exportStar(require("./pinecone/PineconeVectorIndex.cjs"), exports);
-__exportStar(require("./upsertTextChunks.cjs"), exports);

package/vector-index/index.d.ts

@@ -1,6 +1,3 @@
 export * from "./VectorIndex.js";
-export * from "./VectorIndexSimilarTextChunkRetriever.js";
-export * from "./VectorIndexTextChunkStore.js";
 export * from "./memory/MemoryVectorIndex.js";
 export * from "./pinecone/PineconeVectorIndex.js";
-export * from "./upsertTextChunks.js";

package/vector-index/index.js

@@ -1,6 +1,3 @@
 export * from "./VectorIndex.js";
-export * from "./VectorIndexSimilarTextChunkRetriever.js";
-export * from "./VectorIndexTextChunkStore.js";
 export * from "./memory/MemoryVectorIndex.js";
 export * from "./pinecone/PineconeVectorIndex.js";
-export * from "./upsertTextChunks.js";

package/vector-index/VectorIndexTextChunkStore.cjs

@@ -1,77 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.VectorIndexTextChunkStore = void 0;
-const nanoid_1 = require("nanoid");
-const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
-class VectorIndexTextChunkStore {
-    constructor({ index, generateId = nanoid_1.nanoid, embeddingModel, queryFunctionId, upsertFunctionId, }) {
-        Object.defineProperty(this, "_index", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "generateId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "embeddingModel", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "queryFunctionId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "upsertFunctionId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this._index = index;
-        this.generateId = generateId;
-        this.embeddingModel = embeddingModel;
-        this.queryFunctionId = queryFunctionId;
-        this.upsertFunctionId = upsertFunctionId;
-    }
-    async upsertChunk({ id = this.generateId(), chunk, }, options) {
-        this.upsertManyChunks({
-            ids: [id],
-            chunks: [chunk],
-        }, options);
-    }
-    async upsertManyChunks({ ids, chunks, }, options) {
-        const { embeddings } = await (0, embedText_js_1.embedTexts)(this.embeddingModel, chunks.map((chunk) => chunk.content), {
-            functionId: this.upsertFunctionId,
-            run: options?.run,
-        });
-        this._index.upsertMany(embeddings.map((embedding, i) => ({
-            id: ids?.[i] ?? this.generateId(),
-            vector: embedding,
-            data: chunks[i],
-        })));
-    }
-    async retrieveSimilarTextChunks(queryText, options) {
-        const { embedding } = await (0, embedText_js_1.embedText)(this.embeddingModel, queryText, {
-            functionId: this.queryFunctionId,
-            run: options?.run,
-        });
-        const queryResult = await this._index.queryByVector({
-            queryVector: embedding,
-            maxResults: 1,
-            similarityThreshold: undefined,
-        });
-        return queryResult.map((item) => item.data);
-    }
-    get index() {
-        return this._index.asIndex();
-    }
-}
-exports.VectorIndexTextChunkStore = VectorIndexTextChunkStore;

package/vector-index/VectorIndexTextChunkStore.d.ts

@@ -1,35 +0,0 @@
-import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
-import { Run } from "../run/Run.js";
-import { TextChunk } from "../text-chunk/TextChunk.js";
-import { TextChunkRetrieverSettings } from "../text-chunk/retrieve-text-chunks/TextChunkRetriever.js";
-import { VectorIndex } from "./VectorIndex.js";
-import { FunctionOptions } from "../model-function/FunctionOptions.js";
-export declare class VectorIndexTextChunkStore<CHUNK extends TextChunk, INDEX, MODEL extends TextEmbeddingModel<unknown, TextEmbeddingModelSettings>> {
-    private readonly _index;
-    private readonly generateId;
-    private readonly embeddingModel;
-    private readonly queryFunctionId?;
-    private readonly upsertFunctionId?;
-    constructor({ index, generateId, embeddingModel, queryFunctionId, upsertFunctionId, }: {
-        index: VectorIndex<CHUNK, INDEX>;
-        generateId?: () => string;
-        embeddingModel: MODEL;
-        queryFunctionId?: string;
-        upsertFunctionId?: string;
-    });
-    upsertChunk({ id, chunk, }: {
-        id?: string;
-        keyText: string;
-        chunk: CHUNK;
-    }, options?: {
-        run?: Run;
-    }): Promise<void>;
-    upsertManyChunks({ ids, chunks, }: {
-        ids?: Array<string | undefined>;
-        chunks: CHUNK[];
-    }, options?: {
-        run?: Run;
-    }): Promise<void>;
-    retrieveSimilarTextChunks(queryText: string, options?: FunctionOptions<TextChunkRetrieverSettings> | undefined): Promise<CHUNK[]>;
-    get index(): INDEX;
-}

package/vector-index/VectorIndexTextChunkStore.js

@@ -1,73 +0,0 @@
-import { nanoid as createId } from "nanoid";
-import { embedText, embedTexts, } from "../model-function/embed-text/embedText.js";
-export class VectorIndexTextChunkStore {
-    constructor({ index, generateId = createId, embeddingModel, queryFunctionId, upsertFunctionId, }) {
-        Object.defineProperty(this, "_index", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "generateId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "embeddingModel", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "queryFunctionId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "upsertFunctionId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this._index = index;
-        this.generateId = generateId;
-        this.embeddingModel = embeddingModel;
-        this.queryFunctionId = queryFunctionId;
-        this.upsertFunctionId = upsertFunctionId;
-    }
-    async upsertChunk({ id = this.generateId(), chunk, }, options) {
-        this.upsertManyChunks({
-            ids: [id],
-            chunks: [chunk],
-        }, options);
-    }
-    async upsertManyChunks({ ids, chunks, }, options) {
-        const { embeddings } = await embedTexts(this.embeddingModel, chunks.map((chunk) => chunk.content), {
-            functionId: this.upsertFunctionId,
-            run: options?.run,
-        });
-        this._index.upsertMany(embeddings.map((embedding, i) => ({
-            id: ids?.[i] ?? this.generateId(),
-            vector: embedding,
-            data: chunks[i],
-        })));
-    }
-    async retrieveSimilarTextChunks(queryText, options) {
-        const { embedding } = await embedText(this.embeddingModel, queryText, {
-            functionId: this.queryFunctionId,
-            run: options?.run,
-        });
-        const queryResult = await this._index.queryByVector({
-            queryVector: embedding,
-            maxResults: 1,
-            similarityThreshold: undefined,
-        });
-        return queryResult.map((item) => item.data);
-    }
-    get index() {
-        return this._index.asIndex();
-    }
-}