modelfusion 0.4.0 → 0.5.0

package/README.md

@@ -317,12 +317,12 @@ const embeddingModel = new OpenAITextEmbeddingModel({
 await upsertTextChunks({
   vectorIndex,
   embeddingModel,
-  chunks: texts.map((text) => ({ content: text })),
+  chunks: texts.map((text) => ({ text })),
 });
 
 // retrieve text chunks from the vector index - usually done at query time:
 const { chunks } = await retrieveTextChunks(
-  new VectorIndexSimilarTextChunkRetriever({
+  new SimilarTextChunksFromVectorIndexRetriever({
     vectorIndex,
     embeddingModel,
     maxResults: 3,
@@ -440,6 +440,12 @@ TypeScript implementation of the classic [BabyAGI](https://github.com/yoheinakaj
 
 Small agent that solves middle school math problems. It uses a calculator tool to solve the problems.
 
+### [Chat with PDF](https://github.com/lgrammel/modelfusion/tree/main/examples/pdf-chat-terminal)
+
+> _terminal app_, _PDF parsing_, _in memory vector indices_, _retrieval augmented generation_, _hypothetical document embedding_
+
+Ask questions about a PDF document and get answers from the document.
+
 ### [PDF to Tweet](https://github.com/lgrammel/modelfusion/tree/main/examples/pdf-to-tweet)
 
 > _terminal app_, _PDF parsing_, _recursive information extraction_, _in memory vector index, \_style example retrieval_, _OpenAI GPT-4_, _cost calculation_

package/composed-function/summarize/summarizeRecursivelyWithTextGenerationAndTokenSplitting.cjs

@@ -9,17 +9,16 @@ const summarizeRecursively_js_1 = require("./summarizeRecursively.cjs");
  * It automatically splits the text into optimal chunks that are small enough to be processed by the model,
  * while leaving enough space for the model to generate text.
  */
-async function summarizeRecursivelyWithTextGenerationAndTokenSplitting({ text, model, prompt, reservedCompletionTokens, join, }, options) {
+async function summarizeRecursivelyWithTextGenerationAndTokenSplitting({ text, model, prompt, tokenLimit = model.contextWindowSize -
+    (model.maxCompletionTokens ?? model.contextWindowSize / 4), join, }, options) {
     const emptyPromptTokens = await model.countPromptTokens(await prompt({ text: "" }));
     return (0, summarizeRecursively_js_1.summarizeRecursively)({
-        split: (0, splitRecursively_js_1.splitRecursivelyAtTokenAsSplitFunction)({
+        split: (0, splitRecursively_js_1.splitRecursivelyAtToken)({
             tokenizer: model.tokenizer,
-            maxChunkSize: model.contextWindowSize -
-                reservedCompletionTokens -
-                emptyPromptTokens,
+            maxChunkSize: tokenLimit - emptyPromptTokens,
         }),
         summarize: async (input) => {
-            const { text } = await (0, generateText_js_1.generateText)(model.withMaxCompletionTokens(reservedCompletionTokens), await prompt(input), options);
+            const { text } = await (0, generateText_js_1.generateText)(model, await prompt(input), options);
             return text;
         },
         join,

package/composed-function/summarize/summarizeRecursivelyWithTextGenerationAndTokenSplitting.d.ts

@@ -1,4 +1,4 @@
-import { TextGenerationModelSettings, TextGenerationModel } from "../../model-function/generate-text/TextGenerationModel.js";
+import { TextGenerationModel, TextGenerationModelSettings } from "../../model-function/generate-text/TextGenerationModel.js";
 import { FullTokenizer } from "../../model-function/tokenize-text/Tokenizer.js";
 import { Run } from "../../run/Run.js";
 /**
@@ -6,7 +6,7 @@ import { Run } from "../../run/Run.js";
  * It automatically splits the text into optimal chunks that are small enough to be processed by the model,
  * while leaving enough space for the model to generate text.
  */
-export declare function summarizeRecursivelyWithTextGenerationAndTokenSplitting<PROMPT>({ text, model, prompt, reservedCompletionTokens, join, }: {
+export declare function summarizeRecursivelyWithTextGenerationAndTokenSplitting<PROMPT>({ text, model, prompt, tokenLimit, join, }: {
     text: string;
     model: TextGenerationModel<PROMPT, any, any, TextGenerationModelSettings> & {
         contextWindowSize: number;
@@ -16,7 +16,7 @@ export declare function summarizeRecursivelyWithTextGenerationAndTokenSplitting<
     prompt: (input: {
         text: string;
     }) => Promise<PROMPT>;
-    reservedCompletionTokens: number;
+    tokenLimit?: number;
     join?: (texts: Array<string>) => string;
 }, options?: {
     functionId?: string;

package/composed-function/summarize/summarizeRecursivelyWithTextGenerationAndTokenSplitting.js

@@ -1,22 +1,21 @@
 import { generateText } from "../../model-function/generate-text/generateText.js";
-import { splitRecursivelyAtTokenAsSplitFunction } from "../../text-chunk/split/splitRecursively.js";
+import { splitRecursivelyAtToken } from "../../text-chunk/split/splitRecursively.js";
 import { summarizeRecursively } from "./summarizeRecursively.js";
 /**
  * Recursively summarizes a text using a text generation model, e.g. for summarization or text extraction.
  * It automatically splits the text into optimal chunks that are small enough to be processed by the model,
  * while leaving enough space for the model to generate text.
  */
-export async function summarizeRecursivelyWithTextGenerationAndTokenSplitting({ text, model, prompt, reservedCompletionTokens, join, }, options) {
+export async function summarizeRecursivelyWithTextGenerationAndTokenSplitting({ text, model, prompt, tokenLimit = model.contextWindowSize -
+    (model.maxCompletionTokens ?? model.contextWindowSize / 4), join, }, options) {
     const emptyPromptTokens = await model.countPromptTokens(await prompt({ text: "" }));
     return summarizeRecursively({
-        split: splitRecursivelyAtTokenAsSplitFunction({
+        split: splitRecursivelyAtToken({
             tokenizer: model.tokenizer,
-            maxChunkSize: model.contextWindowSize -
-                reservedCompletionTokens -
-                emptyPromptTokens,
+            maxChunkSize: tokenLimit - emptyPromptTokens,
         }),
         summarize: async (input) => {
-            const { text } = await generateText(model.withMaxCompletionTokens(reservedCompletionTokens), await prompt(input), options);
+            const { text } = await generateText(model, await prompt(input), options);
             return text;
         },
         join,

package/model-function/generate-text/streamText.d.ts

@@ -1,7 +1,7 @@
 import { FunctionOptions } from "../FunctionOptions.js";
+import { CallMetadata } from "../executeCall.js";
 import { DeltaEvent } from "./DeltaEvent.js";
 import { TextGenerationModel, TextGenerationModelSettings } from "./TextGenerationModel.js";
-import { CallMetadata } from "model-function/executeCall.js";
 export declare function streamText<PROMPT, FULL_DELTA, SETTINGS extends TextGenerationModelSettings>(model: TextGenerationModel<PROMPT, unknown, FULL_DELTA, SETTINGS> & {
     generateDeltaStreamResponse: (prompt: PROMPT, options: FunctionOptions<SETTINGS>) => PromiseLike<AsyncIterable<DeltaEvent<FULL_DELTA>>>;
     extractTextDelta: (fullDelta: FULL_DELTA) => string | undefined;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build AI applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [

package/prompt/chat/trimChatPrompt.cjs

@@ -12,7 +12,8 @@ const validateChatPrompt_js_1 = require("./validateChatPrompt.cjs");
  *
  * @see https://modelfusion.dev/guide/function/generate-text/prompt-mapping#limiting-the-chat-length
  */
-async function trimChatPrompt({ prompt, model, tokenLimit = model.contextWindowSize - model.maxCompletionTokens, }) {
+async function trimChatPrompt({ prompt, model, tokenLimit = model.contextWindowSize -
+    (model.maxCompletionTokens ?? model.contextWindowSize / 4), }) {
     (0, validateChatPrompt_js_1.validateChatPrompt)(prompt);
     const startsWithSystemMessage = "system" in prompt[0];
     const systemMessage = startsWithSystemMessage ? [prompt[0]] : [];

package/prompt/chat/trimChatPrompt.d.ts

@@ -1,4 +1,4 @@
-import { TextGenerationModel } from "model-function/generate-text/TextGenerationModel.js";
+import { TextGenerationModel } from "../../model-function/generate-text/TextGenerationModel.js";
 import { ChatPrompt } from "./ChatPrompt.js";
 /**
  * Keeps only the most recent messages in the prompt, while leaving enough space for the completion.
@@ -14,7 +14,6 @@ export declare function trimChatPrompt({ prompt, model, tokenLimit, }: {
     prompt: ChatPrompt;
     model: TextGenerationModel<ChatPrompt, any, any, any> & {
         contextWindowSize: number;
-        maxCompletionTokens: number;
         countPromptTokens: (prompt: ChatPrompt) => PromiseLike<number>;
     };
     tokenLimit?: number;

package/prompt/chat/trimChatPrompt.js

@@ -9,7 +9,8 @@ import { validateChatPrompt } from "./validateChatPrompt.js";
  *
  * @see https://modelfusion.dev/guide/function/generate-text/prompt-mapping#limiting-the-chat-length
  */
-export async function trimChatPrompt({ prompt, model, tokenLimit = model.contextWindowSize - model.maxCompletionTokens, }) {
+export async function trimChatPrompt({ prompt, model, tokenLimit = model.contextWindowSize -
+    (model.maxCompletionTokens ?? model.contextWindowSize / 4), }) {
     validateChatPrompt(prompt);
     const startsWithSystemMessage = "system" in prompt[0];
     const systemMessage = startsWithSystemMessage ? [prompt[0]] : [];

package/{vector-index/VectorIndexSimilarTextChunkRetriever.cjs → text-chunk/SimilarTextChunksFromVectorIndexRetriever.cjs}

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.VectorIndexSimilarTextChunkRetriever = void 0;
+exports.SimilarTextChunksFromVectorIndexRetriever = void 0;
 const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
-class VectorIndexSimilarTextChunkRetriever {
+class SimilarTextChunksFromVectorIndexRetriever {
     constructor({ vectorIndex, embeddingModel, maxResults, similarityThreshold, }) {
         Object.defineProperty(this, "vectorIndex", {
             enumerable: true,
@@ -48,10 +48,10 @@ class VectorIndexSimilarTextChunkRetriever {
         return queryResult.map((item) => item.data);
     }
     withSettings(additionalSettings) {
-        return new VectorIndexSimilarTextChunkRetriever(Object.assign({}, this.settings, additionalSettings, {
+        return new SimilarTextChunksFromVectorIndexRetriever(Object.assign({}, this.settings, additionalSettings, {
             vectorIndex: this.vectorIndex,
             embeddingModel: this.embeddingModel,
         }));
     }
 }
-exports.VectorIndexSimilarTextChunkRetriever = VectorIndexSimilarTextChunkRetriever;
+exports.SimilarTextChunksFromVectorIndexRetriever = SimilarTextChunksFromVectorIndexRetriever;

package/{vector-index/VectorIndexSimilarTextChunkRetriever.d.ts → text-chunk/SimilarTextChunksFromVectorIndexRetriever.d.ts}

@@ -1,20 +1,20 @@
 import { FunctionOptions } from "../model-function/FunctionOptions.js";
 import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
-import { TextChunk } from "../text-chunk/TextChunk.js";
-import { TextChunkRetriever, TextChunkRetrieverSettings } from "../text-chunk/retrieve-text-chunks/TextChunkRetriever.js";
-import { VectorIndex } from "./VectorIndex.js";
-export interface VectorIndexTextChunkRetrieverSettings {
+import { TextChunk } from "./TextChunk.js";
+import { TextChunkRetriever, TextChunkRetrieverSettings } from "./retrieve-text-chunks/TextChunkRetriever.js";
+import { VectorIndex } from "../vector-index/VectorIndex.js";
+export interface SimilarTextChunksFromVectorIndexRetrieverSettings {
     maxResults?: number;
     similarityThreshold?: number;
 }
-export declare class VectorIndexSimilarTextChunkRetriever<CHUNK extends TextChunk, INDEX, SETTINGS extends TextEmbeddingModelSettings> implements TextChunkRetriever<CHUNK, string, VectorIndexTextChunkRetrieverSettings> {
+export declare class SimilarTextChunksFromVectorIndexRetriever<CHUNK extends TextChunk, INDEX, SETTINGS extends TextEmbeddingModelSettings> implements TextChunkRetriever<CHUNK, string, SimilarTextChunksFromVectorIndexRetrieverSettings> {
     private readonly vectorIndex;
     private readonly embeddingModel;
     private readonly settings;
     constructor({ vectorIndex, embeddingModel, maxResults, similarityThreshold, }: {
         vectorIndex: VectorIndex<CHUNK, INDEX>;
         embeddingModel: TextEmbeddingModel<unknown, SETTINGS>;
-    } & VectorIndexTextChunkRetrieverSettings);
+    } & SimilarTextChunksFromVectorIndexRetrieverSettings);
     retrieveTextChunks(query: string, options?: FunctionOptions<TextChunkRetrieverSettings>): Promise<CHUNK[]>;
-    withSettings(additionalSettings: Partial<VectorIndexTextChunkRetrieverSettings>): this;
+    withSettings(additionalSettings: Partial<SimilarTextChunksFromVectorIndexRetrieverSettings>): this;
 }

package/{vector-index/VectorIndexSimilarTextChunkRetriever.js → text-chunk/SimilarTextChunksFromVectorIndexRetriever.js}

@@ -1,5 +1,5 @@
 import { embedText } from "../model-function/embed-text/embedText.js";
-export class VectorIndexSimilarTextChunkRetriever {
+export class SimilarTextChunksFromVectorIndexRetriever {
     constructor({ vectorIndex, embeddingModel, maxResults, similarityThreshold, }) {
         Object.defineProperty(this, "vectorIndex", {
             enumerable: true,
@@ -45,7 +45,7 @@ export class VectorIndexSimilarTextChunkRetriever {
         return queryResult.map((item) => item.data);
     }
     withSettings(additionalSettings) {
-        return new VectorIndexSimilarTextChunkRetriever(Object.assign({}, this.settings, additionalSettings, {
+        return new SimilarTextChunksFromVectorIndexRetriever(Object.assign({}, this.settings, additionalSettings, {
             vectorIndex: this.vectorIndex,
             embeddingModel: this.embeddingModel,
         }));

package/text-chunk/TextChunk.d.ts

@@ -1,3 +1,3 @@
 export type TextChunk = {
-    content: string;
+    text: string;
 };

package/text-chunk/index.cjs

@@ -14,9 +14,12 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./SimilarTextChunksFromVectorIndexRetriever.cjs"), exports);
 __exportStar(require("./TextChunk.cjs"), exports);
 __exportStar(require("./retrieve-text-chunks/TextChunkRetriever.cjs"), exports);
 __exportStar(require("./retrieve-text-chunks/retrieveTextChunks.cjs"), exports);
 __exportStar(require("./split/SplitFunction.cjs"), exports);
 __exportStar(require("./split/splitOnSeparator.cjs"), exports);
 __exportStar(require("./split/splitRecursively.cjs"), exports);
+__exportStar(require("./split/splitTextChunks.cjs"), exports);
+__exportStar(require("./upsertTextChunks.cjs"), exports);

package/text-chunk/index.d.ts

@@ -1,6 +1,9 @@
+export * from "./SimilarTextChunksFromVectorIndexRetriever.js";
 export * from "./TextChunk.js";
 export * from "./retrieve-text-chunks/TextChunkRetriever.js";
 export * from "./retrieve-text-chunks/retrieveTextChunks.js";
 export * from "./split/SplitFunction.js";
 export * from "./split/splitOnSeparator.js";
 export * from "./split/splitRecursively.js";
+export * from "./split/splitTextChunks.js";
+export * from "./upsertTextChunks.js";

package/text-chunk/index.js

@@ -1,6 +1,9 @@
+export * from "./SimilarTextChunksFromVectorIndexRetriever.js";
 export * from "./TextChunk.js";
 export * from "./retrieve-text-chunks/TextChunkRetriever.js";
 export * from "./retrieve-text-chunks/retrieveTextChunks.js";
 export * from "./split/SplitFunction.js";
 export * from "./split/splitOnSeparator.js";
 export * from "./split/splitRecursively.js";
+export * from "./split/splitTextChunks.js";
+export * from "./upsertTextChunks.js";

package/text-chunk/split/splitRecursively.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitRecursivelyAtTokenAsSplitFunction = exports.splitRecursivelyAtToken = exports.splitRecursivelyAtCharacterAsSplitFunction = exports.splitRecursivelyAtCharacter = exports.splitRecursively = void 0;
+exports.splitRecursivelyAtToken = exports.splitRecursivelyAtCharacter = void 0;
 // when segments is a string, it splits by character, otherwise according to the provided segments
 function splitRecursively({ maxChunkSize, segments, }) {
     if (segments.length < maxChunkSize) {
@@ -20,22 +20,13 @@ function splitRecursively({ maxChunkSize, segments, }) {
         }),
     ];
 }
-exports.splitRecursively = splitRecursively;
-const splitRecursivelyAtCharacter = async ({ maxChunkSize, text, }) => splitRecursively({
+const splitRecursivelyAtCharacter = ({ maxChunkSize }) => async ({ text }) => splitRecursively({
     maxChunkSize,
     segments: text,
 });
 exports.splitRecursivelyAtCharacter = splitRecursivelyAtCharacter;
-const splitRecursivelyAtCharacterAsSplitFunction = ({ maxChunkSize }) => async ({ text }) => (0, exports.splitRecursivelyAtCharacter)({ maxChunkSize, text });
-exports.splitRecursivelyAtCharacterAsSplitFunction = splitRecursivelyAtCharacterAsSplitFunction;
-const splitRecursivelyAtToken = async ({ tokenizer, maxChunkSize, text, }) => splitRecursively({
+const splitRecursivelyAtToken = ({ tokenizer, maxChunkSize, }) => async ({ text }) => splitRecursively({
     maxChunkSize,
     segments: (await tokenizer.tokenizeWithTexts(text)).tokenTexts,
 });
 exports.splitRecursivelyAtToken = splitRecursivelyAtToken;
-const splitRecursivelyAtTokenAsSplitFunction = ({ tokenizer, maxChunkSize, }) => async ({ text }) => (0, exports.splitRecursivelyAtToken)({
-    tokenizer,
-    maxChunkSize,
-    text,
-});
-exports.splitRecursivelyAtTokenAsSplitFunction = splitRecursivelyAtTokenAsSplitFunction;

package/text-chunk/split/splitRecursively.d.ts

@@ -1,22 +1,9 @@
 import { FullTokenizer } from "../../model-function/tokenize-text/Tokenizer.js";
 import { SplitFunction } from "./SplitFunction.js";
-export declare function splitRecursively({ maxChunkSize, segments, }: {
-    maxChunkSize: number;
-    segments: string | Array<string>;
-}): Array<string>;
-export declare const splitRecursivelyAtCharacter: ({ maxChunkSize, text, }: {
-    maxChunkSize: number;
-    text: string;
-}) => Promise<string[]>;
-export declare const splitRecursivelyAtCharacterAsSplitFunction: ({ maxChunkSize }: {
+export declare const splitRecursivelyAtCharacter: ({ maxChunkSize }: {
     maxChunkSize: number;
 }) => SplitFunction;
-export declare const splitRecursivelyAtToken: ({ tokenizer, maxChunkSize, text, }: {
-    tokenizer: FullTokenizer;
-    maxChunkSize: number;
-    text: string;
-}) => Promise<string[]>;
-export declare const splitRecursivelyAtTokenAsSplitFunction: ({ tokenizer, maxChunkSize, }: {
+export declare const splitRecursivelyAtToken: ({ tokenizer, maxChunkSize, }: {
     tokenizer: FullTokenizer;
     maxChunkSize: number;
 }) => SplitFunction;

package/text-chunk/split/splitRecursively.js

@@ -1,5 +1,5 @@
 // when segments is a string, it splits by character, otherwise according to the provided segments
-export function splitRecursively({ maxChunkSize, segments, }) {
+function splitRecursively({ maxChunkSize, segments, }) {
     if (segments.length < maxChunkSize) {
         return Array.isArray(segments) ? [segments.join("")] : [segments];
     }
@@ -17,17 +17,11 @@ export function splitRecursively({ maxChunkSize, segments, }) {
         }),
     ];
 }
-export const splitRecursivelyAtCharacter = async ({ maxChunkSize, text, }) => splitRecursively({
+export const splitRecursivelyAtCharacter = ({ maxChunkSize }) => async ({ text }) => splitRecursively({
     maxChunkSize,
     segments: text,
 });
-export const splitRecursivelyAtCharacterAsSplitFunction = ({ maxChunkSize }) => async ({ text }) => splitRecursivelyAtCharacter({ maxChunkSize, text });
-export const splitRecursivelyAtToken = async ({ tokenizer, maxChunkSize, text, }) => splitRecursively({
+export const splitRecursivelyAtToken = ({ tokenizer, maxChunkSize, }) => async ({ text }) => splitRecursively({
     maxChunkSize,
     segments: (await tokenizer.tokenizeWithTexts(text)).tokenTexts,
 });
-export const splitRecursivelyAtTokenAsSplitFunction = ({ tokenizer, maxChunkSize, }) => async ({ text }) => splitRecursivelyAtToken({
-    tokenizer,
-    maxChunkSize,
-    text,
-});

package/text-chunk/split/splitTextChunks.cjs

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitTextChunks = void 0;
+async function splitTextChunks(splitFunction, inputs) {
+    const pageChunks = await Promise.all(inputs.map(async (input) => {
+        const parts = await splitFunction(input);
+        return parts.map((text) => ({
+            ...input,
+            text,
+        }));
+    }));
+    return pageChunks.flat();
+}
+exports.splitTextChunks = splitTextChunks;

package/text-chunk/split/splitTextChunks.d.ts

@@ -0,0 +1,3 @@
+import { TextChunk } from "../TextChunk.js";
+import { SplitFunction } from "./SplitFunction.js";
+export declare function splitTextChunks<CHUNK extends TextChunk>(splitFunction: SplitFunction, inputs: CHUNK[]): Promise<CHUNK[]>;

package/text-chunk/split/splitTextChunks.js

@@ -0,0 +1,10 @@
+export async function splitTextChunks(splitFunction, inputs) {
+    const pageChunks = await Promise.all(inputs.map(async (input) => {
+        const parts = await splitFunction(input);
+        return parts.map((text) => ({
+            ...input,
+            text,
+        }));
+    }));
+    return pageChunks.flat();
+}

package/{vector-index → text-chunk}/upsertTextChunks.cjs

@@ -5,7 +5,7 @@ const nanoid_1 = require("nanoid");
 const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
 async function upsertTextChunks({ vectorIndex, embeddingModel, generateId = nanoid_1.nanoid, chunks, ids, }, options) {
     // many embedding models support bulk embedding, so we first embed all texts:
-    const { embeddings } = await (0, embedText_js_1.embedTexts)(embeddingModel, chunks.map((chunk) => chunk.content), options);
+    const { embeddings } = await (0, embedText_js_1.embedTexts)(embeddingModel, chunks.map((chunk) => chunk.text), options);
     await vectorIndex.upsertMany(chunks.map((chunk, i) => ({
         id: ids?.[i] ?? generateId(),
         vector: embeddings[i],

package/{vector-index → text-chunk}/upsertTextChunks.d.ts

@@ -1,7 +1,7 @@
 import { FunctionOptions } from "../model-function/FunctionOptions.js";
 import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
-import { TextChunk } from "../text-chunk/TextChunk.js";
-import { VectorIndex } from "./VectorIndex.js";
+import { TextChunk } from "./TextChunk.js";
+import { VectorIndex } from "../vector-index/VectorIndex.js";
 export declare function upsertTextChunks<CHUNK extends TextChunk, SETTINGS extends TextEmbeddingModelSettings>({ vectorIndex, embeddingModel, generateId, chunks, ids, }: {
     vectorIndex: VectorIndex<CHUNK, unknown>;
     embeddingModel: TextEmbeddingModel<unknown, SETTINGS>;

package/{vector-index → text-chunk}/upsertTextChunks.js

@@ -2,7 +2,7 @@ import { nanoid as createId } from "nanoid";
 import { embedTexts } from "../model-function/embed-text/embedText.js";
 export async function upsertTextChunks({ vectorIndex, embeddingModel, generateId = createId, chunks, ids, }, options) {
     // many embedding models support bulk embedding, so we first embed all texts:
-    const { embeddings } = await embedTexts(embeddingModel, chunks.map((chunk) => chunk.content), options);
+    const { embeddings } = await embedTexts(embeddingModel, chunks.map((chunk) => chunk.text), options);
     await vectorIndex.upsertMany(chunks.map((chunk, i) => ({
         id: ids?.[i] ?? generateId(),
         vector: embeddings[i],

package/vector-index/index.cjs

@@ -15,8 +15,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./VectorIndex.cjs"), exports);
-__exportStar(require("./VectorIndexSimilarTextChunkRetriever.cjs"), exports);
-__exportStar(require("./VectorIndexTextChunkStore.cjs"), exports);
 __exportStar(require("./memory/MemoryVectorIndex.cjs"), exports);
 __exportStar(require("./pinecone/PineconeVectorIndex.cjs"), exports);
-__exportStar(require("./upsertTextChunks.cjs"), exports);

package/vector-index/index.d.ts

@@ -1,6 +1,3 @@
 export * from "./VectorIndex.js";
-export * from "./VectorIndexSimilarTextChunkRetriever.js";
-export * from "./VectorIndexTextChunkStore.js";
 export * from "./memory/MemoryVectorIndex.js";
 export * from "./pinecone/PineconeVectorIndex.js";
-export * from "./upsertTextChunks.js";

package/vector-index/index.js

@@ -1,6 +1,3 @@
 export * from "./VectorIndex.js";
-export * from "./VectorIndexSimilarTextChunkRetriever.js";
-export * from "./VectorIndexTextChunkStore.js";
 export * from "./memory/MemoryVectorIndex.js";
 export * from "./pinecone/PineconeVectorIndex.js";
-export * from "./upsertTextChunks.js";

package/vector-index/VectorIndexTextChunkStore.cjs

@@ -1,77 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.VectorIndexTextChunkStore = void 0;
-const nanoid_1 = require("nanoid");
-const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
-class VectorIndexTextChunkStore {
-    constructor({ index, generateId = nanoid_1.nanoid, embeddingModel, queryFunctionId, upsertFunctionId, }) {
-        Object.defineProperty(this, "_index", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "generateId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "embeddingModel", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "queryFunctionId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "upsertFunctionId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this._index = index;
-        this.generateId = generateId;
-        this.embeddingModel = embeddingModel;
-        this.queryFunctionId = queryFunctionId;
-        this.upsertFunctionId = upsertFunctionId;
-    }
-    async upsertChunk({ id = this.generateId(), chunk, }, options) {
-        this.upsertManyChunks({
-            ids: [id],
-            chunks: [chunk],
-        }, options);
-    }
-    async upsertManyChunks({ ids, chunks, }, options) {
-        const { embeddings } = await (0, embedText_js_1.embedTexts)(this.embeddingModel, chunks.map((chunk) => chunk.content), {
-            functionId: this.upsertFunctionId,
-            run: options?.run,
-        });
-        this._index.upsertMany(embeddings.map((embedding, i) => ({
-            id: ids?.[i] ?? this.generateId(),
-            vector: embedding,
-            data: chunks[i],
-        })));
-    }
-    async retrieveSimilarTextChunks(queryText, options) {
-        const { embedding } = await (0, embedText_js_1.embedText)(this.embeddingModel, queryText, {
-            functionId: this.queryFunctionId,
-            run: options?.run,
-        });
-        const queryResult = await this._index.queryByVector({
-            queryVector: embedding,
-            maxResults: 1,
-            similarityThreshold: undefined,
-        });
-        return queryResult.map((item) => item.data);
-    }
-    get index() {
-        return this._index.asIndex();
-    }
-}
-exports.VectorIndexTextChunkStore = VectorIndexTextChunkStore;

package/vector-index/VectorIndexTextChunkStore.d.ts

@@ -1,35 +0,0 @@
-import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
-import { Run } from "../run/Run.js";
-import { TextChunk } from "../text-chunk/TextChunk.js";
-import { TextChunkRetrieverSettings } from "../text-chunk/retrieve-text-chunks/TextChunkRetriever.js";
-import { VectorIndex } from "./VectorIndex.js";
-import { FunctionOptions } from "../model-function/FunctionOptions.js";
-export declare class VectorIndexTextChunkStore<CHUNK extends TextChunk, INDEX, MODEL extends TextEmbeddingModel<unknown, TextEmbeddingModelSettings>> {
-    private readonly _index;
-    private readonly generateId;
-    private readonly embeddingModel;
-    private readonly queryFunctionId?;
-    private readonly upsertFunctionId?;
-    constructor({ index, generateId, embeddingModel, queryFunctionId, upsertFunctionId, }: {
-        index: VectorIndex<CHUNK, INDEX>;
-        generateId?: () => string;
-        embeddingModel: MODEL;
-        queryFunctionId?: string;
-        upsertFunctionId?: string;
-    });
-    upsertChunk({ id, chunk, }: {
-        id?: string;
-        keyText: string;
-        chunk: CHUNK;
-    }, options?: {
-        run?: Run;
-    }): Promise<void>;
-    upsertManyChunks({ ids, chunks, }: {
-        ids?: Array<string | undefined>;
-        chunks: CHUNK[];
-    }, options?: {
-        run?: Run;
-    }): Promise<void>;
-    retrieveSimilarTextChunks(queryText: string, options?: FunctionOptions<TextChunkRetrieverSettings> | undefined): Promise<CHUNK[]>;
-    get index(): INDEX;
-}

package/vector-index/VectorIndexTextChunkStore.js

@@ -1,73 +0,0 @@
-import { nanoid as createId } from "nanoid";
-import { embedText, embedTexts, } from "../model-function/embed-text/embedText.js";
-export class VectorIndexTextChunkStore {
-    constructor({ index, generateId = createId, embeddingModel, queryFunctionId, upsertFunctionId, }) {
-        Object.defineProperty(this, "_index", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "generateId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "embeddingModel", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "queryFunctionId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "upsertFunctionId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this._index = index;
-        this.generateId = generateId;
-        this.embeddingModel = embeddingModel;
-        this.queryFunctionId = queryFunctionId;
-        this.upsertFunctionId = upsertFunctionId;
-    }
-    async upsertChunk({ id = this.generateId(), chunk, }, options) {
-        this.upsertManyChunks({
-            ids: [id],
-            chunks: [chunk],
-        }, options);
-    }
-    async upsertManyChunks({ ids, chunks, }, options) {
-        const { embeddings } = await embedTexts(this.embeddingModel, chunks.map((chunk) => chunk.content), {
-            functionId: this.upsertFunctionId,
-            run: options?.run,
-        });
-        this._index.upsertMany(embeddings.map((embedding, i) => ({
-            id: ids?.[i] ?? this.generateId(),
-            vector: embedding,
-            data: chunks[i],
-        })));
-    }
-    async retrieveSimilarTextChunks(queryText, options) {
-        const { embedding } = await embedText(this.embeddingModel, queryText, {
-            functionId: this.queryFunctionId,
-            run: options?.run,
-        });
-        const queryResult = await this._index.queryByVector({
-            queryVector: embedding,
-            maxResults: 1,
-            similarityThreshold: undefined,
-        });
-        return queryResult.map((item) => item.data);
-    }
-    get index() {
-        return this._index.asIndex();
-    }
-}