modelfusion 0.133.0 → 0.135.0

package/model-function/generate-text/prompt-template/index.cjs

@@ -1,43 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.VicunaPrompt = exports.TextPrompt = exports.SynthiaPrompt = exports.NeuralChatPrompt = exports.MistralInstructPrompt = exports.Llama2Prompt = exports.ChatMLPrompt = exports.AlpacaPrompt = void 0;
-exports.AlpacaPrompt = __importStar(require("./AlpacaPromptTemplate.cjs"));
-exports.ChatMLPrompt = __importStar(require("./ChatMLPromptTemplate.cjs"));
-__exportStar(require("./ChatPrompt.cjs"), exports);
-__exportStar(require("./ContentPart.cjs"), exports);
-__exportStar(require("./InstructionPrompt.cjs"), exports);
-__exportStar(require("./InvalidPromptError.cjs"), exports);
-exports.Llama2Prompt = __importStar(require("./Llama2PromptTemplate.cjs"));
-exports.MistralInstructPrompt = __importStar(require("./MistralInstructPromptTemplate.cjs"));
-exports.NeuralChatPrompt = __importStar(require("./NeuralChatPromptTemplate.cjs"));
-__exportStar(require("./PromptTemplateProvider.cjs"), exports);
-exports.SynthiaPrompt = __importStar(require("./SynthiaPromptTemplate.cjs"));
-exports.TextPrompt = __importStar(require("./TextPromptTemplate.cjs"));
-exports.VicunaPrompt = __importStar(require("./VicunaPromptTemplate.cjs"));
-__exportStar(require("./trimChatPrompt.cjs"), exports);

package/model-function/generate-text/prompt-template/index.d.ts

@@ -1,14 +0,0 @@
-export * as AlpacaPrompt from "./AlpacaPromptTemplate.js";
-export * as ChatMLPrompt from "./ChatMLPromptTemplate.js";
-export * from "./ChatPrompt.js";
-export * from "./ContentPart.js";
-export * from "./InstructionPrompt.js";
-export * from "./InvalidPromptError.js";
-export * as Llama2Prompt from "./Llama2PromptTemplate.js";
-export * as MistralInstructPrompt from "./MistralInstructPromptTemplate.js";
-export * as NeuralChatPrompt from "./NeuralChatPromptTemplate.js";
-export * from "./PromptTemplateProvider.js";
-export * as SynthiaPrompt from "./SynthiaPromptTemplate.js";
-export * as TextPrompt from "./TextPromptTemplate.js";
-export * as VicunaPrompt from "./VicunaPromptTemplate.js";
-export * from "./trimChatPrompt.js";

package/model-function/generate-text/prompt-template/index.js

@@ -1,14 +0,0 @@
-export * as AlpacaPrompt from "./AlpacaPromptTemplate.js";
-export * as ChatMLPrompt from "./ChatMLPromptTemplate.js";
-export * from "./ChatPrompt.js";
-export * from "./ContentPart.js";
-export * from "./InstructionPrompt.js";
-export * from "./InvalidPromptError.js";
-export * as Llama2Prompt from "./Llama2PromptTemplate.js";
-export * as MistralInstructPrompt from "./MistralInstructPromptTemplate.js";
-export * as NeuralChatPrompt from "./NeuralChatPromptTemplate.js";
-export * from "./PromptTemplateProvider.js";
-export * as SynthiaPrompt from "./SynthiaPromptTemplate.js";
-export * as TextPrompt from "./TextPromptTemplate.js";
-export * as VicunaPrompt from "./VicunaPromptTemplate.js";
-export * from "./trimChatPrompt.js";

package/model-function/generate-text/prompt-template/trimChatPrompt.cjs

@@ -1,46 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.trimChatPrompt = void 0;
-/**
- * Keeps only the most recent messages in the prompt, while leaving enough space for the completion.
- *
- * It will remove user-ai message pairs that don't fit. The result is always a valid chat prompt.
- *
- * When the minimal chat prompt (system message + last user message) is already too long, it will only
- * return this minimal chat prompt.
- *
- * @see https://modelfusion.dev/guide/function/generate-text#limiting-the-chat-length
- */
-async function trimChatPrompt({ prompt, model, tokenLimit = model.contextWindowSize -
-    (model.settings.maxGenerationTokens ?? model.contextWindowSize / 4), }) {
-    let minimalPrompt = {
-        system: prompt.system,
-        messages: [prompt.messages[prompt.messages.length - 1]], // last user message
-    };
-    // check if the minimal prompt is already too long
-    const promptTokenCount = await model.countPromptTokens(minimalPrompt);
-    // the minimal chat prompt is already over the token limit and cannot be trimmed further:
-    if (promptTokenCount > tokenLimit) {
-        return minimalPrompt;
-    }
-    // inner messages
-    const innerMessages = prompt.messages.slice(0, -1);
-    // taking always a pair of user-message and ai-message from the end, moving backwards
-    for (let i = innerMessages.length - 1; i >= 0; i -= 2) {
-        const assistantMessage = innerMessages[i];
-        const userMessage = innerMessages[i - 1];
-        // create a temporary prompt and check if it fits within the token limit
-        const attemptedPrompt = {
-            system: prompt.system,
-            messages: [userMessage, assistantMessage, ...minimalPrompt.messages],
-        };
-        const tokenCount = await model.countPromptTokens(attemptedPrompt);
-        if (tokenCount > tokenLimit) {
-            break;
-        }
-        // if it fits, its the new minimal prompt
-        minimalPrompt = attemptedPrompt;
-    }
-    return minimalPrompt;
-}
-exports.trimChatPrompt = trimChatPrompt;

package/model-function/generate-text/prompt-template/trimChatPrompt.d.ts

@@ -1,17 +0,0 @@
-import { HasContextWindowSize, HasTokenizer, TextGenerationModel, TextGenerationModelSettings } from "../TextGenerationModel.js";
-import { ChatPrompt } from "./ChatPrompt.js";
-/**
- * Keeps only the most recent messages in the prompt, while leaving enough space for the completion.
- *
- * It will remove user-ai message pairs that don't fit. The result is always a valid chat prompt.
- *
- * When the minimal chat prompt (system message + last user message) is already too long, it will only
- * return this minimal chat prompt.
- *
- * @see https://modelfusion.dev/guide/function/generate-text#limiting-the-chat-length
- */
-export declare function trimChatPrompt({ prompt, model, tokenLimit, }: {
-    prompt: ChatPrompt;
-    model: TextGenerationModel<ChatPrompt, TextGenerationModelSettings> & HasTokenizer<ChatPrompt> & HasContextWindowSize;
-    tokenLimit?: number;
-}): Promise<ChatPrompt>;

package/model-function/generate-text/prompt-template/trimChatPrompt.js

@@ -1,42 +0,0 @@
-/**
- * Keeps only the most recent messages in the prompt, while leaving enough space for the completion.
- *
- * It will remove user-ai message pairs that don't fit. The result is always a valid chat prompt.
- *
- * When the minimal chat prompt (system message + last user message) is already too long, it will only
- * return this minimal chat prompt.
- *
- * @see https://modelfusion.dev/guide/function/generate-text#limiting-the-chat-length
- */
-export async function trimChatPrompt({ prompt, model, tokenLimit = model.contextWindowSize -
-    (model.settings.maxGenerationTokens ?? model.contextWindowSize / 4), }) {
-    let minimalPrompt = {
-        system: prompt.system,
-        messages: [prompt.messages[prompt.messages.length - 1]], // last user message
-    };
-    // check if the minimal prompt is already too long
-    const promptTokenCount = await model.countPromptTokens(minimalPrompt);
-    // the minimal chat prompt is already over the token limit and cannot be trimmed further:
-    if (promptTokenCount > tokenLimit) {
-        return minimalPrompt;
-    }
-    // inner messages
-    const innerMessages = prompt.messages.slice(0, -1);
-    // taking always a pair of user-message and ai-message from the end, moving backwards
-    for (let i = innerMessages.length - 1; i >= 0; i -= 2) {
-        const assistantMessage = innerMessages[i];
-        const userMessage = innerMessages[i - 1];
-        // create a temporary prompt and check if it fits within the token limit
-        const attemptedPrompt = {
-            system: prompt.system,
-            messages: [userMessage, assistantMessage, ...minimalPrompt.messages],
-        };
-        const tokenCount = await model.countPromptTokens(attemptedPrompt);
-        if (tokenCount > tokenLimit) {
-            break;
-        }
-        // if it fits, its the new minimal prompt
-        minimalPrompt = attemptedPrompt;
-    }
-    return minimalPrompt;
-}

package/model-function/generate-text/streamText.cjs

@@ -1,54 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.streamText = void 0;
-const executeStreamCall_js_1 = require("../executeStreamCall.cjs");
-async function streamText({ model, prompt, fullResponse, ...options }) {
-    const shouldTrimWhitespace = model.settings.trimWhitespace ?? true;
-    let accumulatedText = "";
-    let isFirstDelta = true;
-    let trailingWhitespace = "";
-    let resolveText;
-    const textPromise = new Promise((resolve) => {
-        resolveText = resolve;
-    });
-    const callResponse = await (0, executeStreamCall_js_1.executeStreamCall)({
-        functionType: "stream-text",
-        input: prompt,
-        model,
-        options,
-        startStream: async (options) => model.doStreamText(prompt, options),
-        processDelta: (delta) => {
-            let textDelta = model.extractTextDelta(delta.deltaValue);
-            if (textDelta == null || textDelta.length === 0) {
-                return undefined;
-            }
-            if (shouldTrimWhitespace) {
-                textDelta = isFirstDelta
-                    ? // remove leading whitespace:
-                        textDelta.trimStart()
-                    : // restore trailing whitespace from previous chunk:
-                        trailingWhitespace + textDelta;
-                // trim trailing whitespace and store it for the next chunk:
-                const trailingWhitespaceMatch = textDelta.match(/\s+$/);
-                trailingWhitespace = trailingWhitespaceMatch
-                    ? trailingWhitespaceMatch[0]
-                    : "";
-                textDelta = textDelta.trimEnd();
-            }
-            isFirstDelta = false;
-            accumulatedText += textDelta;
-            return textDelta;
-        },
-        onDone: () => {
-            resolveText(accumulatedText);
-        },
-    });
-    return fullResponse
-        ? {
-            textStream: callResponse.value,
-            textPromise,
-            metadata: callResponse.metadata,
-        }
-        : callResponse.value;
-}
-exports.streamText = streamText;

package/model-function/generate-text/streamText.d.ts

@@ -1,42 +0,0 @@
-import { FunctionOptions } from "../../core/FunctionOptions.js";
-import { ModelCallMetadata } from "../ModelCallMetadata.js";
-import { TextStreamingModel } from "./TextGenerationModel.js";
-/**
- * Stream the generated text for a prompt as an async iterable.
- *
- * The prompt depends on the model used.
- * For instance, OpenAI completion models expect a string prompt,
- * whereas OpenAI chat models expect an array of chat messages.
- *
- * @see https://modelfusion.dev/guide/function/generate-text
- *
- * @example
- * const textStream = await streamText({
- *   model: openai.CompletionTextGenerator(...),
- *   prompt: "Write a short story about a robot learning to love:\n\n"
- * });
- *
- * for await (const textPart of textStream) {
- *   // ...
- * }
- *
- * @param {TextStreamingModel<PROMPT>} model - The model to stream text from.
- * @param {PROMPT} prompt - The prompt to use for text generation.
- * @param {FunctionOptions} [options] - Optional parameters for the function.
- *
- * @returns {AsyncIterableResultPromise<string>} An async iterable promise that yields the generated text.
- */
-export declare function streamText<PROMPT>(args: {
-    model: TextStreamingModel<PROMPT>;
-    prompt: PROMPT;
-    fullResponse?: false;
-} & FunctionOptions): Promise<AsyncIterable<string>>;
-export declare function streamText<PROMPT>(args: {
-    model: TextStreamingModel<PROMPT>;
-    prompt: PROMPT;
-    fullResponse: true;
-} & FunctionOptions): Promise<{
-    textStream: AsyncIterable<string>;
-    textPromise: PromiseLike<string>;
-    metadata: Omit<ModelCallMetadata, "durationInMs" | "finishTimestamp">;
-}>;

package/model-function/generate-text/streamText.js

@@ -1,50 +0,0 @@
-import { executeStreamCall } from "../executeStreamCall.js";
-export async function streamText({ model, prompt, fullResponse, ...options }) {
-    const shouldTrimWhitespace = model.settings.trimWhitespace ?? true;
-    let accumulatedText = "";
-    let isFirstDelta = true;
-    let trailingWhitespace = "";
-    let resolveText;
-    const textPromise = new Promise((resolve) => {
-        resolveText = resolve;
-    });
-    const callResponse = await executeStreamCall({
-        functionType: "stream-text",
-        input: prompt,
-        model,
-        options,
-        startStream: async (options) => model.doStreamText(prompt, options),
-        processDelta: (delta) => {
-            let textDelta = model.extractTextDelta(delta.deltaValue);
-            if (textDelta == null || textDelta.length === 0) {
-                return undefined;
-            }
-            if (shouldTrimWhitespace) {
-                textDelta = isFirstDelta
-                    ? // remove leading whitespace:
-                        textDelta.trimStart()
-                    : // restore trailing whitespace from previous chunk:
-                        trailingWhitespace + textDelta;
-                // trim trailing whitespace and store it for the next chunk:
-                const trailingWhitespaceMatch = textDelta.match(/\s+$/);
-                trailingWhitespace = trailingWhitespaceMatch
-                    ? trailingWhitespaceMatch[0]
-                    : "";
-                textDelta = textDelta.trimEnd();
-            }
-            isFirstDelta = false;
-            accumulatedText += textDelta;
-            return textDelta;
-        },
-        onDone: () => {
-            resolveText(accumulatedText);
-        },
-    });
-    return fullResponse
-        ? {
-            textStream: callResponse.value,
-            textPromise,
-            metadata: callResponse.metadata,
-        }
-        : callResponse.value;
-}

package/model-function/generate-transcription/TranscriptionEvent.cjs

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/generate-transcription/TranscriptionEvent.d.ts

@@ -1,18 +0,0 @@
-import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelCallEvent.js";
-export interface TranscriptionStartedEvent extends BaseModelCallStartedEvent {
-    functionType: "generate-transcription";
-}
-export type TranscriptionFinishedEventResult = {
-    status: "success";
-    rawResponse: unknown;
-    value: string;
-} | {
-    status: "error";
-    error: unknown;
-} | {
-    status: "abort";
-};
-export interface TranscriptionFinishedEvent extends BaseModelCallFinishedEvent {
-    functionType: "generate-transcription";
-    result: TranscriptionFinishedEventResult;
-}

package/model-function/generate-transcription/TranscriptionEvent.js

@@ -1 +0,0 @@
-export {};

package/model-function/generate-transcription/TranscriptionModel.cjs

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/generate-transcription/TranscriptionModel.d.ts

@@ -1,14 +0,0 @@
-import { DataContent } from "../../util/format/DataContent.js";
-import { FunctionCallOptions } from "../../core/FunctionOptions.js";
-import { Model, ModelSettings } from "../Model.js";
-export interface TranscriptionModelSettings extends ModelSettings {
-}
-export interface TranscriptionModel<SETTINGS extends TranscriptionModelSettings = TranscriptionModelSettings> extends Model<SETTINGS> {
-    doTranscribe: (input: {
-        mimeType: string;
-        audioData: DataContent;
-    }, options: FunctionCallOptions) => PromiseLike<{
-        rawResponse: unknown;
-        transcription: string;
-    }>;
-}

package/model-function/generate-transcription/TranscriptionModel.js

@@ -1 +0,0 @@
-export {};

package/model-function/generate-transcription/generateTranscription.cjs

@@ -1,22 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateTranscription = void 0;
-const executeStandardCall_js_1 = require("../executeStandardCall.cjs");
-async function generateTranscription({ model, audioData, mimeType, fullResponse, ...options }) {
-    const input = { mimeType, audioData };
-    const callResponse = await (0, executeStandardCall_js_1.executeStandardCall)({
-        functionType: "generate-transcription",
-        input,
-        model,
-        options,
-        generateResponse: async (options) => {
-            const result = await model.doTranscribe(input, options);
-            return {
-                rawResponse: result.rawResponse,
-                extractedValue: result.transcription,
-            };
-        },
-    });
-    return fullResponse ? callResponse : callResponse.value;
-}
-exports.generateTranscription = generateTranscription;

package/model-function/generate-transcription/generateTranscription.d.ts

@@ -1,41 +0,0 @@
-import { FunctionOptions } from "../../core/FunctionOptions.js";
-import { AudioMimeType } from "../../util/audio/AudioMimeType.js";
-import { DataContent } from "../../util/format/DataContent.js";
-import { ModelCallMetadata } from "../ModelCallMetadata.js";
-import { TranscriptionModel, TranscriptionModelSettings } from "./TranscriptionModel.js";
-/**
- * Transcribe audio data into text. Also called speech-to-text (STT) or automatic speech recognition (ASR).
- *
- * @see https://modelfusion.dev/guide/function/generate-transcription
- *
- * @example
- * const audioData = await fs.promises.readFile("data/test.mp3");
- *
- * const transcription = await generateTranscription({
- *   model: openai.Transcriber({ model: "whisper-1" }),
- *   mimeType: "audio/mp3",
- *   audioData,
- * });
- *
- * @param {TranscriptionModel<DATA, TranscriptionModelSettings>} options.model - The model to use for transcription.
- * @param {AudioMimeType} options.model - The MIME type of the audio data.
- * @param {DataContent} options.model - The audio data to transcribe. Can be a base64-encoded string, a Uint8Array, or a Buffer.
- *
- * @returns {Promise<string>} A promise that resolves to the transcribed text.
- */
-export declare function generateTranscription(args: {
-    model: TranscriptionModel<TranscriptionModelSettings>;
-    mimeType: AudioMimeType | (string & {});
-    audioData: DataContent;
-    fullResponse?: false;
-} & FunctionOptions): Promise<string>;
-export declare function generateTranscription(args: {
-    model: TranscriptionModel<TranscriptionModelSettings>;
-    mimeType: AudioMimeType | (string & {});
-    audioData: DataContent;
-    fullResponse: true;
-} & FunctionOptions): Promise<{
-    value: string;
-    rawResponse: unknown;
-    metadata: ModelCallMetadata;
-}>;

package/model-function/generate-transcription/generateTranscription.js

@@ -1,18 +0,0 @@
-import { executeStandardCall } from "../executeStandardCall.js";
-export async function generateTranscription({ model, audioData, mimeType, fullResponse, ...options }) {
-    const input = { mimeType, audioData };
-    const callResponse = await executeStandardCall({
-        functionType: "generate-transcription",
-        input,
-        model,
-        options,
-        generateResponse: async (options) => {
-            const result = await model.doTranscribe(input, options);
-            return {
-                rawResponse: result.rawResponse,
-                extractedValue: result.transcription,
-            };
-        },
-    });
-    return fullResponse ? callResponse : callResponse.value;
-}

package/model-function/index.cjs

@@ -1,38 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./Delta.cjs"), exports);
-__exportStar(require("./Model.cjs"), exports);
-__exportStar(require("./ModelCallEvent.cjs"), exports);
-__exportStar(require("./ModelCallMetadata.cjs"), exports);
-__exportStar(require("./ModelInformation.cjs"), exports);
-__exportStar(require("./PromptTemplate.cjs"), exports);
-__exportStar(require("./classify/index.cjs"), exports);
-__exportStar(require("./embed/EmbeddingEvent.cjs"), exports);
-__exportStar(require("./embed/EmbeddingModel.cjs"), exports);
-__exportStar(require("./embed/embed.cjs"), exports);
-__exportStar(require("./generate-image/ImageGenerationEvent.cjs"), exports);
-__exportStar(require("./generate-image/ImageGenerationModel.cjs"), exports);
-__exportStar(require("./generate-image/PromptTemplateImageGenerationModel.cjs"), exports);
-__exportStar(require("./generate-image/generateImage.cjs"), exports);
-__exportStar(require("./generate-speech/index.cjs"), exports);
-__exportStar(require("./generate-object/index.cjs"), exports);
-__exportStar(require("./generate-text/index.cjs"), exports);
-__exportStar(require("./generate-transcription/TranscriptionEvent.cjs"), exports);
-__exportStar(require("./generate-transcription/TranscriptionModel.cjs"), exports);
-__exportStar(require("./generate-transcription/generateTranscription.cjs"), exports);
-__exportStar(require("./tokenize-text/Tokenizer.cjs"), exports);
-__exportStar(require("./tokenize-text/countTokens.cjs"), exports);

package/model-function/index.d.ts

@@ -1,22 +0,0 @@
-export * from "./Delta.js";
-export * from "./Model.js";
-export * from "./ModelCallEvent.js";
-export * from "./ModelCallMetadata.js";
-export * from "./ModelInformation.js";
-export * from "./PromptTemplate.js";
-export * from "./classify/index.js";
-export * from "./embed/EmbeddingEvent.js";
-export * from "./embed/EmbeddingModel.js";
-export * from "./embed/embed.js";
-export * from "./generate-image/ImageGenerationEvent.js";
-export * from "./generate-image/ImageGenerationModel.js";
-export * from "./generate-image/PromptTemplateImageGenerationModel.js";
-export * from "./generate-image/generateImage.js";
-export * from "./generate-speech/index.js";
-export * from "./generate-object/index.js";
-export * from "./generate-text/index.js";
-export * from "./generate-transcription/TranscriptionEvent.js";
-export * from "./generate-transcription/TranscriptionModel.js";
-export * from "./generate-transcription/generateTranscription.js";
-export * from "./tokenize-text/Tokenizer.js";
-export * from "./tokenize-text/countTokens.js";

package/model-function/index.js

@@ -1,22 +0,0 @@
-export * from "./Delta.js";
-export * from "./Model.js";
-export * from "./ModelCallEvent.js";
-export * from "./ModelCallMetadata.js";
-export * from "./ModelInformation.js";
-export * from "./PromptTemplate.js";
-export * from "./classify/index.js";
-export * from "./embed/EmbeddingEvent.js";
-export * from "./embed/EmbeddingModel.js";
-export * from "./embed/embed.js";
-export * from "./generate-image/ImageGenerationEvent.js";
-export * from "./generate-image/ImageGenerationModel.js";
-export * from "./generate-image/PromptTemplateImageGenerationModel.js";
-export * from "./generate-image/generateImage.js";
-export * from "./generate-speech/index.js";
-export * from "./generate-object/index.js";
-export * from "./generate-text/index.js";
-export * from "./generate-transcription/TranscriptionEvent.js";
-export * from "./generate-transcription/TranscriptionModel.js";
-export * from "./generate-transcription/generateTranscription.js";
-export * from "./tokenize-text/Tokenizer.js";
-export * from "./tokenize-text/countTokens.js";

package/model-function/tokenize-text/Tokenizer.cjs

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/tokenize-text/Tokenizer.d.ts

@@ -1,43 +0,0 @@
-/**
- * Interface for a basic tokenizer capable of converting text into tokens.
- *
- * This serves as the base for tokenization functionalities where the focus is on the transformation of input text into a series of numeric tokens.
- */
-export interface BasicTokenizer {
-    /**
-     * Asynchronously tokenize the given text into a sequence of numeric tokens.
-     *
-     * @param text - Input text string that needs to be tokenized.
-     * @returns A promise containing an array of numbers, where each number is a token representing a part or the whole of the input text.
-     */
-    tokenize: (text: string) => PromiseLike<Array<number>>;
-}
-/**
- * Interface for a comprehensive tokenizer that extends the basic tokenization capabilities.
- *
- * In addition to basic tokenization, this interface provides methods for detokenization and
- * retrieving the original text corresponding to each token, enabling a more informative and reversible transformation process.
- */
-export interface FullTokenizer extends BasicTokenizer {
-    /**
-     * Asynchronously tokenize the given text, providing both the numeric tokens and their corresponding text.
-     *
-     * @param text - Input text string to be tokenized.
-     * @returns A promise containing an object with two arrays:
-     *          1. `tokens` - An array of numbers where each number is a token.
-     *          2. `tokenTexts` - An array of strings where each string represents the original text corresponding to each token.
-     */
-    tokenizeWithTexts: (text: string) => PromiseLike<{
-        tokens: Array<number>;
-        tokenTexts: Array<string>;
-    }>;
-    /**
-     * Asynchronously revert a sequence of numeric tokens back into the original text.
-     * Detokenization is the process of transforming tokens back to a human-readable format, and it's essential in scenarios
-     * where the output needs to be interpretable or when the tokenization process has to be reversible.
-     *
-     * @param tokens - An array of numeric tokens to be converted back to text.
-     * @returns A promise containing a string that represents the original text corresponding to the sequence of input tokens.
-     */
-    detokenize: (tokens: Array<number>) => PromiseLike<string>;
-}

package/model-function/tokenize-text/Tokenizer.js

@@ -1 +0,0 @@
-export {};

package/model-function/tokenize-text/countTokens.cjs

@@ -1,10 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.countTokens = void 0;
-/**
- * Count the number of tokens in the given text.
- */
-async function countTokens(tokenizer, text) {
-    return (await tokenizer.tokenize(text)).length;
-}
-exports.countTokens = countTokens;

package/model-function/tokenize-text/countTokens.d.ts

@@ -1,5 +0,0 @@
-import { BasicTokenizer } from "./Tokenizer.js";
-/**
- * Count the number of tokens in the given text.
- */
-export declare function countTokens(tokenizer: BasicTokenizer, text: string): Promise<number>;

package/model-function/tokenize-text/countTokens.js

@@ -1,6 +0,0 @@
-/**
- * Count the number of tokens in the given text.
- */
-export async function countTokens(tokenizer, text) {
-    return (await tokenizer.tokenize(text)).length;
-}