modelfusion 0.28.0 → 0.29.0

package/README.md

@@ -314,6 +314,21 @@ const speech = await synthesizeSpeech(
 
 Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs), [LMNT](https://modelfusion.dev/integration/model-provider/lmnt)
 
+### [Describe Image](https://modelfusion.dev/guide/function/describe-image)
+
+Describe an image as text, e.g. for image captioning or OCR.
+
+```ts
+const text = await describeImage(
+  new HuggingFaceImageDescriptionModel({
+    model: "nlpconnect/vit-gpt2-image-captioning",
+  }),
+  data: buffer
+);
+```
+
+Providers: [HuggingFace](/integration/model-provider/huggingface)
+
 ### [Generate Image](https://modelfusion.dev/guide/function/generate-image)
 
 Generate a base64-encoded image from a prompt.
@@ -412,7 +427,8 @@ Integrations: [Helicone](https://modelfusion.dev/integration/observability/helic
   - [Tokenize Text](https://modelfusion.dev/guide/function/tokenize-text)
   - [Transcribe Speech](https://modelfusion.dev/guide/function/transcribe-speech)
   - [Synthesize Speech](https://modelfusion.dev/guide/function/synthesize-speech)
-  - [Generate images](https://modelfusion.dev/guide/function/generate-image)
+  - [Describe Image](https://modelfusion.dev/guide/function/describe-image)
+  - [Generate Image](https://modelfusion.dev/guide/function/generate-image)
 - [Tools](https://modelfusion.dev/guide/tools)
 - [Text Chunks](https://modelfusion.dev/guide/text-chunk/)
   - [Split Text](https://modelfusion.dev/guide/text-chunk/split)

package/core/api/postToApi.d.ts

@@ -21,7 +21,7 @@ export declare const postToApi: <T>({ url, headers, body, successfulResponseHand
     url: string;
     headers?: Record<string, string> | undefined;
     body: {
-        content: string | FormData;
+        content: string | FormData | Buffer;
         values: unknown;
     };
     failedResponseHandler: ResponseHandler<Error>;

package/model-function/ModelCallEvent.d.ts

@@ -1,5 +1,6 @@
 import { BaseFunctionFinishedEvent, BaseFunctionStartedEvent } from "../core/FunctionEvent.js";
 import { ModelInformation } from "./ModelInformation.js";
+import { ImageDescriptionFinishedEvent, ImageDescriptionStartedEvent } from "./describe-image/ImageDescriptionEvent.js";
 import { TextEmbeddingFinishedEvent, TextEmbeddingStartedEvent } from "./embed-text/TextEmbeddingEvent.js";
 import { ImageGenerationFinishedEvent, ImageGenerationStartedEvent } from "./generate-image/ImageGenerationEvent.js";
 import { JsonGenerationFinishedEvent, JsonGenerationStartedEvent } from "./generate-json/JsonGenerationEvent.js";
@@ -47,5 +48,5 @@ export interface BaseModelCallFinishedEvent extends BaseFunctionFinishedEvent {
      */
     result: BaseModelCallFinishedEventResult;
 }
-export type ModelCallStartedEvent = ImageGenerationStartedEvent | JsonGenerationStartedEvent | SpeechSynthesisStartedEvent | TextEmbeddingStartedEvent | TextGenerationStartedEvent | TextStreamingStartedEvent | TranscriptionStartedEvent;
-export type ModelCallFinishedEvent = ImageGenerationFinishedEvent | JsonGenerationFinishedEvent | SpeechSynthesisFinishedEvent | TextEmbeddingFinishedEvent | TextGenerationFinishedEvent | TextStreamingFinishedEvent | TranscriptionFinishedEvent;
+export type ModelCallStartedEvent = ImageDescriptionStartedEvent | ImageGenerationStartedEvent | JsonGenerationStartedEvent | SpeechSynthesisStartedEvent | TextEmbeddingStartedEvent | TextGenerationStartedEvent | TextStreamingStartedEvent | TranscriptionStartedEvent;
+export type ModelCallFinishedEvent = ImageDescriptionFinishedEvent | ImageGenerationFinishedEvent | JsonGenerationFinishedEvent | SpeechSynthesisFinishedEvent | TextEmbeddingFinishedEvent | TextGenerationFinishedEvent | TextStreamingFinishedEvent | TranscriptionFinishedEvent;

package/model-function/SuccessfulModelCall.d.ts

@@ -1,7 +1,7 @@
 import { FunctionEvent } from "../core/FunctionEvent.js";
 import { ModelInformation } from "./ModelInformation.js";
 export type SuccessfulModelCall = {
-    type: "image-generation" | "json-generation" | "json-or-text-generation" | "speech-synthesis" | "text-embedding" | "text-generation" | "text-streaming" | "transcription";
+    type: "image-description" | "image-generation" | "json-generation" | "json-or-text-generation" | "speech-synthesis" | "text-embedding" | "text-generation" | "text-streaming" | "transcription";
     model: ModelInformation;
     settings: unknown;
     response: unknown;

package/model-function/describe-image/ImageDescriptionEvent.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/describe-image/ImageDescriptionEvent.d.ts

@@ -0,0 +1,18 @@
+import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelCallEvent.js";
+export interface ImageDescriptionStartedEvent extends BaseModelCallStartedEvent {
+    functionType: "image-description";
+}
+export type ImageDescriptionFinishedEventResult = {
+    status: "success";
+    response: unknown;
+    output: string;
+} | {
+    status: "error";
+    error: unknown;
+} | {
+    status: "abort";
+};
+export interface ImageDescriptionFinishedEvent extends BaseModelCallFinishedEvent {
+    functionType: "image-description";
+    result: ImageDescriptionFinishedEventResult;
+}

package/model-function/describe-image/ImageDescriptionEvent.js

@@ -0,0 +1 @@
+export {};

package/model-function/describe-image/ImageDescriptionModel.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/describe-image/ImageDescriptionModel.d.ts

@@ -0,0 +1,8 @@
+import { ModelFunctionOptions } from "../ModelFunctionOptions.js";
+import { Model, ModelSettings } from "../Model.js";
+export interface ImageDescriptionModelSettings extends ModelSettings {
+}
+export interface ImageDescriptionModel<DATA, RESPONSE, SETTINGS extends ImageDescriptionModelSettings> extends Model<SETTINGS> {
+    generateImageDescriptionResponse: (data: DATA, options?: ModelFunctionOptions<SETTINGS>) => PromiseLike<RESPONSE>;
+    extractImageDescription: (response: RESPONSE) => string;
+}

package/model-function/describe-image/ImageDescriptionModel.js

@@ -0,0 +1 @@
+export {};

package/model-function/describe-image/describeImage.cjs

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.describeImage = void 0;
+const executeCall_js_1 = require("../executeCall.cjs");
+/**
+ * Describe an image as text.
+ *
+ * Depending on the model, this can be used for image captioning, for describing the contents of an image, or for OCR.
+ */
+function describeImage(model, data, options) {
+    return (0, executeCall_js_1.executeCall)({
+        functionType: "image-description",
+        input: data,
+        model,
+        options,
+        generateResponse: (options) => model.generateImageDescriptionResponse(data, options),
+        extractOutputValue: model.extractImageDescription,
+    });
+}
+exports.describeImage = describeImage;

package/model-function/describe-image/describeImage.d.ts

@@ -0,0 +1,9 @@
+import { ModelFunctionOptions } from "../ModelFunctionOptions.js";
+import { ModelFunctionPromise } from "../executeCall.js";
+import { ImageDescriptionModel, ImageDescriptionModelSettings } from "./ImageDescriptionModel.js";
+/**
+ * Describe an image as text.
+ *
+ * Depending on the model, this can be used for image captioning, for describing the contents of an image, or for OCR.
+ */
+export declare function describeImage<DATA, RESPONSE, SETTINGS extends ImageDescriptionModelSettings>(model: ImageDescriptionModel<DATA, RESPONSE, SETTINGS>, data: DATA, options?: ModelFunctionOptions<SETTINGS>): ModelFunctionPromise<string, RESPONSE>;

package/model-function/describe-image/describeImage.js

@@ -0,0 +1,16 @@
+import { executeCall } from "../executeCall.js";
+/**
+ * Describe an image as text.
+ *
+ * Depending on the model, this can be used for image captioning, for describing the contents of an image, or for OCR.
+ */
+export function describeImage(model, data, options) {
+    return executeCall({
+        functionType: "image-description",
+        input: data,
+        model,
+        options,
+        generateResponse: (options) => model.generateImageDescriptionResponse(data, options),
+        extractOutputValue: model.extractImageDescription,
+    });
+}

package/model-function/index.cjs

@@ -19,6 +19,9 @@ __exportStar(require("./ModelCallEvent.cjs"), exports);
 __exportStar(require("./ModelFunctionOptions.cjs"), exports);
 __exportStar(require("./ModelInformation.cjs"), exports);
 __exportStar(require("./SuccessfulModelCall.cjs"), exports);
+__exportStar(require("./describe-image/ImageDescriptionEvent.cjs"), exports);
+__exportStar(require("./describe-image/ImageDescriptionModel.cjs"), exports);
+__exportStar(require("./describe-image/describeImage.cjs"), exports);
 __exportStar(require("./embed-text/TextEmbeddingEvent.cjs"), exports);
 __exportStar(require("./embed-text/TextEmbeddingModel.cjs"), exports);
 __exportStar(require("./embed-text/embedText.cjs"), exports);

package/model-function/index.d.ts

@@ -3,6 +3,9 @@ export * from "./ModelCallEvent.js";
 export * from "./ModelFunctionOptions.js";
 export * from "./ModelInformation.js";
 export * from "./SuccessfulModelCall.js";
+export * from "./describe-image/ImageDescriptionEvent.js";
+export * from "./describe-image/ImageDescriptionModel.js";
+export * from "./describe-image/describeImage.js";
 export * from "./embed-text/TextEmbeddingEvent.js";
 export * from "./embed-text/TextEmbeddingModel.js";
 export * from "./embed-text/embedText.js";

package/model-function/index.js

@@ -3,6 +3,9 @@ export * from "./ModelCallEvent.js";
 export * from "./ModelFunctionOptions.js";
 export * from "./ModelInformation.js";
 export * from "./SuccessfulModelCall.js";
+export * from "./describe-image/ImageDescriptionEvent.js";
+export * from "./describe-image/ImageDescriptionModel.js";
+export * from "./describe-image/describeImage.js";
 export * from "./embed-text/TextEmbeddingEvent.js";
 export * from "./embed-text/TextEmbeddingModel.js";
 export * from "./embed-text/embedText.js";

package/model-provider/huggingface/HuggingFaceError.cjs

@@ -8,10 +8,12 @@ const secure_json_parse_1 = __importDefault(require("secure-json-parse"));
 const zod_1 = require("zod");
 const ApiCallError_js_1 = require("../../core/api/ApiCallError.cjs");
 exports.huggingFaceErrorDataSchema = zod_1.z.object({
-    error: zod_1.z.array(zod_1.z.string()),
+    error: zod_1.z.array(zod_1.z.string()).or(zod_1.z.string()),
 });
 class HuggingFaceError extends ApiCallError_js_1.ApiCallError {
-    constructor({ data, statusCode, url, requestBodyValues, message = data.error.join("\n\n"), }) {
+    constructor({ data, statusCode, url, requestBodyValues, message = typeof data.error === "string"
+        ? data.error
+        : data.error.join("\n\n"), }) {
         super({ message, statusCode, requestBodyValues, url });
         Object.defineProperty(this, "data", {
             enumerable: true,

package/model-provider/huggingface/HuggingFaceError.d.ts

@@ -2,11 +2,11 @@ import { z } from "zod";
 import { ApiCallError } from "../../core/api/ApiCallError.js";
 import { ResponseHandler } from "../../core/api/postToApi.js";
 export declare const huggingFaceErrorDataSchema: z.ZodObject<{
-    error: z.ZodArray<z.ZodString, "many">;
+    error: z.ZodUnion<[z.ZodArray<z.ZodString, "many">, z.ZodString]>;
 }, "strip", z.ZodTypeAny, {
-    error: string[];
+    error: (string | string[]) & (string | string[] | undefined);
 }, {
-    error: string[];
+    error: (string | string[]) & (string | string[] | undefined);
 }>;
 export type HuggingFaceErrorData = z.infer<typeof huggingFaceErrorDataSchema>;
 export declare class HuggingFaceError extends ApiCallError {

package/model-provider/huggingface/HuggingFaceError.js

@@ -2,10 +2,12 @@ import SecureJSON from "secure-json-parse";
 import { z } from "zod";
 import { ApiCallError } from "../../core/api/ApiCallError.js";
 export const huggingFaceErrorDataSchema = z.object({
-    error: z.array(z.string()),
+    error: z.array(z.string()).or(z.string()),
 });
 export class HuggingFaceError extends ApiCallError {
-    constructor({ data, statusCode, url, requestBodyValues, message = data.error.join("\n\n"), }) {
+    constructor({ data, statusCode, url, requestBodyValues, message = typeof data.error === "string"
+        ? data.error
+        : data.error.join("\n\n"), }) {
         super({ message, statusCode, requestBodyValues, url });
         Object.defineProperty(this, "data", {
             enumerable: true,

package/model-provider/huggingface/HuggingFaceImageDescriptionModel.cjs

@@ -0,0 +1,81 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HuggingFaceImageDescriptionModel = void 0;
+const zod_1 = __importDefault(require("zod"));
+const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
+const postToApi_js_1 = require("../../core/api/postToApi.cjs");
+const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
+const HuggingFaceApiConfiguration_js_1 = require("./HuggingFaceApiConfiguration.cjs");
+const HuggingFaceError_js_1 = require("./HuggingFaceError.cjs");
+/**
+ * Create an image to text model that calls a Hugging Face Image-to-Text Inference API.
+ *
+ * @see https://huggingface.co/tasks/image-to-text
+ */
+class HuggingFaceImageDescriptionModel extends AbstractModel_js_1.AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "huggingface"
+        });
+        Object.defineProperty(this, "countPromptTokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(data, options) {
+        const run = options?.run;
+        const settings = options?.settings;
+        const callSettings = {
+            ...this.settings,
+            ...settings,
+            abortSignal: run?.abortSignal,
+            data,
+        };
+        return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
+            retry: callSettings.api?.retry,
+            throttle: callSettings.api?.throttle,
+            call: async () => callHuggingFaceImageDescriptionAPI(callSettings),
+        });
+    }
+    get settingsForEvent() {
+        return {};
+    }
+    generateImageDescriptionResponse(data, options) {
+        return this.callAPI(data, options);
+    }
+    extractImageDescription(response) {
+        return response[0].generated_text;
+    }
+    withSettings(additionalSettings) {
+        return new HuggingFaceImageDescriptionModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+exports.HuggingFaceImageDescriptionModel = HuggingFaceImageDescriptionModel;
+const huggingFaceImageDescriptionResponseSchema = zod_1.default.array(zod_1.default.object({
+    generated_text: zod_1.default.string(),
+}));
+async function callHuggingFaceImageDescriptionAPI({ api = new HuggingFaceApiConfiguration_js_1.HuggingFaceApiConfiguration(), abortSignal, model, data, }) {
+    return (0, postToApi_js_1.postToApi)({
+        url: api.assembleUrl(`/${model}`),
+        headers: api.headers,
+        body: {
+            content: data,
+            values: {},
+        },
+        failedResponseHandler: HuggingFaceError_js_1.failedHuggingFaceCallResponseHandler,
+        successfulResponseHandler: (0, postToApi_js_1.createJsonResponseHandler)(huggingFaceImageDescriptionResponseSchema),
+        abortSignal,
+    });
+}

package/model-provider/huggingface/HuggingFaceImageDescriptionModel.d.ts

@@ -0,0 +1,37 @@
+/// <reference types="node" resolution-mode="require"/>
+import z from "zod";
+import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { ModelFunctionOptions } from "../../model-function/ModelFunctionOptions.js";
+import { ImageDescriptionModel, ImageDescriptionModelSettings } from "../../model-function/describe-image/ImageDescriptionModel.js";
+export interface HuggingFaceImageDescriptionModelSettings extends ImageDescriptionModelSettings {
+    api?: ApiConfiguration;
+    model: string;
+}
+/**
+ * Create an image to text model that calls a Hugging Face Image-to-Text Inference API.
+ *
+ * @see https://huggingface.co/tasks/image-to-text
+ */
+export declare class HuggingFaceImageDescriptionModel extends AbstractModel<HuggingFaceImageDescriptionModelSettings> implements ImageDescriptionModel<Buffer, HuggingFaceImageDescriptionResponse, HuggingFaceImageDescriptionModelSettings> {
+    constructor(settings: HuggingFaceImageDescriptionModelSettings);
+    readonly provider = "huggingface";
+    get modelName(): string;
+    callAPI(data: Buffer, options?: ModelFunctionOptions<HuggingFaceImageDescriptionModelSettings>): Promise<HuggingFaceImageDescriptionResponse>;
+    get settingsForEvent(): Partial<HuggingFaceImageDescriptionModelSettings>;
+    readonly countPromptTokens: undefined;
+    generateImageDescriptionResponse(data: Buffer, options?: ModelFunctionOptions<HuggingFaceImageDescriptionModelSettings>): Promise<{
+        generated_text: string;
+    }[]>;
+    extractImageDescription(response: HuggingFaceImageDescriptionResponse): string;
+    withSettings(additionalSettings: Partial<HuggingFaceImageDescriptionModelSettings>): this;
+}
+declare const huggingFaceImageDescriptionResponseSchema: z.ZodArray<z.ZodObject<{
+    generated_text: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    generated_text: string;
+}, {
+    generated_text: string;
+}>, "many">;
+export type HuggingFaceImageDescriptionResponse = z.infer<typeof huggingFaceImageDescriptionResponseSchema>;
+export {};

package/model-provider/huggingface/HuggingFaceImageDescriptionModel.js

@@ -0,0 +1,74 @@
+import z from "zod";
+import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
+import { createJsonResponseHandler, postToApi, } from "../../core/api/postToApi.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { HuggingFaceApiConfiguration } from "./HuggingFaceApiConfiguration.js";
+import { failedHuggingFaceCallResponseHandler } from "./HuggingFaceError.js";
+/**
+ * Create an image to text model that calls a Hugging Face Image-to-Text Inference API.
+ *
+ * @see https://huggingface.co/tasks/image-to-text
+ */
+export class HuggingFaceImageDescriptionModel extends AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "huggingface"
+        });
+        Object.defineProperty(this, "countPromptTokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(data, options) {
+        const run = options?.run;
+        const settings = options?.settings;
+        const callSettings = {
+            ...this.settings,
+            ...settings,
+            abortSignal: run?.abortSignal,
+            data,
+        };
+        return callWithRetryAndThrottle({
+            retry: callSettings.api?.retry,
+            throttle: callSettings.api?.throttle,
+            call: async () => callHuggingFaceImageDescriptionAPI(callSettings),
+        });
+    }
+    get settingsForEvent() {
+        return {};
+    }
+    generateImageDescriptionResponse(data, options) {
+        return this.callAPI(data, options);
+    }
+    extractImageDescription(response) {
+        return response[0].generated_text;
+    }
+    withSettings(additionalSettings) {
+        return new HuggingFaceImageDescriptionModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+const huggingFaceImageDescriptionResponseSchema = z.array(z.object({
+    generated_text: z.string(),
+}));
+async function callHuggingFaceImageDescriptionAPI({ api = new HuggingFaceApiConfiguration(), abortSignal, model, data, }) {
+    return postToApi({
+        url: api.assembleUrl(`/${model}`),
+        headers: api.headers,
+        body: {
+            content: data,
+            values: {},
+        },
+        failedResponseHandler: failedHuggingFaceCallResponseHandler,
+        successfulResponseHandler: createJsonResponseHandler(huggingFaceImageDescriptionResponseSchema),
+        abortSignal,
+    });
+}

package/model-provider/huggingface/index.cjs

@@ -18,5 +18,6 @@ exports.HuggingFaceError = void 0;
 __exportStar(require("./HuggingFaceApiConfiguration.cjs"), exports);
 var HuggingFaceError_js_1 = require("./HuggingFaceError.cjs");
 Object.defineProperty(exports, "HuggingFaceError", { enumerable: true, get: function () { return HuggingFaceError_js_1.HuggingFaceError; } });
+__exportStar(require("./HuggingFaceImageDescriptionModel.cjs"), exports);
 __exportStar(require("./HuggingFaceTextEmbeddingModel.cjs"), exports);
 __exportStar(require("./HuggingFaceTextGenerationModel.cjs"), exports);

package/model-provider/huggingface/index.d.ts

@@ -1,4 +1,5 @@
 export * from "./HuggingFaceApiConfiguration.js";
 export { HuggingFaceError, HuggingFaceErrorData } from "./HuggingFaceError.js";
+export * from "./HuggingFaceImageDescriptionModel.js";
 export * from "./HuggingFaceTextEmbeddingModel.js";
 export * from "./HuggingFaceTextGenerationModel.js";

package/model-provider/huggingface/index.js

@@ -1,4 +1,5 @@
 export * from "./HuggingFaceApiConfiguration.js";
 export { HuggingFaceError } from "./HuggingFaceError.js";
+export * from "./HuggingFaceImageDescriptionModel.js";
 export * from "./HuggingFaceTextEmbeddingModel.js";
 export * from "./HuggingFaceTextGenerationModel.js";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build AI applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.28.0",
+  "version": "0.29.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [