@llumiverse/core 0.14.0 → 0.16.0

package/src/options/vertexai.ts

@@ -0,0 +1,312 @@
+import { ModelOptionsInfo, ModelOptionInfoItem, OptionType, SharedOptions, ModelOptions } from "../types.js";
+import { textOptionsFallback } from "../options.js";
+
+// Union type of all Bedrock options
+export type VertexAIOptions = ImagenOptions | VertexAIClaudeOptions;
+
+export enum ImagenTaskType {
+    TEXT_IMAGE = "TEXT_IMAGE",
+    EDIT_MODE_INPAINT_REMOVAL = "EDIT_MODE_INPAINT_REMOVAL",
+    EDIT_MODE_INPAINT_INSERTION = "EDIT_MODE_INPAINT_INSERTION",
+    EDIT_MODE_BGSWAP = "EDIT_MODE_BGSWAP",
+    EDIT_MODE_OUTPAINT = "EDIT_MODE_OUTPAINT",
+    CUSTOMIZATION_SUBJECT = "CUSTOMIZATION_SUBJECT",
+    CUSTOMIZATION_STYLE = "CUSTOMIZATION_STYLE",
+    CUSTOMIZATION_CONTROLLED = "CUSTOMIZATION_CONTROLLED",
+    CUSTOMIZATION_INSTRUCT = "CUSTOMIZATION_INSTRUCT",
+}
+
+export enum ImagenMaskMode {
+    MASK_MODE_USER_PROVIDED = "MASK_MODE_USER_PROVIDED",
+    MASK_MODE_BACKGROUND = "MASK_MODE_BACKGROUND",
+    MASK_MODE_FOREGROUND = "MASK_MODE_FOREGROUND",
+    MASK_MODE_SEMANTIC = "MASK_MODE_SEMANTIC",
+}
+
+export interface ImagenOptions {
+    _option_id: "vertexai-imagen"
+
+    //General and generate options
+    number_of_images?: number;
+    seed?: number;
+    person_generation?: "dont_allow" | "allow_adults" | "allow_all";
+    safety_setting?: "block_none" | "block_only_high" | "block_medium_and_above" | "block_low_and_above"; //The "off" option does not seem to work for Imagen 3, might be only for text models
+    image_file_type?: "image/jpeg" | "image/png";
+    jpeg_compression_quality?: number;
+    aspect_ratio?: "1:1" | "4:3" | "3:4" | "16:9" | "9:16";
+    add_watermark?: boolean;
+    enhance_prompt?: boolean;
+
+    //Capability options
+    edit_mode?: ImagenTaskType
+    guidance_scale?: number;
+    edit_steps?: number;
+    mask_mode?: ImagenMaskMode;
+    mask_dilation?: number;
+    mask_class?: number[];
+
+    //Customization options
+    controlType: "CONTROL_TYPE_FACE_MESH" | "CONTROL_TYPE_CANNY" | "CONTROL_TYPE_SCRIBBLE";
+    controlImageComputation?: boolean;
+    subjectType: "SUBJECT_TYPE_PERSON" | "SUBJECT_TYPE_ANIMAL" | "SUBJECT_TYPE_PRODUCT" | "SUBJECT_TYPE_DEFAULT";
+}
+
+export interface VertexAIClaudeOptions {
+    _option_id: "vertexai-claude"
+    max_tokens?: number;
+    temperature?: number;
+    top_p?: number;
+    top_k?: number;
+    stop_sequence?: string[];
+    thinking_mode?: boolean;
+    thinking_budget_tokens?: number;
+}
+
+export function getVertexAiOptions(model: string, option?: ModelOptions): ModelOptionsInfo {
+    if (model.includes("imagen-3.0")) {
+        const commonOptions: ModelOptionInfoItem[] = [
+            {
+                name: SharedOptions.number_of_images, type: OptionType.numeric, min: 1, max: 4, default: 1,
+                integer: true, description: "Number of Images to generate",
+            },
+            {
+                name: SharedOptions.seed, type: OptionType.numeric, min: 0, max: 4294967295, default: 12,
+                integer: true, description: "The seed of the generated image"
+            },
+            {
+                name: "person_generation", type: OptionType.enum, enum: { "Disallow the inclusion of people or faces in images": "dont_allow", "Allow generation of adults only": "allow_adult", "Allow generation of people of all ages": "allow_all" },
+                default: "allow_adult", description: "The safety setting for allowing the generation of people in the image"
+            },
+            {
+                name: "safety_setting", type: OptionType.enum, enum: { "Block very few problematic prompts and responses": "block_none", "Block only few problematic prompts and responses": "block_only_high", "Block some problematic prompts and responses": "block_medium_and_above", "Strictest filtering": "block_low_and_above" },
+                default: "block_medium_and_above", description: "The overall safety setting"
+            },
+        ];
+
+
+        const outputOptions: ModelOptionInfoItem[] = [
+            {
+                name: "image_file_type", type: OptionType.enum, enum: { "JPEG": "image/jpeg", "PNG": "image/png" },
+                default: "image/png", description: "The file type of the generated image",
+                refresh: true,
+            },
+        ]
+
+        const jpegQuality: ModelOptionInfoItem = {
+            name: "jpeg_compression_quality", type: OptionType.numeric, min: 0, max: 100, default: 75,
+            integer: true, description: "The compression quality of the JPEG image",
+        }
+
+        if ((option as ImagenOptions)?.image_file_type === "image/jpeg") {
+            outputOptions.push(jpegQuality);
+        }
+        if (model.includes("generate")) {
+            //Generate models
+            const modeOptions: ModelOptionInfoItem[]
+                = [
+                    {
+                        name: "aspect_ratio", type: OptionType.enum, enum: { "1:1": "1:1", "4:3": "4:3", "3:4": "3:4", "16:9": "16:9", "9:16": "9:16" },
+                        default: "1:1", description: "The aspect ratio of the generated image"
+                    },
+                    {
+                        name: "add_watermark", type: OptionType.boolean, default: false, description: "Add an invisible watermark to the generated image, useful for detection of AI images"
+                    },
+
+                ];
+
+            const enhanceOptions: ModelOptionInfoItem[] = !model.includes("generate-001") ? [
+                {
+                    name: "enhance_prompt", type: OptionType.boolean, default: true, description: "VertexAI automatically rewrites the prompt to better reflect the prompt's intent."
+                },
+            ] : [];
+
+            return {
+                _option_id: "vertexai-imagen",
+                options: [
+                    ...commonOptions,
+                    ...modeOptions,
+                    ...outputOptions,
+                    ...enhanceOptions,
+                ]
+            };
+        }
+        if (model.includes("capability")) {
+            //Edit models
+            let guidanceScaleDefault = 75;
+            if ((option as ImagenOptions)?.edit_mode === ImagenTaskType.EDIT_MODE_INPAINT_INSERTION) {
+                guidanceScaleDefault = 60;
+            }
+
+            const modeOptions: ModelOptionInfoItem[] = [
+                {
+                    name: "edit_mode", type: OptionType.enum,
+                    enum: {
+                        "EDIT_MODE_INPAINT_REMOVAL": "EDIT_MODE_INPAINT_REMOVAL",
+                        "EDIT_MODE_INPAINT_INSERTION": "EDIT_MODE_INPAINT_INSERTION",
+                        "EDIT_MODE_BGSWAP": "EDIT_MODE_BGSWAP",
+                        "EDIT_MODE_OUTPAINT": "EDIT_MODE_OUTPAINT",
+                        "CUSTOMIZATION_SUBJECT": "CUSTOMIZATION_SUBJECT",
+                        "CUSTOMIZATION_STYLE": "CUSTOMIZATION_STYLE",
+                        "CUSTOMIZATION_CONTROLLED": "CUSTOMIZATION_CONTROLLED",
+                        "CUSTOMIZATION_INSTRUCT": "CUSTOMIZATION_INSTRUCT",
+                    },
+                    description: "The editing mode. CUSTOMIZATION options use few-shot learning to generate images based on a few examples."
+                },
+                
+                {
+                    name: "guidance_scale", type: OptionType.numeric, min: 0, max: 500, default: guidanceScaleDefault,
+                    integer: true, description: "How closely the generation follows the prompt"
+                }
+            ];
+
+            const maskOptions: ModelOptionInfoItem[] = ((option as ImagenOptions)?.edit_mode?.includes("EDIT")) ? [
+                {
+                    name: "mask_mode", type: OptionType.enum,
+                    enum: {
+                        "MASK_MODE_USER_PROVIDED": "MASK_MODE_USER_PROVIDED",
+                        "MASK_MODE_BACKGROUND": "MASK_MODE_BACKGROUND",
+                        "MASK_MODE_FOREGROUND": "MASK_MODE_FOREGROUND",
+                        "MASK_MODE_SEMANTIC": "MASK_MODE_SEMANTIC",
+                    },
+                    default: "MASK_MODE_USER_PROVIDED",
+                    description: "How should the mask for the generation be provided"
+                },
+                {
+                    name: "mask_dilation", type: OptionType.numeric, min: 0, max: 1,
+                    integer: true, description: "The mask dilation, grows the mask by a percetage of image width to compensate for imprecise masks."
+                },
+            ] : [];
+
+            const maskClassOptions: ModelOptionInfoItem[] = ((option as ImagenOptions)?.mask_mode === ImagenMaskMode.MASK_MODE_SEMANTIC) ? [
+                {
+                    name: "mask_class", type: OptionType.string_list, default: [],
+                    description: "Input Class IDs. Create a mask based on image class, based on https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api-customization#segment-ids"
+                }
+            ] : [];
+
+            const editOptions: ModelOptionInfoItem[] = (option as ImagenOptions)?.edit_mode?.includes("EDIT") ? [
+                {
+                    name: "edit_steps", type: OptionType.numeric, default: 75,
+                    integer: true, description: "The number of steps for the base image generation, more steps means more time and better quality"
+                },
+            ] : [];
+
+            const customizationOptions: ModelOptionInfoItem[] = (option as ImagenOptions)?.edit_mode === ImagenTaskType.CUSTOMIZATION_CONTROLLED
+            || (option as ImagenOptions)?.edit_mode === ImagenTaskType.CUSTOMIZATION_SUBJECT ? [
+                {
+                    name: "controlType", type: OptionType.enum, enum: { "Face Mesh": "CONTROL_TYPE_FACE_MESH", "Canny": "CONTROL_TYPE_CANNY", "Scribble": "CONTROL_TYPE_SCRIBBLE" },
+                    default: "CONTROL_TYPE_CANNY", description: "Method used to generate the control image"
+                },
+                {
+                    name: "controlImageComputation", type: OptionType.boolean, default: true, description: "Should the control image be computed from the input image, or is it provided"
+                }
+            ] : [];
+
+            return {
+                _option_id: "vertexai-imagen",
+                options: [
+                    ...modeOptions,
+                    ...commonOptions,
+                    ...maskOptions,
+                    ...maskClassOptions,
+                    ...editOptions,
+                    ...customizationOptions,
+                    ...outputOptions,
+                ]
+            };
+        }
+    }
+    else if (model.includes("gemini")) {
+        const max_tokens_limit = getGeminiMaxTokensLimit(model);
+        const excludeOptions = ["max_tokens", "presence_penalty"];
+        let commonOptions = textOptionsFallback.options.filter((option) => !excludeOptions.includes(option.name));
+        if (model.includes("1.5")) {
+            commonOptions = commonOptions.filter((option) => option.name !== "frequency_penalty");
+        }
+        const max_tokens: ModelOptionInfoItem[] = [{
+            name: SharedOptions.max_tokens, type: OptionType.numeric, min: 1, max: max_tokens_limit,
+            integer: true, step: 200, description: "The maximum number of tokens to generate"
+        }];
+        return {
+            _option_id: "vertexai-gemini",
+            options: [
+                ...max_tokens,
+                ...commonOptions,
+            ]
+        };
+    }
+    else if (model.includes("claude")) {
+        const max_tokens_limit = getClaudeMaxTokensLimit(model, option as VertexAIClaudeOptions);
+        const excludeOptions = ["max_tokens", "presence_penalty", "frequency_penalty"];
+        let commonOptions = textOptionsFallback.options.filter((option) => !excludeOptions.includes(option.name));
+        const max_tokens: ModelOptionInfoItem[] = [{
+            name: SharedOptions.max_tokens, type: OptionType.numeric, min: 1, max: max_tokens_limit,
+            integer: true, step: 200, description: "The maximum number of tokens to generate"
+        }];
+
+        if (model.includes("3-7")) {
+            const claudeModeOptions: ModelOptionInfoItem[] = [
+                {
+                    name: "thinking_mode",
+                    type: OptionType.boolean,
+                    default: false,
+                    description: "If true, use the extended reasoning mode"
+                },
+            ];
+            const claudeThinkingOptions: ModelOptionInfoItem[] = (option as VertexAIClaudeOptions)?.thinking_mode ? [
+                {
+                    name: "thinking_budget_tokens",
+                    type: OptionType.numeric,
+                    min: 1024,
+                    default: 4000,
+                    integer: true,
+                    step: 100,
+                    description: "The target number of tokens to use for reasoning, not a hard limit."
+                },
+            ] : [];
+
+            return {
+                _option_id: "vertexai-claude",
+                options: [
+                    ...max_tokens,
+                    ...commonOptions,
+                    ...claudeModeOptions,
+                    ...claudeThinkingOptions,
+                ]
+            };
+        }
+        return {
+            _option_id: "vertexai-claude",
+            options: [
+                ...max_tokens,
+                ...commonOptions,
+            ]
+        };
+    }
+    return textOptionsFallback;
+}
+function getGeminiMaxTokensLimit(model: string): number {
+    if (model.includes("thinking")) {
+        return 65536;
+    }
+    if (model.includes("ultra") || model.includes("vision")) {
+        return 2048;
+    }
+    return 8192;
+}
+function getClaudeMaxTokensLimit(model: string, option?: VertexAIClaudeOptions): number {
+    if (model.includes("3-7")) {
+        if (option && option?.thinking_mode) {
+            return 128000;
+        } else {
+            return 8192;
+        }
+    }
+    else if (model.includes("3-5")) {
+        return 8192;
+    }
+    else {
+        return 4096;
+    }
+}
+

package/src/options.ts

@@ -0,0 +1,62 @@
+import { ModelOptions, ModelOptionsInfo, OptionType, SharedOptions } from "./types.js";
+import { getBedrockOptions } from "./options/bedrock.js";
+import { getVertexAiOptions } from "./options/vertexai.js";
+import { getOpenAiOptions } from "./options/openai.js";
+import { getGroqOptions } from "./options/groq.js";
+
+export interface TextFallbackOptions {
+    _option_id: "text-fallback";    //For specific models should be format as "provider-model"
+    max_tokens?: number;
+    temperature?: number;
+    top_p?: number;
+    top_k?: number;
+    presence_penalty?: number;
+    frequency_penalty?: number;
+    stop_sequence?: string[];
+}
+
+export const textOptionsFallback: ModelOptionsInfo = {
+    _option_id: "text-fallback",
+    options: [
+        {
+            name: SharedOptions.max_tokens, type: OptionType.numeric, min: 1,
+            integer: true, step: 200, description: "The maximum number of tokens to generate"
+        },
+        {
+            name: SharedOptions.temperature, type: OptionType.numeric, min: 0.0, default: 0.7,
+            integer: false, step: 0.1, description: "A higher temperature biases toward less likely tokens, making the model more creative"
+        },
+        {
+            name: SharedOptions.top_p, type: OptionType.numeric, min: 0, max: 1,
+            integer: false, step: 0.1, description: "Limits token sampling to the cumulative probability of the top p tokens"
+        },
+        {
+            name: SharedOptions.top_k, type: OptionType.numeric, min: 1,
+            integer: true, step: 1, description: "Limits token sampling to the top k tokens"
+        },
+        {
+            name: SharedOptions.presence_penalty, type: OptionType.numeric, min: -2.0, max: 2.0,
+            integer: false, step: 0.1, description: "Penalise tokens if they appear at least once in the text"
+        },
+        {
+            name: SharedOptions.frequency_penalty, type: OptionType.numeric, min: -2.0, max: 2.0,
+            integer: false, step: 0.1, description: "Penalise tokens based on their frequency in the text"
+        },
+        { name: SharedOptions.stop_sequence, type: OptionType.string_list, value: [], description: "The generation will halt if one of the stop sequences is output" },
+    ]
+};
+
+export function getOptions(provider?: string, model?: string, options?: ModelOptions): ModelOptionsInfo {
+    switch (provider) {
+        case "bedrock":
+            return getBedrockOptions(model ?? "", options);
+        case "vertexai":
+            return getVertexAiOptions(model ?? "", options);
+        case "openai":
+            return getOpenAiOptions(model ?? "", options);
+        case "groq":
+            return getGroqOptions(model ?? "", options);
+        default:
+            return textOptionsFallback;
+    }
+}

package/src/types.ts

@@ -1,21 +1,26 @@
-import { JSONSchema4 } from "json-schema";
-import { PromptFormatter } from "./formatters/index.js";
-import { JSONObject } from "./json.js";
+import { JSONSchema4 } from 'json-schema';
+
+import { PromptFormatter } from './formatters/index.js';
+import { JSONObject } from './json.js';
+import { TextFallbackOptions } from './options.js';
+import { VertexAIOptions } from './options/vertexai.js';
+import { BedrockOptions } from './options/bedrock.js';
+import { OpenAiOptions } from './options/openai.js';
 
 export interface EmbeddingsOptions {
     /**
-     * The content to generate the embeddings for. Required.
+     * The text to generate the embeddings for. One of text or image is required.
      */
-    content: string;
+    text?: string;
     /**
-     * The model to use to generate the embeddings. Optional.
+     * The image to generate embeddings for
      */
-    model?: string;
+    image?: string
     /**
-     * Additional options for the embeddings generation. Optional.
-     * The supported properties depends on the target implementation.
+     * The model to use to generate the embeddings. Optional.
      */
-    [key: string]: any;
+    model?: string;
+
 }
 
 export interface EmbeddingsResult {
@@ -26,32 +31,60 @@ export interface EmbeddingsResult {
     /**
      * The model used to hgenerate the embeddings.
      */
-    model?: string;
+    model: string;
     /**
      * Number of tokens of the input text.
      */
     token_count?: number;
-    /**
-     * Additional properties. Depends on the target implementation.
-     */
-    [key: string]: any;
+
 }
 
 export interface ResultValidationError {
-    code: 'validation_error' | 'json_error';
+    code: 'validation_error' | 'json_error' | 'content_policy_violation';
     message: string;
     data?: string;
 }
 
+//ResultT should be either JSONObject or string
+//Internal structure used in driver implementation.
+export interface CompletionChunkObject<ResultT = any> {
+    result: ResultT;
+    token_usage?: ExecutionTokenUsage;
+    finish_reason?: "stop" | "length" | string;
+}
+
+//Internal structure used in driver implementation.
+export type CompletionChunk = CompletionChunkObject | string;
+
+export interface ToolDefinition {
+    name: string,
+    description?: string,
+    input_schema: {
+        type: 'object';
+        properties?: unknown | null | undefined;
+        [k: string]: unknown;
+    },
+}
+
+export interface ToolUse {
+    id: string,
+    name: string,
+    input: JSONObject | null
+}
+
+//ResultT should be either JSONObject or string
 export interface Completion<ResultT = any> {
     // the driver impl must return the result and optionally the token_usage. the execution time is computed by the extended abstract driver
     result: ResultT;
     token_usage?: ExecutionTokenUsage;
-
+    /**
+     * Contains the tools from which the model awaits information.
+     */
+    tool_use?: ToolUse[];
     /**
      * The finish reason as reported by the model: stop | length or other model specific values
      */
-    finish_reason?: "stop" | "length" | string;
+    finish_reason?: "stop" | "length" | "tool_use" | string;
 
     /**
      * Set only if a result validation error occured, otherwise if the result is valid the error field is undefined
@@ -64,6 +97,16 @@ export interface Completion<ResultT = any> {
      */
     original_response?: Record<string, any>;
 
+    /**
+     * The conversation context. This is an opaque structure that can be passed to the next request to restore the context.
+     */
+    conversation?: unknown;
+}
+
+export interface ImageGeneration {
+
+    images?: string[];
+
 }
 
 export interface ExecutionResponse<PromptT = any> extends Completion {
@@ -72,6 +115,10 @@ export interface ExecutionResponse<PromptT = any> extends Completion {
      * The time it took to execute the request in seconds
      */
     execution_time?: number;
+    /**
+     * The number of chunks for streamed executions
+     */
+    chunks?: number;
 }
 
 
@@ -90,6 +137,9 @@ export interface DriverOptions {
     logger?: Logger | "console";
 }
 
+//Options are split into PromptOptions, ModelOptions and ExecutionOptions.
+//ExecutionOptions are most often used within llumiverse as they are the most complete.
+//The base types are useful for external code that needs to interact with llumiverse.
 export interface PromptOptions {
     model: string;
     /**
@@ -99,50 +149,100 @@ export interface PromptOptions {
     format?: PromptFormatter;
     result_schema?: JSONSchema4;
 }
-export interface ExecutionOptions extends PromptOptions {
-    temperature?: number;
-    max_tokens?: number;
-    stop_sequence?: string | string[];
 
+export interface ExecutionOptions extends PromptOptions {
     /**
-     * restricts the selection of tokens to the “k” most likely options, based on their probabilities
-     * Lower values make the model more deterministic, more focused. Examples:
-     * - 10 - result will be highly controlled anc contextually relevant
-     * - 50 - result will be more creative but maintaining a balance between control and creativity
-     * - 100 - will lead to more creative and less predictable outputs
-     * It will be ignored on OpenAI since it does not support it
+     * If set to true the original response from the target LLM will be included in the response under the original_response field.
+     * This is useful for debugging and for some advanced use cases.
+     * It is ignored on streaming requests
      */
-    top_k?: number;
-
+    include_original_response?: boolean;
+    model_options?: ModelOptions;
+    output_modality: Modalities;
     /**
-     * An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.
-     * Either use temperature or top_p, not both
+     * Available tools for the request
      */
-    top_p?: number;
-
+    tools?: ToolDefinition[];
     /**
-     * Only supported for OpenAI. Look at OpenAI documentation for more detailsx
+     * This is an opaque structure that provides a conversation context
+     * Each driver implementation will return a conversation property in the execution response
+     * that can be passed here to restore the context when a new prompt is sent to the model.
      */
-    top_logprobs?: number;
+    conversation?: unknown | null;
+}
 
-    /**
-     * Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.
-     * Ignored for models which doesn;t support it
-     */
-    presence_penalty?: number;
+//Common names to share between different models
+export enum SharedOptions {
+    //Text
+    max_tokens = "max_tokens",
+    temperature = "temperature",
+    top_p = "top_p",
+    top_k = "top_k",
+    presence_penalty = "presence_penalty",
+    frequency_penalty = "frequency_penalty",
+    stop_sequence = "stop_sequence",
+
+    //Image
+    seed = "seed",
+    number_of_images = "number_of_images",
+}
 
-    /**
-     * Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.
-     * Ignored for models which doesn;t support it
-     */
-    frequency_penalty?: number;
+export enum OptionType {
+    numeric = "numeric",
+    enum = "enum",
+    boolean = "boolean",
+    string_list = "string_list"
+}
 
-    /**
-     * If set to true the original response from the target LLM will be included in the response under the original_response field.
-     * This is useful for debugging and for some advanced use cases.
-     * It is ignored on streaming requests
-     */
-    include_original_response?: boolean;
+// ============== Model Options ===============
+
+export type ModelOptions = TextFallbackOptions | VertexAIOptions | BedrockOptions | OpenAiOptions;
+
+// ============== Option Info ===============
+
+export interface ModelOptionsInfo {
+    options: ModelOptionInfoItem[];
+    _option_id: string; //Should follow same ids as ModelOptions
+}
+
+export type ModelOptionInfoItem = NumericOptionInfo | EnumOptionInfo | BooleanOptionInfo | StringListOptionInfo;
+interface OptionInfoPrototype {
+    type: OptionType;
+    name: string;
+    description?: string;
+
+    //If this is true, whether other options apply is dependent on this option
+    //Therefore, if this option is changed, the set of available options should be refreshed.
+    refresh?: boolean;
+}
+
+export interface NumericOptionInfo extends OptionInfoPrototype {
+    type: OptionType.numeric;
+    value?: number;
+    min?: number;
+    max?: number;
+    step?: number;
+    integer?: boolean;
+    default?: number;
+}
+
+export interface EnumOptionInfo extends OptionInfoPrototype {
+    type: OptionType.enum;
+    value?: string;
+    enum: Record<string, string>;
+    default?: string;
+}
+
+export interface BooleanOptionInfo extends OptionInfoPrototype {
+    type: OptionType.boolean;
+    value?: boolean;
+    default?: boolean;
+}
+
+export interface StringListOptionInfo extends OptionInfoPrototype {
+    type: OptionType.string_list;
+    value?: string[];
+    default?: string[];
 }
 
 // ============== Prompts ===============
@@ -151,11 +251,21 @@ export enum PromptRole {
     system = "system",
     user = "user",
     assistant = "assistant",
+    negative = "negative",
+    mask = "mask",
+    /**
+     * Used to send the response of a tool
+     */
+    tool = "tool"
 }
 
 export interface PromptSegment {
     role: PromptRole;
     content: string;
+    /**
+     * The tool use id if the segment is a tool response
+     */
+    tool_use_id?: string;
     files?: DataSource[]
 }
 
@@ -165,6 +275,11 @@ export interface ExecutionTokenUsage {
     total?: number;
 }
 
+export enum Modalities {
+    text = "text",
+    image = "image"
+}
+
 
 // ============== AI MODEL ==============