@llumiverse/core 0.16.0 → 0.18.0

package/src/options/vertexai.ts

@@ -1,312 +0,0 @@
-import { ModelOptionsInfo, ModelOptionInfoItem, OptionType, SharedOptions, ModelOptions } from "../types.js";
-import { textOptionsFallback } from "../options.js";
-
-// Union type of all Bedrock options
-export type VertexAIOptions = ImagenOptions | VertexAIClaudeOptions;
-
-export enum ImagenTaskType {
-    TEXT_IMAGE = "TEXT_IMAGE",
-    EDIT_MODE_INPAINT_REMOVAL = "EDIT_MODE_INPAINT_REMOVAL",
-    EDIT_MODE_INPAINT_INSERTION = "EDIT_MODE_INPAINT_INSERTION",
-    EDIT_MODE_BGSWAP = "EDIT_MODE_BGSWAP",
-    EDIT_MODE_OUTPAINT = "EDIT_MODE_OUTPAINT",
-    CUSTOMIZATION_SUBJECT = "CUSTOMIZATION_SUBJECT",
-    CUSTOMIZATION_STYLE = "CUSTOMIZATION_STYLE",
-    CUSTOMIZATION_CONTROLLED = "CUSTOMIZATION_CONTROLLED",
-    CUSTOMIZATION_INSTRUCT = "CUSTOMIZATION_INSTRUCT",
-}
-
-export enum ImagenMaskMode {
-    MASK_MODE_USER_PROVIDED = "MASK_MODE_USER_PROVIDED",
-    MASK_MODE_BACKGROUND = "MASK_MODE_BACKGROUND",
-    MASK_MODE_FOREGROUND = "MASK_MODE_FOREGROUND",
-    MASK_MODE_SEMANTIC = "MASK_MODE_SEMANTIC",
-}
-
-export interface ImagenOptions {
-    _option_id: "vertexai-imagen"
-
-    //General and generate options
-    number_of_images?: number;
-    seed?: number;
-    person_generation?: "dont_allow" | "allow_adults" | "allow_all";
-    safety_setting?: "block_none" | "block_only_high" | "block_medium_and_above" | "block_low_and_above"; //The "off" option does not seem to work for Imagen 3, might be only for text models
-    image_file_type?: "image/jpeg" | "image/png";
-    jpeg_compression_quality?: number;
-    aspect_ratio?: "1:1" | "4:3" | "3:4" | "16:9" | "9:16";
-    add_watermark?: boolean;
-    enhance_prompt?: boolean;
-
-    //Capability options
-    edit_mode?: ImagenTaskType
-    guidance_scale?: number;
-    edit_steps?: number;
-    mask_mode?: ImagenMaskMode;
-    mask_dilation?: number;
-    mask_class?: number[];
-
-    //Customization options
-    controlType: "CONTROL_TYPE_FACE_MESH" | "CONTROL_TYPE_CANNY" | "CONTROL_TYPE_SCRIBBLE";
-    controlImageComputation?: boolean;
-    subjectType: "SUBJECT_TYPE_PERSON" | "SUBJECT_TYPE_ANIMAL" | "SUBJECT_TYPE_PRODUCT" | "SUBJECT_TYPE_DEFAULT";
-}
-
-export interface VertexAIClaudeOptions {
-    _option_id: "vertexai-claude"
-    max_tokens?: number;
-    temperature?: number;
-    top_p?: number;
-    top_k?: number;
-    stop_sequence?: string[];
-    thinking_mode?: boolean;
-    thinking_budget_tokens?: number;
-}
-
-export function getVertexAiOptions(model: string, option?: ModelOptions): ModelOptionsInfo {
-    if (model.includes("imagen-3.0")) {
-        const commonOptions: ModelOptionInfoItem[] = [
-            {
-                name: SharedOptions.number_of_images, type: OptionType.numeric, min: 1, max: 4, default: 1,
-                integer: true, description: "Number of Images to generate",
-            },
-            {
-                name: SharedOptions.seed, type: OptionType.numeric, min: 0, max: 4294967295, default: 12,
-                integer: true, description: "The seed of the generated image"
-            },
-            {
-                name: "person_generation", type: OptionType.enum, enum: { "Disallow the inclusion of people or faces in images": "dont_allow", "Allow generation of adults only": "allow_adult", "Allow generation of people of all ages": "allow_all" },
-                default: "allow_adult", description: "The safety setting for allowing the generation of people in the image"
-            },
-            {
-                name: "safety_setting", type: OptionType.enum, enum: { "Block very few problematic prompts and responses": "block_none", "Block only few problematic prompts and responses": "block_only_high", "Block some problematic prompts and responses": "block_medium_and_above", "Strictest filtering": "block_low_and_above" },
-                default: "block_medium_and_above", description: "The overall safety setting"
-            },
-        ];
-
-
-        const outputOptions: ModelOptionInfoItem[] = [
-            {
-                name: "image_file_type", type: OptionType.enum, enum: { "JPEG": "image/jpeg", "PNG": "image/png" },
-                default: "image/png", description: "The file type of the generated image",
-                refresh: true,
-            },
-        ]
-
-        const jpegQuality: ModelOptionInfoItem = {
-            name: "jpeg_compression_quality", type: OptionType.numeric, min: 0, max: 100, default: 75,
-            integer: true, description: "The compression quality of the JPEG image",
-        }
-
-        if ((option as ImagenOptions)?.image_file_type === "image/jpeg") {
-            outputOptions.push(jpegQuality);
-        }
-        if (model.includes("generate")) {
-            //Generate models
-            const modeOptions: ModelOptionInfoItem[]
-                = [
-                    {
-                        name: "aspect_ratio", type: OptionType.enum, enum: { "1:1": "1:1", "4:3": "4:3", "3:4": "3:4", "16:9": "16:9", "9:16": "9:16" },
-                        default: "1:1", description: "The aspect ratio of the generated image"
-                    },
-                    {
-                        name: "add_watermark", type: OptionType.boolean, default: false, description: "Add an invisible watermark to the generated image, useful for detection of AI images"
-                    },
-
-                ];
-
-            const enhanceOptions: ModelOptionInfoItem[] = !model.includes("generate-001") ? [
-                {
-                    name: "enhance_prompt", type: OptionType.boolean, default: true, description: "VertexAI automatically rewrites the prompt to better reflect the prompt's intent."
-                },
-            ] : [];
-
-            return {
-                _option_id: "vertexai-imagen",
-                options: [
-                    ...commonOptions,
-                    ...modeOptions,
-                    ...outputOptions,
-                    ...enhanceOptions,
-                ]
-            };
-        }
-        if (model.includes("capability")) {
-            //Edit models
-            let guidanceScaleDefault = 75;
-            if ((option as ImagenOptions)?.edit_mode === ImagenTaskType.EDIT_MODE_INPAINT_INSERTION) {
-                guidanceScaleDefault = 60;
-            }
-
-            const modeOptions: ModelOptionInfoItem[] = [
-                {
-                    name: "edit_mode", type: OptionType.enum,
-                    enum: {
-                        "EDIT_MODE_INPAINT_REMOVAL": "EDIT_MODE_INPAINT_REMOVAL",
-                        "EDIT_MODE_INPAINT_INSERTION": "EDIT_MODE_INPAINT_INSERTION",
-                        "EDIT_MODE_BGSWAP": "EDIT_MODE_BGSWAP",
-                        "EDIT_MODE_OUTPAINT": "EDIT_MODE_OUTPAINT",
-                        "CUSTOMIZATION_SUBJECT": "CUSTOMIZATION_SUBJECT",
-                        "CUSTOMIZATION_STYLE": "CUSTOMIZATION_STYLE",
-                        "CUSTOMIZATION_CONTROLLED": "CUSTOMIZATION_CONTROLLED",
-                        "CUSTOMIZATION_INSTRUCT": "CUSTOMIZATION_INSTRUCT",
-                    },
-                    description: "The editing mode. CUSTOMIZATION options use few-shot learning to generate images based on a few examples."
-                },
-                
-                {
-                    name: "guidance_scale", type: OptionType.numeric, min: 0, max: 500, default: guidanceScaleDefault,
-                    integer: true, description: "How closely the generation follows the prompt"
-                }
-            ];
-
-            const maskOptions: ModelOptionInfoItem[] = ((option as ImagenOptions)?.edit_mode?.includes("EDIT")) ? [
-                {
-                    name: "mask_mode", type: OptionType.enum,
-                    enum: {
-                        "MASK_MODE_USER_PROVIDED": "MASK_MODE_USER_PROVIDED",
-                        "MASK_MODE_BACKGROUND": "MASK_MODE_BACKGROUND",
-                        "MASK_MODE_FOREGROUND": "MASK_MODE_FOREGROUND",
-                        "MASK_MODE_SEMANTIC": "MASK_MODE_SEMANTIC",
-                    },
-                    default: "MASK_MODE_USER_PROVIDED",
-                    description: "How should the mask for the generation be provided"
-                },
-                {
-                    name: "mask_dilation", type: OptionType.numeric, min: 0, max: 1,
-                    integer: true, description: "The mask dilation, grows the mask by a percetage of image width to compensate for imprecise masks."
-                },
-            ] : [];
-
-            const maskClassOptions: ModelOptionInfoItem[] = ((option as ImagenOptions)?.mask_mode === ImagenMaskMode.MASK_MODE_SEMANTIC) ? [
-                {
-                    name: "mask_class", type: OptionType.string_list, default: [],
-                    description: "Input Class IDs. Create a mask based on image class, based on https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api-customization#segment-ids"
-                }
-            ] : [];
-
-            const editOptions: ModelOptionInfoItem[] = (option as ImagenOptions)?.edit_mode?.includes("EDIT") ? [
-                {
-                    name: "edit_steps", type: OptionType.numeric, default: 75,
-                    integer: true, description: "The number of steps for the base image generation, more steps means more time and better quality"
-                },
-            ] : [];
-
-            const customizationOptions: ModelOptionInfoItem[] = (option as ImagenOptions)?.edit_mode === ImagenTaskType.CUSTOMIZATION_CONTROLLED
-            || (option as ImagenOptions)?.edit_mode === ImagenTaskType.CUSTOMIZATION_SUBJECT ? [
-                {
-                    name: "controlType", type: OptionType.enum, enum: { "Face Mesh": "CONTROL_TYPE_FACE_MESH", "Canny": "CONTROL_TYPE_CANNY", "Scribble": "CONTROL_TYPE_SCRIBBLE" },
-                    default: "CONTROL_TYPE_CANNY", description: "Method used to generate the control image"
-                },
-                {
-                    name: "controlImageComputation", type: OptionType.boolean, default: true, description: "Should the control image be computed from the input image, or is it provided"
-                }
-            ] : [];
-
-            return {
-                _option_id: "vertexai-imagen",
-                options: [
-                    ...modeOptions,
-                    ...commonOptions,
-                    ...maskOptions,
-                    ...maskClassOptions,
-                    ...editOptions,
-                    ...customizationOptions,
-                    ...outputOptions,
-                ]
-            };
-        }
-    }
-    else if (model.includes("gemini")) {
-        const max_tokens_limit = getGeminiMaxTokensLimit(model);
-        const excludeOptions = ["max_tokens", "presence_penalty"];
-        let commonOptions = textOptionsFallback.options.filter((option) => !excludeOptions.includes(option.name));
-        if (model.includes("1.5")) {
-            commonOptions = commonOptions.filter((option) => option.name !== "frequency_penalty");
-        }
-        const max_tokens: ModelOptionInfoItem[] = [{
-            name: SharedOptions.max_tokens, type: OptionType.numeric, min: 1, max: max_tokens_limit,
-            integer: true, step: 200, description: "The maximum number of tokens to generate"
-        }];
-        return {
-            _option_id: "vertexai-gemini",
-            options: [
-                ...max_tokens,
-                ...commonOptions,
-            ]
-        };
-    }
-    else if (model.includes("claude")) {
-        const max_tokens_limit = getClaudeMaxTokensLimit(model, option as VertexAIClaudeOptions);
-        const excludeOptions = ["max_tokens", "presence_penalty", "frequency_penalty"];
-        let commonOptions = textOptionsFallback.options.filter((option) => !excludeOptions.includes(option.name));
-        const max_tokens: ModelOptionInfoItem[] = [{
-            name: SharedOptions.max_tokens, type: OptionType.numeric, min: 1, max: max_tokens_limit,
-            integer: true, step: 200, description: "The maximum number of tokens to generate"
-        }];
-
-        if (model.includes("3-7")) {
-            const claudeModeOptions: ModelOptionInfoItem[] = [
-                {
-                    name: "thinking_mode",
-                    type: OptionType.boolean,
-                    default: false,
-                    description: "If true, use the extended reasoning mode"
-                },
-            ];
-            const claudeThinkingOptions: ModelOptionInfoItem[] = (option as VertexAIClaudeOptions)?.thinking_mode ? [
-                {
-                    name: "thinking_budget_tokens",
-                    type: OptionType.numeric,
-                    min: 1024,
-                    default: 4000,
-                    integer: true,
-                    step: 100,
-                    description: "The target number of tokens to use for reasoning, not a hard limit."
-                },
-            ] : [];
-
-            return {
-                _option_id: "vertexai-claude",
-                options: [
-                    ...max_tokens,
-                    ...commonOptions,
-                    ...claudeModeOptions,
-                    ...claudeThinkingOptions,
-                ]
-            };
-        }
-        return {
-            _option_id: "vertexai-claude",
-            options: [
-                ...max_tokens,
-                ...commonOptions,
-            ]
-        };
-    }
-    return textOptionsFallback;
-}
-function getGeminiMaxTokensLimit(model: string): number {
-    if (model.includes("thinking")) {
-        return 65536;
-    }
-    if (model.includes("ultra") || model.includes("vision")) {
-        return 2048;
-    }
-    return 8192;
-}
-function getClaudeMaxTokensLimit(model: string, option?: VertexAIClaudeOptions): number {
-    if (model.includes("3-7")) {
-        if (option && option?.thinking_mode) {
-            return 128000;
-        } else {
-            return 8192;
-        }
-    }
-    else if (model.includes("3-5")) {
-        return 8192;
-    }
-    else {
-        return 4096;
-    }
-}
-

package/src/types.ts

@@ -1,381 +0,0 @@
-import { JSONSchema4 } from 'json-schema';
-
-import { PromptFormatter } from './formatters/index.js';
-import { JSONObject } from './json.js';
-import { TextFallbackOptions } from './options.js';
-import { VertexAIOptions } from './options/vertexai.js';
-import { BedrockOptions } from './options/bedrock.js';
-import { OpenAiOptions } from './options/openai.js';
-
-export interface EmbeddingsOptions {
-    /**
-     * The text to generate the embeddings for. One of text or image is required.
-     */
-    text?: string;
-    /**
-     * The image to generate embeddings for
-     */
-    image?: string
-    /**
-     * The model to use to generate the embeddings. Optional.
-     */
-    model?: string;
-
-}
-
-export interface EmbeddingsResult {
-    /**
-     * The embedding vectors corresponding to the words in the input text.
-     */
-    values: number[];
-    /**
-     * The model used to hgenerate the embeddings.
-     */
-    model: string;
-    /**
-     * Number of tokens of the input text.
-     */
-    token_count?: number;
-
-}
-
-export interface ResultValidationError {
-    code: 'validation_error' | 'json_error' | 'content_policy_violation';
-    message: string;
-    data?: string;
-}
-
-//ResultT should be either JSONObject or string
-//Internal structure used in driver implementation.
-export interface CompletionChunkObject<ResultT = any> {
-    result: ResultT;
-    token_usage?: ExecutionTokenUsage;
-    finish_reason?: "stop" | "length" | string;
-}
-
-//Internal structure used in driver implementation.
-export type CompletionChunk = CompletionChunkObject | string;
-
-export interface ToolDefinition {
-    name: string,
-    description?: string,
-    input_schema: {
-        type: 'object';
-        properties?: unknown | null | undefined;
-        [k: string]: unknown;
-    },
-}
-
-export interface ToolUse {
-    id: string,
-    name: string,
-    input: JSONObject | null
-}
-
-//ResultT should be either JSONObject or string
-export interface Completion<ResultT = any> {
-    // the driver impl must return the result and optionally the token_usage. the execution time is computed by the extended abstract driver
-    result: ResultT;
-    token_usage?: ExecutionTokenUsage;
-    /**
-     * Contains the tools from which the model awaits information.
-     */
-    tool_use?: ToolUse[];
-    /**
-     * The finish reason as reported by the model: stop | length or other model specific values
-     */
-    finish_reason?: "stop" | "length" | "tool_use" | string;
-
-    /**
-     * Set only if a result validation error occured, otherwise if the result is valid the error field is undefined
-     * This can only be set if the result_schema is set and the reuslt could not be parsed as a json or if the result does not match the schema
-     */
-    error?: ResultValidationError;
-
-    /**
-     * The original response. Only included if the option include_original_response is set to true and the request is made using execute. Not supported when streaming.
-     */
-    original_response?: Record<string, any>;
-
-    /**
-     * The conversation context. This is an opaque structure that can be passed to the next request to restore the context.
-     */
-    conversation?: unknown;
-}
-
-export interface ImageGeneration {
-
-    images?: string[];
-
-}
-
-export interface ExecutionResponse<PromptT = any> extends Completion {
-    prompt: PromptT;
-    /**
-     * The time it took to execute the request in seconds
-     */
-    execution_time?: number;
-    /**
-     * The number of chunks for streamed executions
-     */
-    chunks?: number;
-}
-
-
-export interface CompletionStream<PromptT = any> extends AsyncIterable<string> {
-    completion: ExecutionResponse<PromptT> | undefined;
-}
-
-export interface Logger {
-    debug: (...obj: any[]) => void;
-    info: (...obj: any[]) => void;
-    warn: (...obj: any[]) => void;
-    error: (...obj: any[]) => void;
-}
-
-export interface DriverOptions {
-    logger?: Logger | "console";
-}
-
-//Options are split into PromptOptions, ModelOptions and ExecutionOptions.
-//ExecutionOptions are most often used within llumiverse as they are the most complete.
-//The base types are useful for external code that needs to interact with llumiverse.
-export interface PromptOptions {
-    model: string;
-    /**
-     * A custom formatter to use for format the final model prompt from the input prompt segments.
-     * If no one is specified the driver will choose a formatter compatible with the target model
-     */
-    format?: PromptFormatter;
-    result_schema?: JSONSchema4;
-}
-
-export interface ExecutionOptions extends PromptOptions {
-    /**
-     * If set to true the original response from the target LLM will be included in the response under the original_response field.
-     * This is useful for debugging and for some advanced use cases.
-     * It is ignored on streaming requests
-     */
-    include_original_response?: boolean;
-    model_options?: ModelOptions;
-    output_modality: Modalities;
-    /**
-     * Available tools for the request
-     */
-    tools?: ToolDefinition[];
-    /**
-     * This is an opaque structure that provides a conversation context
-     * Each driver implementation will return a conversation property in the execution response
-     * that can be passed here to restore the context when a new prompt is sent to the model.
-     */
-    conversation?: unknown | null;
-}
-
-//Common names to share between different models
-export enum SharedOptions {
-    //Text
-    max_tokens = "max_tokens",
-    temperature = "temperature",
-    top_p = "top_p",
-    top_k = "top_k",
-    presence_penalty = "presence_penalty",
-    frequency_penalty = "frequency_penalty",
-    stop_sequence = "stop_sequence",
-
-    //Image
-    seed = "seed",
-    number_of_images = "number_of_images",
-}
-
-export enum OptionType {
-    numeric = "numeric",
-    enum = "enum",
-    boolean = "boolean",
-    string_list = "string_list"
-}
-
-// ============== Model Options ===============
-
-export type ModelOptions = TextFallbackOptions | VertexAIOptions | BedrockOptions | OpenAiOptions;
-
-// ============== Option Info ===============
-
-export interface ModelOptionsInfo {
-    options: ModelOptionInfoItem[];
-    _option_id: string; //Should follow same ids as ModelOptions
-}
-
-export type ModelOptionInfoItem = NumericOptionInfo | EnumOptionInfo | BooleanOptionInfo | StringListOptionInfo;
-interface OptionInfoPrototype {
-    type: OptionType;
-    name: string;
-    description?: string;
-
-    //If this is true, whether other options apply is dependent on this option
-    //Therefore, if this option is changed, the set of available options should be refreshed.
-    refresh?: boolean;
-}
-
-export interface NumericOptionInfo extends OptionInfoPrototype {
-    type: OptionType.numeric;
-    value?: number;
-    min?: number;
-    max?: number;
-    step?: number;
-    integer?: boolean;
-    default?: number;
-}
-
-export interface EnumOptionInfo extends OptionInfoPrototype {
-    type: OptionType.enum;
-    value?: string;
-    enum: Record<string, string>;
-    default?: string;
-}
-
-export interface BooleanOptionInfo extends OptionInfoPrototype {
-    type: OptionType.boolean;
-    value?: boolean;
-    default?: boolean;
-}
-
-export interface StringListOptionInfo extends OptionInfoPrototype {
-    type: OptionType.string_list;
-    value?: string[];
-    default?: string[];
-}
-
-// ============== Prompts ===============
-export enum PromptRole {
-    safety = "safety",
-    system = "system",
-    user = "user",
-    assistant = "assistant",
-    negative = "negative",
-    mask = "mask",
-    /**
-     * Used to send the response of a tool
-     */
-    tool = "tool"
-}
-
-export interface PromptSegment {
-    role: PromptRole;
-    content: string;
-    /**
-     * The tool use id if the segment is a tool response
-     */
-    tool_use_id?: string;
-    files?: DataSource[]
-}
-
-export interface ExecutionTokenUsage {
-    prompt?: number;
-    result?: number;
-    total?: number;
-}
-
-export enum Modalities {
-    text = "text",
-    image = "image"
-}
-
-
-// ============== AI MODEL ==============
-
-export interface AIModel<ProviderKeys = string> {
-    id: string; //id of the model known by the provider
-    name: string; //human readable name
-    provider: ProviderKeys; //provider name
-    description?: string;
-    version?: string; //if any version is specified
-    type?: ModelType; //type of the model
-    tags?: string[]; //tags for searching
-    owner?: string; //owner of the model
-    status?: AIModelStatus; //status of the model
-    can_stream?: boolean; //if the model's reponse can be streamed
-    is_custom?: boolean; //if the model is a custom model (a trained model)
-    is_multimodal?: boolean //if the model support files and images
-}
-
-export enum AIModelStatus {
-    Available = "available",
-    Pending = "pending",
-    Stopped = "stopped",
-    Unavailable = "unavailable",
-    Unknown = "unknown"
-}
-
-/**
- * payload to list available models for an enviroment
- * @param environmentId id of the environment
- * @param query text to search for in model name/description
- * @param type type of the model
- * @param tags tags for searching
- */
-export interface ModelSearchPayload {
-    text: string;
-    type?: ModelType;
-    tags?: string[];
-    owner?: string;
-}
-
-
-export enum ModelType {
-    Classifier = "classifier",
-    Regressor = "regressor",
-    Clustering = "clustering",
-    AnomalyDetection = "anomaly-detection",
-    TimeSeries = "time-series",
-    Text = "text",
-    Image = "image",
-    Audio = "audio",
-    Video = "video",
-    Embedding = "embedding",
-    Chat = "chat",
-    Code = "code",
-    NLP = "nlp",
-    MultiModal = "multi-modal",
-    Test = "test",
-    Other = "other",
-    Unknown = "unknown"
-}
-
-
-// ============== training =====================
-
-
-
-export interface DataSource {
-    name: string;
-    mime_type?: string;
-    getStream(): Promise<ReadableStream<Uint8Array | string>>;
-    getURL(): Promise<string>;
-}
-
-export interface TrainingOptions {
-    name: string; // the new model name
-    model: string; // the model to train
-    params?: JSONObject; // the training parameters
-}
-
-export interface TrainingPromptOptions {
-    segments: PromptSegment[];
-    completion: string | JSONObject;
-    model: string; // the model to train
-    schema?: JSONSchema4; // the resuilt schema f any
-}
-
-export enum TrainingJobStatus {
-    running = "running",
-    succeeded = "succeeded",
-    failed = "failed",
-    cancelled = "cancelled",
-}
-
-export interface TrainingJob {
-    id: string; // id of the training job
-    status: TrainingJobStatus; // status of the training job - depends on the implementation
-    details?: string;
-    model?: string; // the name of the fine tuned model which is created
-}