@llumiverse/core 0.17.0 → 0.18.0

package/src/capability/openai.ts

@@ -0,0 +1,124 @@
+import { ModelModalities } from "@llumiverse/common";
+
+// Record of OpenAI model capabilities keyed by model ID (lowercased)
+const RECORD_MODEL_CAPABILITIES: Record<string, { input: ModelModalities; output: ModelModalities; tool_support?: boolean }> = {
+    "chatgpt-4o-latest": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-3.5-turbo": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-3.5-turbo-0125": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-3.5-turbo-1106": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-3.5-turbo-16k": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-3.5-turbo-instruct": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-3.5-turbo-instruct-0914": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-4": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-4-0125-preview": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-4-0613": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-4-1106-preview": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-4-turbo": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-4-turbo-2024-04-09": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-4-turbo-preview": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-4.1": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4.1-2025-04-14": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4.1-mini": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4.1-mini-2025-04-14": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4.1-nano": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4.1-nano-2025-04-14": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4.5-preview": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4.5-preview-2025-02-27": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o-2024-05-13": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o-2024-08-06": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o-2024-11-20": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o-mini": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o-mini-2024-07-18": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o-mini-search-preview": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o-mini-search-preview-2025-03-11": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o-search-preview": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o-search-preview-2025-03-11": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-image-1": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: false, image: true, video: false, audio: false, embed: false }, tool_support: false },
+    "o1": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o1-2024-12-17": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o1-mini": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "o1-mini-2024-09-12": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "o1-preview": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o1-preview-2024-09-12": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o1-pro": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o1-pro-2025-03-19": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o3": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o3-2025-04-16": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o3-mini": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o3-mini-2025-01-31": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o4-mini": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o4-mini-2025-04-16": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true }
+};
+
+// Populate RECORD_FAMILY_CAPABILITIES as a const record (lowest common denominator for each family)
+const RECORD_FAMILY_CAPABILITIES: Record<string, { input: ModelModalities; output: ModelModalities; tool_support?: boolean }> = {
+    "gpt-3.5-turbo": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-4": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gpt-4.1": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4.5": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gpt-4o": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o1": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o1-mini": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "o1-pro": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o3": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o3-mini": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "o4-mini": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true }
+};
+
+// Fallback pattern lists for inferring modalities and tool support
+const IMAGE_INPUT_MODELS = ["image"];
+const VIDEO_INPUT_MODELS = ["video"];
+const AUDIO_INPUT_MODELS = ["audio"];
+const TEXT_INPUT_MODELS = ["text"];
+const IMAGE_OUTPUT_MODELS = ["image"];
+const VIDEO_OUTPUT_MODELS = ["video"];
+const AUDIO_OUTPUT_MODELS = ["audio"];
+const TEXT_OUTPUT_MODELS = ["text"];
+const EMBEDDING_OUTPUT_MODELS = ["embed"];
+const TOOL_SUPPORT_MODELS = ["tool", "o1", "o3", "o4", "gpt-4.1", "gpt-4o"];
+
+function modelMatches(modelName: string, patterns: string[]): boolean {
+    return patterns.some(pattern => modelName.includes(pattern));
+}
+
+function normalizeOpenAIModelName(modelName: string): string {
+    return modelName.toLowerCase();
+}
+
+/**
+ * Get the full ModelCapabilities for an OpenAI model.
+ * Checks RECORD_MODEL_CAPABILITIES first, then falls back to pattern-based inference.
+ */
+export function getModelCapabilitiesOpenAI(model: string): { input: ModelModalities; output: ModelModalities; tool_support?: boolean } {
+    let normalized = normalizeOpenAIModelName(model);
+    const record = RECORD_MODEL_CAPABILITIES[normalized];
+    if (record) return record;
+    let bestFamilyKey = undefined;
+    let bestFamilyLength = 0;
+    for (const key of Object.keys(RECORD_FAMILY_CAPABILITIES)) {
+        if (normalized.startsWith(key) && key.length > bestFamilyLength) {
+            bestFamilyKey = key;
+            bestFamilyLength = key.length;
+        }
+    }
+    if (bestFamilyKey) {
+        return RECORD_FAMILY_CAPABILITIES[bestFamilyKey];
+    }
+    const input: ModelModalities = {
+        text: modelMatches(normalized, TEXT_INPUT_MODELS) || undefined,
+        image: modelMatches(normalized, IMAGE_INPUT_MODELS) || undefined,
+        video: modelMatches(normalized, VIDEO_INPUT_MODELS) || undefined,
+        audio: modelMatches(normalized, AUDIO_INPUT_MODELS) || undefined,
+        embed: false
+    };
+    const output: ModelModalities = {
+        text: modelMatches(normalized, TEXT_OUTPUT_MODELS) || undefined,
+        image: modelMatches(normalized, IMAGE_OUTPUT_MODELS) || undefined,
+        video: modelMatches(normalized, VIDEO_OUTPUT_MODELS) || undefined,
+        audio: modelMatches(normalized, AUDIO_OUTPUT_MODELS) || undefined,
+        embed: modelMatches(normalized, EMBEDDING_OUTPUT_MODELS) || undefined
+    };
+    const tool_support = modelMatches(normalized, TOOL_SUPPORT_MODELS) || undefined;
+    return { input, output, tool_support };
+}

package/src/capability/vertexai.ts

@@ -0,0 +1,88 @@
+import { ModelModalities } from "@llumiverse/common";
+
+// Record of Vertex AI model capabilities keyed by model ID (last path segment, lowercased)
+const RECORD_MODEL_CAPABILITIES: Record<string, { input: ModelModalities; output: ModelModalities; tool_support?: boolean }> = {
+    "imagen-3.0-generate-002": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: false, image: true, video: false, audio: false, embed: false }, tool_support: false },
+    "imagen-3.0-capability-001": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: false, image: true, video: false, audio: false, embed: false }, tool_support: false },
+    "gemini-1.5-flash-002": { input: { text: true, image: true, video: true, audio: true, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gemini-1.5-pro-002": { input: { text: true, image: true, video: true, audio: true, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gemini-2.0-flash-001": { input: { text: true, image: true, video: true, audio: true, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gemini-2.0-flash-lite-001": { input: { text: true, image: true, video: true, audio: true, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: false },
+    "gemini-2.5-flash-preview-04-17": { input: { text: true, image: true, video: true, audio: true, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gemini-2.5-pro-preview-05-06": { input: { text: true, image: true, video: true, audio: true, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "claude-3-opus": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "claude-3-haiku": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "claude-3-5-sonnet": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "claude-3-5-haiku": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "claude-3-5-sonnet-v2": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "claude-3-7-sonnet": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true }
+};
+
+// Populate RECORD_FAMILY_CAPABILITIES as a const record (lowest common denominator for each family)
+const RECORD_FAMILY_CAPABILITIES: Record<string, { input: ModelModalities; output: ModelModalities; tool_support?: boolean }> = {
+    "gemini-1.5": { input: { text: true, image: true, video: true, audio: true, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gemini-2.0": { input: { text: true, image: true, video: true, audio: true, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "gemini-2.5": { input: { text: true, image: true, video: true, audio: true, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "imagen-3.0": { input: { text: true, image: true, video: false, audio: false, embed: false }, output: { text: false, image: true, video: false, audio: false, embed: false }, tool_support: false },
+    "claude-3-5": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "claude-3-7": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true },
+    "claude-3": { input: { text: true, image: false, video: false, audio: false, embed: false }, output: { text: true, image: false, video: false, audio: false, embed: false }, tool_support: true }
+};
+
+// Fallback pattern lists for inferring modalities and tool support
+const IMAGE_INPUT_MODELS = ["image"];
+const VIDEO_INPUT_MODELS = ["video"];
+const AUDIO_INPUT_MODELS = ["audio"];
+const TEXT_INPUT_MODELS = ["text"];
+const IMAGE_OUTPUT_MODELS = ["image"];
+const VIDEO_OUTPUT_MODELS = ["video"];
+const AUDIO_OUTPUT_MODELS = ["audio"];
+const TEXT_OUTPUT_MODELS = ["text"];
+const EMBEDDING_OUTPUT_MODELS = ["embed"];
+const TOOL_SUPPORT_MODELS = ["tool", "sonnet", "opus", "gemini", "claude-3-5", "claude-3-7"];
+
+function modelMatches(modelName: string, patterns: string[]): boolean {
+    return patterns.some(pattern => modelName.includes(pattern));
+}
+
+function normalizeVertexAIModelName(modelName: string): string {
+    const segments = modelName.toLowerCase().split("/");
+    return segments[segments.length - 1];
+}
+
+/**
+ * Get the full ModelCapabilities for a Vertex AI model.
+ * Checks RECORD_MODEL_CAPABILITIES first, then falls back to pattern-based inference.
+ */
+export function getModelCapabilitiesVertexAI(model: string): { input: ModelModalities; output: ModelModalities; tool_support?: boolean } {
+    let normalized = normalizeVertexAIModelName(model);
+    const record = RECORD_MODEL_CAPABILITIES[normalized];
+    if (record) return record;
+    let bestFamilyKey = undefined;
+    let bestFamilyLength = 0;
+    for (const key of Object.keys(RECORD_FAMILY_CAPABILITIES)) {
+        if (normalized.startsWith(key) && key.length > bestFamilyLength) {
+            bestFamilyKey = key;
+            bestFamilyLength = key.length;
+        }
+    }
+    if (bestFamilyKey) {
+        return RECORD_FAMILY_CAPABILITIES[bestFamilyKey];
+    }
+    const input: ModelModalities = {
+        text: modelMatches(normalized, TEXT_INPUT_MODELS) || undefined,
+        image: modelMatches(normalized, IMAGE_INPUT_MODELS) || undefined,
+        video: modelMatches(normalized, VIDEO_INPUT_MODELS) || undefined,
+        audio: modelMatches(normalized, AUDIO_INPUT_MODELS) || undefined,
+        embed: false
+    };
+    const output: ModelModalities = {
+        text: modelMatches(normalized, TEXT_OUTPUT_MODELS) || undefined,
+        image: modelMatches(normalized, IMAGE_OUTPUT_MODELS) || undefined,
+        video: modelMatches(normalized, VIDEO_OUTPUT_MODELS) || undefined,
+        audio: modelMatches(normalized, AUDIO_OUTPUT_MODELS) || undefined,
+        embed: modelMatches(normalized, EMBEDDING_OUTPUT_MODELS) || undefined
+    };
+    const tool_support = modelMatches(normalized, TOOL_SUPPORT_MODELS) || undefined;
+    return { input, output, tool_support };
+}

package/src/capability.ts

@@ -0,0 +1,49 @@
+import { getModelCapabilitiesBedrock } from "./capability/bedrock.js";
+import { getModelCapabilitiesOpenAI } from "./capability/openai.js";
+import { getModelCapabilitiesVertexAI } from "./capability/vertexai.js";
+import { ModelCapabilities, ModelModalities } from "@llumiverse/common";
+
+export function getModelCapabilities(model: string, provider?: string): ModelCapabilities {
+    const capabilities = _getModelCapabilities(model, provider);
+    // Globally disable audio and video for all models, as we don't support them yet
+    // We also do not support tool use while streaming
+    // TODO: Remove this when we add support.
+    capabilities.input.audio = false;
+    capabilities.output.audio = false;
+    capabilities.output.video = false;
+    capabilities.tool_support_streaming = false;
+    return capabilities;
+}
+
+function _getModelCapabilities(model: string, provider?: string): ModelCapabilities {
+    switch (provider?.toLowerCase()) {
+        case "vertexai":
+            return getModelCapabilitiesVertexAI(model);
+        case "openai":
+            return getModelCapabilitiesOpenAI(model);
+        case "bedrock":
+            return getModelCapabilitiesBedrock(model);
+        default:
+            // Guess the provider based on the model name
+            if (model.startsWith("gpt")) {
+                return getModelCapabilitiesOpenAI(model);
+            } else if (model.startsWith("publishers/")) {
+                return getModelCapabilitiesVertexAI(model);
+            } else if (model.startsWith("arn:aws")) {
+                return getModelCapabilitiesBedrock(model);
+            }
+            // Fallback to a generic model with no capabilities
+            return { input: {}, output: {} } satisfies ModelCapabilities;
+    }
+}
+
+export function supportsToolUse(model: string, provider?: string, streaming: boolean = false): boolean {
+    const capabilities = getModelCapabilities(model, provider);
+    return streaming ? !!capabilities.tool_support_streaming : !!capabilities.tool_support;
+}
+
+export function modelModalitiesToArray(modalities: ModelModalities): string[] {
+    return Object.entries(modalities)
+        .filter(([_, isSupported]) => isSupported)
+        .map(([modality]) => modality);
+}

package/src/formatters/commons.ts

@@ -1,4 +1,4 @@
-import { JSONSchema } from "../types.js";
+import { JSONSchema } from "@llumiverse/common";
 
 export function getJSONSafetyNotice(schema: JSONSchema) {
     return "The answer must be a JSON object using the following JSON Schema:\n" + JSON.stringify(schema, undefined, 2);

package/src/formatters/generic.ts

@@ -1,5 +1,5 @@
-import { JSONSchema } from "../types.js";
-import { PromptRole, PromptSegment } from "../types.js";
+import { JSONSchema } from "@llumiverse/common";
+import { PromptRole, PromptSegment } from "@llumiverse/common";
 import { getJSONSafetyNotice } from "./commons.js";
 
 interface Labels {

package/src/formatters/index.ts

@@ -1,8 +1,3 @@
-import { JSONSchema } from "../types.js";
-import { PromptSegment } from "../types.js";
-
-export type PromptFormatter<T = any> = (messages: PromptSegment[], schema?: JSONSchema) => T;
-
 export * from "./commons.js";
 export * from "./generic.js";
 export * from "./openai.js";

package/src/formatters/nova.ts

@@ -1,6 +1,6 @@
-import { JSONSchema } from "../types.js";
-import { PromptRole, PromptSegment, readStreamAsBase64 } from "../index.js";
-//import { readStreamAsBase64 } from "../stream.js";
+import { JSONSchema } from "@llumiverse/common";
+import { PromptRole, PromptSegment } from "@llumiverse/common";
+import { readStreamAsBase64 } from "../stream.js";
 import { getJSONSafetyNotice } from "./commons.js";
 
 export interface NovaMessage {
@@ -105,7 +105,7 @@ export async function formatNovaPrompt(segments: PromptSegment[], schema?: JSONS
         safety.push("IMPORTANT: " + getJSONSafetyNotice(schema));
     }
 
-    // messages must contains at least 1 item. If the prompt doesn;t contains a user message (but only system messages)
+    // messages must contains at least 1 item. If the prompt does not contains a user message (but only system messages)
     // we need to put the system messages in the messages array
 
     let systemMessage = system.join('\n').trim();
@@ -119,7 +119,7 @@ export async function formatNovaPrompt(segments: PromptSegment[], schema?: JSONS
         systemMessage = systemMessage + '\n\nIMPORTANT: ' + safety.join('\n');
     }
 
-    /*start Nova's message to amke sure it answers properly in JSON
+    /*start Nova's message to make sure it answers properly in JSON
    if enabled, this requires to add the { to Nova's response*/
 
     if (schema) {
@@ -131,7 +131,7 @@ export async function formatNovaPrompt(segments: PromptSegment[], schema?: JSONS
         });
     }
 
-    // put system mesages first and safety last
+    // put system messages first and safety last
     return {
         system: systemMessage ? [{ text: systemMessage }] : [{ text: "" }],
         messages: messages,

package/src/formatters/openai.ts

@@ -1,6 +1,6 @@
-import { PromptRole, PromptOptions } from "../index.js";
+import { PromptRole, PromptOptions } from "@llumiverse/common";
 import { readStreamAsBase64 } from "../stream.js";
-import { PromptSegment } from "../types.js";
+import { PromptSegment } from "@llumiverse/common";
 
 
 export interface OpenAITextMessage {
@@ -33,6 +33,20 @@ export interface OpenAIContentPartImage {
     }
 }
 
+//Anything before gpt-4o-mini, gpt-4o-mini-2024-07-18, and gpt-4o-2024-08-06 model snapshots.
+//https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat
+export const noStructuredOutputModels: string[] = [
+    "turbo",
+    "davinci",
+    "babbage",
+    "curie",
+    "chatgpt-4o",
+    "gpt-4",
+    "gpt-4o-2024-05-13", //later gpt-4o does support structured output
+    "o1-preview",
+    "o1-mini",
+]
+
 /**
  * OpenAI text only prompts
  * @param segments
@@ -56,7 +70,7 @@ export function formatOpenAILikeTextPrompt(segments: PromptSegment[]): OpenAITex
         }
     }
 
-    // put system mesages first and safety last
+    // put system messages first and safety last
     return system.concat(user).concat(safety);
 }
 
@@ -70,7 +84,7 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
 
         const parts: (OpenAIContentPartImage | OpenAIContentPartText)[] = [];
 
-        //generate the parts based on promptsegment
+        //generate the parts based on PromptSegment
         if (msg.files) {
             for (const file of msg.files) {
                 const stream = await file.getStream();
@@ -147,7 +161,7 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
         })
     }
 
-    // put system mesages first and safety last
+    // put system messages first and safety last
     return ([] as OpenAIInputMessage[]).concat(system).concat(others).concat(safety);
 
 }

package/src/index.ts

@@ -1,5 +1,6 @@
 export * from "./Driver.js";
 export * from "./json.js";
 export * from "./stream.js";
-export * from "./types.js";
-export * from "./options.js";
+export * from "./options.js";
+export * from "./capability.js";
+export * from "@llumiverse/common";

package/src/json.ts

@@ -1,3 +1,4 @@
+import { JSONValue } from "@llumiverse/common";
 
 function extractJsonFromText(text: string): string {
     const start = text.indexOf("{");
@@ -10,15 +11,6 @@ export function extractAndParseJSON(text: string): JSONValue {
     return parseJSON(extractJsonFromText(text));
 }
 
-export type JSONPrimitive = string | number | boolean | null;
-export type JSONArray = JSONValue[];
-export type JSONObject = { [key: string]: JSONValue };
-export type JSONComposite = JSONArray | JSONObject;
-export type JSONValue = JSONPrimitive | JSONComposite;
-
-
-
-
 const RX_DQUOTE = /^"([^"\\]|\\.)*"/us;
 const RX_SQUOTE = /^'([^'\\]|\\.)*'/us;
 const RX_NUMBER = /^-?\d+(\.\d+)?/;
@@ -192,7 +184,7 @@ export function parseJSON(text: string): JSONValue {
         // use a relaxed parser
         try {
             return JsonParser.parse(text);
-        } catch (err2: any) { // throw the original error            
+        } catch (err2: any) { // throw the original error
             throw err;
         }
     }

package/src/options.ts

@@ -1,53 +1,15 @@
-import { ModelOptions, ModelOptionsInfo, OptionType, SharedOptions } from "./types.js";
-import { getBedrockOptions } from "./options/bedrock.js";
-import { getVertexAiOptions } from "./options/vertexai.js";
-import { getOpenAiOptions } from "./options/openai.js";
-import { getGroqOptions } from "./options/groq.js";
+import {
+    getBedrockOptions,
+    textOptionsFallback,
+    getGroqOptions,
+    getOpenAiOptions,
+    getVertexAiOptions,
+    ModelOptions,
+    ModelOptionsInfo
+} from "@llumiverse/common";
 
-export interface TextFallbackOptions {
-    _option_id: "text-fallback";    //For specific models should be format as "provider-model"
-    max_tokens?: number;
-    temperature?: number;
-    top_p?: number;
-    top_k?: number;
-    presence_penalty?: number;
-    frequency_penalty?: number;
-    stop_sequence?: string[];
-}
-
-export const textOptionsFallback: ModelOptionsInfo = {
-    _option_id: "text-fallback",
-    options: [
-        {
-            name: SharedOptions.max_tokens, type: OptionType.numeric, min: 1,
-            integer: true, step: 200, description: "The maximum number of tokens to generate"
-        },
-        {
-            name: SharedOptions.temperature, type: OptionType.numeric, min: 0.0, default: 0.7,
-            integer: false, step: 0.1, description: "A higher temperature biases toward less likely tokens, making the model more creative"
-        },
-        {
-            name: SharedOptions.top_p, type: OptionType.numeric, min: 0, max: 1,
-            integer: false, step: 0.1, description: "Limits token sampling to the cumulative probability of the top p tokens"
-        },
-        {
-            name: SharedOptions.top_k, type: OptionType.numeric, min: 1,
-            integer: true, step: 1, description: "Limits token sampling to the top k tokens"
-        },
-        {
-            name: SharedOptions.presence_penalty, type: OptionType.numeric, min: -2.0, max: 2.0,
-            integer: false, step: 0.1, description: "Penalise tokens if they appear at least once in the text"
-        },
-        {
-            name: SharedOptions.frequency_penalty, type: OptionType.numeric, min: -2.0, max: 2.0,
-            integer: false, step: 0.1, description: "Penalise tokens based on their frequency in the text"
-        },
-        { name: SharedOptions.stop_sequence, type: OptionType.string_list, value: [], description: "The generation will halt if one of the stop sequences is output" },
-    ]
-};
-
-export function getOptions(provider?: string, model?: string, options?: ModelOptions): ModelOptionsInfo {
-    switch (provider) {
+export function getOptions(model: string, provider?: string, options?: ModelOptions): ModelOptionsInfo {
+    switch (provider?.toLowerCase()) {
         case "bedrock":
             return getBedrockOptions(model ?? "", options);
         case "vertexai":
@@ -59,4 +21,4 @@ export function getOptions(provider?: string, model?: string, options?: ModelOpt
         default:
             return textOptionsFallback;
     }
-}
+}

package/src/resolver.ts

@@ -1,8 +1,8 @@
 /**
  * Get the property named by "name" of the given object
- * If an array is idnexed using a string key then a map is done and an array with the content of the properties with that name are returned
+ * If an array is indexed using a string key then a map is done and an array with the content of the properties with that name are returned
  * Ex: docs.text => will return an array of text properties of the docs array
- * @param object the obejct
+ * @param object the object
  * @param name the name of the property.
  * @returns the property value
  */

package/src/validation.ts

@@ -2,7 +2,7 @@ import { Ajv } from 'ajv';
 import addFormats from 'ajv-formats';
 import { extractAndParseJSON } from "./json.js";
 import { resolveField } from './resolver.js';
-import { ResultValidationError } from "./types.js";
+import { ResultValidationError } from "@llumiverse/common";
 
 
 const ajv = new Ajv({
@@ -45,7 +45,7 @@ export function validateResult(data: any, schema: Object) {
     const valid = validate(json);
 
     if (!valid && validate.errors) {
-        let errors = [];    
+        let errors = [];
 
         for (const e of validate.errors) {
             const path = e.instancePath.split("/").slice(1);
@@ -55,7 +55,7 @@ export function validateResult(data: any, schema: Object) {
             const schemaField = resolveField(schema, schemaPath.slice(0, -3));
 
             //ignore date if empty or null
-            if (!value 
+            if (!value
                 && ["date", "date-time"].includes(schemaFieldFormat)
                 && !schemaField?.required?.includes(path[path.length - 1])) {
                 continue;