@steve31415/baselib 3.40.1 → 3.41.0

package/dist/llm/complete.js

@@ -39,6 +39,19 @@ export async function llmComplete(request) {
             isRetryable: false,
         });
     }
+    // A request with images may only run on models that take them. The
+    // requested model not taking them is the caller's bug, so say so rather
+    // than silently answering from a fallback; fallbacks without image input
+    // are skipped in the loop below.
+    const hasImages = (request.images?.length ?? 0) > 0;
+    if (hasImages && !entry.supportsImages) {
+        throw new LLMError({
+            message: `Model ${request.model} does not accept image input. Image-capable models: ${Object.keys(MODEL_REGISTRY).filter((m) => MODEL_REGISTRY[m].supportsImages).join(', ')}`,
+            provider: entry.provider,
+            model: entry.apiModelId,
+            isRetryable: false,
+        });
+    }
     const maxTokens = request.maxTokens ?? DEFAULT_MAX_TOKENS;
     const temperature = request.temperature ?? DEFAULT_TEMPERATURE;
     const timeoutMs = request.timeoutMs ?? DEFAULT_TIMEOUT_MS;
@@ -75,6 +88,12 @@ export async function llmComplete(request) {
         const modelEntry = MODEL_REGISTRY[modelName];
         if (!modelEntry)
             continue;
+        if (hasImages && !modelEntry.supportsImages) {
+            request.logger?.debug('llm.skip_provider', {
+                model: modelName, provider: modelEntry.provider, reason: 'no image input',
+            });
+            continue;
+        }
         const apiKey = request.apiKeys[modelEntry.provider];
         if (!apiKey) {
             request.logger?.debug('llm.skip_provider', {
@@ -88,6 +107,7 @@ export async function llmComplete(request) {
             const result = await provider.complete({
                 apiModelId: modelEntry.apiModelId,
                 prompt: request.prompt,
+                images: hasImages ? request.images : undefined,
                 systemPrompt: request.systemPrompt,
                 maxTokens,
                 temperature,

package/dist/llm/index.d.ts

@@ -1,4 +1,4 @@
 export { llmComplete } from './complete.js';
 export { MODEL_REGISTRY } from './models.js';
 export type { ModelEntry } from './models.js';
-export { LLMError, EFFORT_LEVELS, type Effort, type LLMRequest, type LLMResponse, type FailedAttempt, type FinishCategory, type ProviderName, type ApiKeys, type JsonSchema, } from './types.js';
+export { LLMError, EFFORT_LEVELS, type Effort, type LLMRequest, type LLMResponse, type FailedAttempt, type FinishCategory, type ProviderName, type ApiKeys, type JsonSchema, type LLMImage, type LLMImageMediaType, } from './types.js';

package/dist/llm/models.d.ts

@@ -59,6 +59,13 @@ export interface ModelEntry {
      * `output_config.format` instead.
      */
     supportsForcedToolChoice?: boolean;
+    /**
+     * Whether this model accepts image input (`LLMRequest.images`). Defaults to
+     * `false`: a request with images is rejected for a model without it, and
+     * failover skips such fallbacks. Set only on models whose image handling
+     * was verified live (test/helpers/llm.ts records the probe).
+     */
+    supportsImages?: boolean;
 }
 /**
  * Output budget below which a `thinksByDefault` model is considered at risk of

package/dist/llm/models.js

@@ -26,10 +26,15 @@ export const MODEL_REGISTRY = {
     // hidden reasoning tokens billed as output — hence `supportsTemperature:
     // false` + `thinksByDefault: true` on each. (gpt-5.2 still accepts
     // temperature; verified the same day.)
+    //
+    // `supportsImages`: every entry still served read a JPEG beside a
+    // jsonSchema in the live probe of 2026-10-29 (test/helpers/llm.ts);
+    // the shut-down gemini-2.5 pair is left unmarked.
     // --- Fast tier ---
     'gemini-3.5-flash-lite': {
         provider: 'gemini',
         apiModelId: 'gemini-3.5-flash-lite',
+        supportsImages: true,
         fallbacks: ['gpt-5.6-luna', 'claude-haiku-4-5'],
         // Default thinking_level is `minimal` — reasoning overhead is negligible,
         // so no `thinksByDefault` headroom warning.
@@ -41,6 +46,7 @@ export const MODEL_REGISTRY = {
     'gemini-3.6-flash': {
         provider: 'gemini',
         apiModelId: 'gemini-3.6-flash',
+        supportsImages: true,
         fallbacks: ['gpt-5.6-luna', 'claude-haiku-4-5'],
         // Thinks by default (thinking_level defaults to `medium`); reasoning
         // tokens are billed as output and count against maxOutputTokens.
@@ -56,6 +62,7 @@ export const MODEL_REGISTRY = {
     'gpt-5.6-luna': {
         provider: 'openai',
         apiModelId: 'gpt-5.6-luna',
+        supportsImages: true,
         fallbacks: ['gemini-3.5-flash-lite', 'claude-haiku-4-5'],
         supportsTemperature: false,
         thinksByDefault: true,
@@ -63,6 +70,7 @@ export const MODEL_REGISTRY = {
     'claude-haiku-4-5': {
         provider: 'anthropic',
         apiModelId: 'claude-haiku-4-5-20251001',
+        supportsImages: true,
         fallbacks: ['gemini-3.5-flash-lite', 'gpt-5.6-luna'],
     },
     // Retiring 2026-10-16 — migrate callers to gemini-3.5-flash-lite.
@@ -74,12 +82,14 @@ export const MODEL_REGISTRY = {
     'gemini-3.1-flash-lite': {
         provider: 'gemini',
         apiModelId: 'gemini-3.1-flash-lite',
+        supportsImages: true,
         fallbacks: ['claude-sonnet-4-6'],
         supportsEffort: true,
     },
     'gemini-3-flash-preview': {
         provider: 'gemini',
         apiModelId: 'gemini-3-flash-preview',
+        supportsImages: true,
         fallbacks: ['claude-haiku-4-5', 'gpt-5.6-luna'],
         supportsEffort: true,
         thinkingLevels: ['minimal', 'low', 'medium', 'high'],
@@ -87,6 +97,7 @@ export const MODEL_REGISTRY = {
     'gpt-4o-mini': {
         provider: 'openai',
         apiModelId: 'gpt-4o-mini',
+        supportsImages: true,
         fallbacks: ['gemini-3.5-flash-lite', 'claude-haiku-4-5'],
     },
     // --- Mid tier ---
@@ -94,6 +105,7 @@ export const MODEL_REGISTRY = {
         provider: 'anthropic',
         supportsEffort: true,
         apiModelId: 'claude-sonnet-5',
+        supportsImages: true,
         fallbacks: ['gpt-5.6-terra', 'gemini-3.1-pro-preview'],
         // Sonnet 5 rejects `temperature` (like Opus 4.7+/Fable 5). It supports
         // adaptive thinking but does NOT think when the request omits `thinking`
@@ -103,6 +115,7 @@ export const MODEL_REGISTRY = {
     'gpt-5.6-terra': {
         provider: 'openai',
         apiModelId: 'gpt-5.6-terra',
+        supportsImages: true,
         fallbacks: ['claude-sonnet-5', 'gemini-3.1-pro-preview'],
         supportsTemperature: false,
         thinksByDefault: true,
@@ -111,6 +124,7 @@ export const MODEL_REGISTRY = {
         provider: 'anthropic',
         supportsEffort: true,
         apiModelId: 'claude-sonnet-4-6',
+        supportsImages: true,
         fallbacks: ['gpt-5.6-terra', 'gemini-3.1-pro-preview'],
     },
     // Legacy: consumer-retired 2026-02; API snapshots retire 2026-10 onward.
@@ -118,6 +132,7 @@ export const MODEL_REGISTRY = {
     'gpt-4o': {
         provider: 'openai',
         apiModelId: 'gpt-4o',
+        supportsImages: true,
         fallbacks: ['claude-sonnet-5', 'gemini-3.1-pro-preview'],
     },
     // --- Top tier ---
@@ -125,6 +140,7 @@ export const MODEL_REGISTRY = {
         provider: 'anthropic',
         supportsEffort: true,
         apiModelId: 'claude-opus-5-5',
+        supportsImages: true,
         fallbacks: ['gpt-5.6-sol', 'gemini-3.1-pro-preview'],
         // All verified live 2026-09-22: rejects `temperature` and forced
         // tool_choice (400 each); thinking is always on and its tokens count
@@ -139,6 +155,7 @@ export const MODEL_REGISTRY = {
         provider: 'anthropic',
         supportsEffort: true,
         apiModelId: 'claude-opus-5',
+        supportsImages: true,
         fallbacks: ['gpt-5.6-sol', 'gemini-3.1-pro-preview'],
         // Opus 4.7+ rejects `temperature` outright (400 invalid_request_error).
         supportsTemperature: false,
@@ -152,6 +169,7 @@ export const MODEL_REGISTRY = {
         provider: 'anthropic',
         supportsEffort: true,
         apiModelId: 'claude-opus-4-8',
+        supportsImages: true,
         fallbacks: ['gpt-5.6-sol', 'gemini-3.1-pro-preview'],
         // Opus 4.7+ rejects `temperature` outright (400 invalid_request_error).
         supportsTemperature: false,
@@ -160,6 +178,7 @@ export const MODEL_REGISTRY = {
         provider: 'anthropic',
         supportsEffort: true,
         apiModelId: 'claude-opus-4-6',
+        supportsImages: true,
         fallbacks: ['gpt-5.6-sol', 'gemini-3.1-pro-preview'],
     },
     // Released 2026-09-22; same limits as gpt-5.6-sol (1.05M / 128k). Verified
@@ -169,6 +188,7 @@ export const MODEL_REGISTRY = {
     'gpt-6-sol': {
         provider: 'openai',
         apiModelId: 'gpt-6-sol',
+        supportsImages: true,
         fallbacks: ['claude-opus-5-5', 'gemini-3.1-pro-preview'],
         supportsTemperature: false,
         thinksByDefault: true,
@@ -176,6 +196,7 @@ export const MODEL_REGISTRY = {
     'gpt-5.6-sol': {
         provider: 'openai',
         apiModelId: 'gpt-5.6-sol',
+        supportsImages: true,
         fallbacks: ['claude-opus-5-5', 'gemini-3.1-pro-preview'],
         supportsTemperature: false,
         thinksByDefault: true,
@@ -183,6 +204,7 @@ export const MODEL_REGISTRY = {
     'gpt-5.2': {
         provider: 'openai',
         apiModelId: 'gpt-5.2',
+        supportsImages: true,
         fallbacks: ['claude-opus-5-5', 'gemini-3.1-pro-preview'],
     },
     // Google's best Pro model as of 2026-08 — still Preview-labeled (no GA
@@ -191,6 +213,7 @@ export const MODEL_REGISTRY = {
     'gemini-3.1-pro-preview': {
         provider: 'gemini',
         apiModelId: 'gemini-3.1-pro-preview',
+        supportsImages: true,
         fallbacks: ['claude-opus-5-5', 'gpt-5.6-sol'],
         // Thinks by default (thinking_level defaults to `high`).
         thinksByDefault: true,

package/dist/llm/providers/anthropic.js

@@ -60,6 +60,18 @@ function toAnthropicStrictSchema(node) {
     }
     return out;
 }
+/** The user turn: the bare prompt string, or image blocks then the prompt as a text block. */
+function userContent(request) {
+    if (!request.images)
+        return request.prompt;
+    return [
+        ...request.images.map((image) => ({
+            type: 'image',
+            source: { type: 'base64', media_type: image.mediaType, data: image.data },
+        })),
+        { type: 'text', text: request.prompt },
+    ];
+}
 export const anthropicProvider = {
     name: 'anthropic',
     async complete(request) {
@@ -67,12 +79,13 @@ export const anthropicProvider = {
         request.logger?.info('llm.request', {
             provider: 'anthropic', model: request.apiModelId, endpoint,
             ...(request.callSite ? { callSite: request.callSite } : {}),
+            ...(request.images ? { images: request.images.length } : {}),
         });
         const startTime = Date.now();
         const body = {
             model: request.apiModelId,
             max_tokens: request.maxTokens,
-            messages: [{ role: 'user', content: request.prompt }],
+            messages: [{ role: 'user', content: userContent(request) }],
         };
         // Some models (Opus 4.7+, Sonnet 5, Fable 5, Opus 5.5) reject `temperature`
         // outright with a 400 — omit the field entirely rather than sending it.

package/dist/llm/providers/gemini.js

@@ -64,6 +64,7 @@ export const geminiProvider = {
         request.logger?.info('llm.request', {
             provider: 'gemini', model: request.apiModelId, endpoint,
             ...(request.callSite ? { callSite: request.callSite } : {}),
+            ...(request.images ? { images: request.images.length } : {}),
         });
         const startTime = Date.now();
         const generationConfig = {
@@ -95,7 +96,13 @@ export const geminiProvider = {
             generationConfig.thinkingConfig = { thinkingBudget: request.thinkingBudget };
         }
         const body = {
-            contents: [{ parts: [{ text: request.prompt }] }],
+            // Images go as inline-data parts ahead of the prompt's text part.
+            contents: [{
+                    parts: [
+                        ...(request.images ?? []).map((image) => ({ inlineData: { mimeType: image.mediaType, data: image.data } })),
+                        { text: request.prompt },
+                    ],
+                }],
             generationConfig,
         };
         if (request.systemPrompt) {

package/dist/llm/providers/openai.js

@@ -90,13 +90,26 @@ export const openaiProvider = {
         request.logger?.info('llm.request', {
             provider: 'openai', model: request.apiModelId, endpoint,
             ...(request.callSite ? { callSite: request.callSite } : {}),
+            ...(request.images ? { images: request.images.length } : {}),
         });
         const startTime = Date.now();
         const messages = [];
         if (request.systemPrompt) {
             messages.push({ role: 'system', content: request.systemPrompt });
         }
-        messages.push({ role: 'user', content: request.prompt });
+        // Images go as data-URL `image_url` parts ahead of the prompt's text part.
+        messages.push({
+            role: 'user',
+            content: request.images
+                ? [
+                    ...request.images.map((image) => ({
+                        type: 'image_url',
+                        image_url: { url: `data:${image.mediaType};base64,${image.data}` },
+                    })),
+                    { type: 'text', text: request.prompt },
+                ]
+                : request.prompt,
+        });
         const body = {
             model: request.apiModelId,
             max_completion_tokens: request.maxTokens,

package/dist/llm/types.d.ts

@@ -14,12 +14,26 @@ export interface JsonSchema {
     /** JSON Schema object. */
     schema: Record<string, unknown>;
 }
+/** Media types every image-capable provider accepts (Anthropic, OpenAI and Gemini alike). */
+export type LLMImageMediaType = 'image/jpeg' | 'image/png' | 'image/webp';
+/** An image sent beside the prompt: raw base64, no `data:` URL prefix. */
+export interface LLMImage {
+    mediaType: LLMImageMediaType;
+    data: string;
+}
 /** Request to llmComplete(). */
 export interface LLMRequest {
     /** Model name from the registry (e.g. 'claude-sonnet-4-6', 'gemini-2.5-flash'). */
     model: string;
     /** The user prompt. */
     prompt: string;
+    /**
+     * Images for the model to look at beside the prompt, placed before the
+     * prompt text in the user turn. Only models the registry marks
+     * `supportsImages` are tried: the requested model must be one (else the
+     * call is rejected up front), and failover skips fallbacks that are not.
+     */
+    images?: LLMImage[];
     /** Optional system prompt. */
     systemPrompt?: string;
     /** Max output tokens. Default: 2048. */
@@ -174,6 +188,8 @@ export interface ProviderAdapter {
 export interface ProviderCallRequest {
     apiModelId: string;
     prompt: string;
+    /** Present and non-empty only for a `supportsImages` model (gated in complete.ts). */
+    images?: LLMImage[];
     systemPrompt?: string;
     maxTokens: number;
     temperature: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.40.1",
+  "version": "3.41.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",