@startsimpli/llm 0.4.22 → 0.4.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@startsimpli/llm",
-  "version": "0.4.22",
+  "version": "0.4.23",
   "description": "Generic LLM service infrastructure — provider abstraction, retry logic, JSON extraction",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/__tests__/anthropic-provider.test.ts

@@ -169,3 +169,14 @@ describe('AnthropicProvider — recovering from a parameter the model rejects',
     expect(sent).toHaveLength(1);
   });
 });
+
+describe('images', () => {
+  it('sends images as base64 image blocks before the text', async () => {
+    await provider('claude-opus-5-5').generate('what is this?', 's', { images: [{ data: 'AAAA', mediaType: 'image/png' }] });
+    const body = sent[0] as unknown as { messages: Array<{ content: Array<Record<string, unknown>> }> };
+    const content = body.messages[0]!.content;
+    expect(content[0]).toEqual({ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'AAAA' } });
+    expect(content[1]!.type).toBe('text');
+    expect(String(content[1]!.text)).toContain('what is this?');
+  });
+});

package/src/__tests__/openai-provider.test.ts

@@ -265,3 +265,18 @@ describe('generateStream', () => {
     await expect(provider('gpt-6-astra').generateStream!('u', 's', () => {})).rejects.toThrow(/Empty response/);
   });
 });
+
+describe('images', () => {
+  it('sends images as image_url parts after the text, and plain text without them', async () => {
+    const p = provider('gpt-6-astra');
+    await p.generate('what is this?', 's', { images: [{ data: 'AAAA', mediaType: 'image/png' }] });
+    await p.generate('plain', 's');
+    const withImage = sent[0] as unknown as { messages: Array<{ role: string; content: unknown }> };
+    expect(withImage.messages[1]!.content).toEqual([
+      { type: 'text', text: 'what is this?' },
+      { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },
+    ]);
+    const plain = sent[1] as unknown as { messages: Array<{ role: string; content: unknown }> };
+    expect(plain.messages[1]!.content).toBe('plain');
+  });
+});

package/src/index.ts

@@ -8,6 +8,7 @@ export type {
   LLMResponse,
   GenerateOptions,
   ReasoningEffort,
+  LLMImage,
   ProviderConfig,
   JsonExtractionResult,
   ValidationResult,

package/src/providers/anthropic.ts

@@ -11,9 +11,13 @@ import { LLM_DEFAULTS } from '../types';
 import { LLMProviderError } from '../errors';
 import type { LLMErrorCode } from '../errors';
 
+type AnthropicContentBlock =
+  | { type: 'text'; text: string }
+  | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } };
+
 interface AnthropicMessage {
   role: 'user' | 'assistant';
-  content: string;
+  content: string | AnthropicContentBlock[];
 }
 
 interface AnthropicRequest {
@@ -199,7 +203,18 @@ Please respond with a valid JSON object only. Do not include any text before or
     const requestBody: AnthropicRequest = {
       model,
       max_tokens: maxTokens,
-      messages: [{ role: 'user', content: jsonWrappedPrompt }],
+      messages: [{
+        role: 'user',
+        content: options?.images?.length
+          ? [
+              ...options.images.map((img) => ({
+                type: 'image' as const,
+                source: { type: 'base64' as const, media_type: img.mediaType, data: img.data },
+              })),
+              { type: 'text' as const, text: jsonWrappedPrompt },
+            ]
+          : jsonWrappedPrompt,
+      }],
       system: systemPrompt,
     };
     // Only for models known to take it. The Claude 5 family answers a request

package/src/providers/openai.ts

@@ -18,14 +18,28 @@ import type {
   NativeChatMessage,
   NativeToolResponse,
   ReasoningEffort,
+  LLMImage,
 } from '../types';
 import { LLM_DEFAULTS } from '../types';
 import { LLMProviderError } from '../errors';
 import type { LLMErrorCode } from '../errors';
 
+type OpenAIContentPart =
+  | { type: 'text'; text: string }
+  | { type: 'image_url'; image_url: { url: string } };
+
 interface OpenAIMessage {
   role: 'system' | 'user' | 'assistant';
-  content: string;
+  content: string | OpenAIContentPart[];
+}
+
+/** The user message: plain text, or text plus images for vision models. */
+function userContent(prompt: string, images?: LLMImage[]): string | OpenAIContentPart[] {
+  if (!images?.length) return prompt;
+  return [
+    { type: 'text', text: prompt },
+    ...images.map((img) => ({ type: 'image_url' as const, image_url: { url: `data:${img.mediaType};base64,${img.data}` } })),
+  ];
 }
 
 interface OpenAIRequest {
@@ -324,7 +338,7 @@ export class OpenAIProvider implements ILLMProvider {
       model,
       messages: [
         { role: 'system' as const, content: systemPrompt },
-        { role: 'user' as const, content: userPrompt },
+        { role: 'user' as const, content: userContent(userPrompt, options?.images) },
       ],
       response_format: { type: 'json_object' as const },
       stream: true,
@@ -424,7 +438,7 @@ export class OpenAIProvider implements ILLMProvider {
       model,
       messages: [
         { role: 'system' as const, content: systemPrompt },
-        { role: 'user' as const, content: userPrompt },
+        { role: 'user' as const, content: userContent(userPrompt, options?.images) },
       ],
       response_format: { type: 'json_object' as const },
     };

package/src/types.ts

@@ -30,6 +30,12 @@ export interface LLMResponse {
 /**
  * Options passed through to the provider for a single generation call.
  */
+/** An image attached to a prompt: base64 data (no data: prefix) and its media type. */
+export interface LLMImage {
+  data: string;
+  mediaType: 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp';
+}
+
 export interface GenerateOptions {
   /** Model to use (provider-specific) */
   model?: string;
@@ -45,6 +51,11 @@ export interface GenerateOptions {
    * provider adds headroom for the reasoning on top.
    */
   reasoningEffort?: ReasoningEffort;
+  /**
+   * Images sent with the user prompt, for vision-capable models (e.g. to have
+   * a model look at a rendering of what it designed).
+   */
+  images?: LLMImage[];
   /** Arbitrary extra data the caller wants to thread through (e.g. domain preferences) */
   extra?: Record<string, unknown>;
 }