@startsimpli/llm 0.4.21 → 0.4.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@startsimpli/llm",
-  "version": "0.4.21",
+  "version": "0.4.23",
   "description": "Generic LLM service infrastructure — provider abstraction, retry logic, JSON extraction",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/__tests__/anthropic-provider.test.ts

@@ -169,3 +169,14 @@ describe('AnthropicProvider — recovering from a parameter the model rejects',
     expect(sent).toHaveLength(1);
   });
 });
+
+describe('images', () => {
+  it('sends images as base64 image blocks before the text', async () => {
+    await provider('claude-opus-5-5').generate('what is this?', 's', { images: [{ data: 'AAAA', mediaType: 'image/png' }] });
+    const body = sent[0] as unknown as { messages: Array<{ content: Array<Record<string, unknown>> }> };
+    const content = body.messages[0]!.content;
+    expect(content[0]).toEqual({ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'AAAA' } });
+    expect(content[1]!.type).toBe('text');
+    expect(String(content[1]!.text)).toContain('what is this?');
+  });
+});

package/src/__tests__/openai-provider.test.ts

@@ -212,3 +212,71 @@ describe('response provenance', () => {
     expect(res.model).toBe('server-reported-model');
   });
 });
+
+/** A fetch Response whose body streams OpenAI SSE chunks. */
+function sse(chunks: object[], split = 7) {
+  const text = chunks.map((c) => `data: ${JSON.stringify(c)}\n\n`).join('') + 'data: [DONE]\n\n';
+  // Deliver in small arbitrary slices so the parser must buffer partial lines.
+  const parts: Uint8Array[] = [];
+  const enc = new TextEncoder();
+  for (let i = 0; i < text.length; i += split) parts.push(enc.encode(text.slice(i, i + split)));
+  const body = new ReadableStream<Uint8Array>({ start(c) { parts.forEach((p) => c.enqueue(p)); c.close(); } });
+  return { ok: true, status: 200, body } as unknown as Response;
+}
+
+describe('generateStream', () => {
+  const delta = (content: string) => ({ model: 'gpt-6-astra', choices: [{ index: 0, delta: { content }, finish_reason: null }] });
+
+  it('calls onDelta with each piece of content in order and returns the whole response', async () => {
+    fetchMock.mockImplementationOnce(async (_u: unknown, init: RequestInit) => {
+      sent.push(JSON.parse(String(init.body)) as Sent);
+      return sse([
+        delta('{"pie'), delta('ces":[{"name":"He'), delta('ad"}]}'),
+        { model: 'gpt-6-astra', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] },
+        { model: 'gpt-6-astra', choices: [], usage: { prompt_tokens: 9, completion_tokens: 5, total_tokens: 14 } },
+      ]);
+    });
+    const seen: string[] = [];
+    const res = await provider('gpt-6-astra').generateStream!('u', 's', (d) => seen.push(d));
+    expect(seen).toEqual(['{"pie', 'ces":[{"name":"He', 'ad"}]}']);
+    expect(res.content).toBe('{"pieces":[{"name":"Head"}]}');
+    expect(res.model).toBe('gpt-6-astra');
+    expect(res.provider).toBe('openai');
+    expect(res.finishReason).toBe('stop');
+    expect(res.usage).toEqual({ promptTokens: 9, completionTokens: 5, totalTokens: 14 });
+    const body = sent[0] as Sent & { stream?: boolean; stream_options?: unknown };
+    expect(body.stream).toBe(true);
+    expect(body.stream_options).toEqual({ include_usage: true });
+  });
+
+  it('adapts to a refused parameter before the stream starts', async () => {
+    reject({ code: 'unsupported_value', param: 'temperature', message: "Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported." });
+    fetchMock.mockImplementationOnce(async (_u: unknown, init: RequestInit) => {
+      sent.push(JSON.parse(String(init.body)) as Sent);
+      return sse([delta('{"ok":true}')]);
+    });
+    const res = await provider('gpt-6-sol').generateStream!('u', 's', () => {});
+    expect(res.content).toBe('{"ok":true}');
+    expect(sent[1]!.temperature).toBeUndefined();
+  });
+
+  it('throws on an empty stream, like generate() does on empty content', async () => {
+    fetchMock.mockImplementationOnce(async () => sse([{ model: 'm', choices: [{ index: 0, delta: {}, finish_reason: 'length' }] }]));
+    await expect(provider('gpt-6-astra').generateStream!('u', 's', () => {})).rejects.toThrow(/Empty response/);
+  });
+});
+
+describe('images', () => {
+  it('sends images as image_url parts after the text, and plain text without them', async () => {
+    const p = provider('gpt-6-astra');
+    await p.generate('what is this?', 's', { images: [{ data: 'AAAA', mediaType: 'image/png' }] });
+    await p.generate('plain', 's');
+    const withImage = sent[0] as unknown as { messages: Array<{ role: string; content: unknown }> };
+    expect(withImage.messages[1]!.content).toEqual([
+      { type: 'text', text: 'what is this?' },
+      { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },
+    ]);
+    const plain = sent[1] as unknown as { messages: Array<{ role: string; content: unknown }> };
+    expect(plain.messages[1]!.content).toBe('plain');
+  });
+});

package/src/index.ts

@@ -8,6 +8,7 @@ export type {
   LLMResponse,
   GenerateOptions,
   ReasoningEffort,
+  LLMImage,
   ProviderConfig,
   JsonExtractionResult,
   ValidationResult,

package/src/providers/anthropic.ts

@@ -11,9 +11,13 @@ import { LLM_DEFAULTS } from '../types';
 import { LLMProviderError } from '../errors';
 import type { LLMErrorCode } from '../errors';
 
+type AnthropicContentBlock =
+  | { type: 'text'; text: string }
+  | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } };
+
 interface AnthropicMessage {
   role: 'user' | 'assistant';
-  content: string;
+  content: string | AnthropicContentBlock[];
 }
 
 interface AnthropicRequest {
@@ -199,7 +203,18 @@ Please respond with a valid JSON object only. Do not include any text before or
     const requestBody: AnthropicRequest = {
       model,
       max_tokens: maxTokens,
-      messages: [{ role: 'user', content: jsonWrappedPrompt }],
+      messages: [{
+        role: 'user',
+        content: options?.images?.length
+          ? [
+              ...options.images.map((img) => ({
+                type: 'image' as const,
+                source: { type: 'base64' as const, media_type: img.mediaType, data: img.data },
+              })),
+              { type: 'text' as const, text: jsonWrappedPrompt },
+            ]
+          : jsonWrappedPrompt,
+      }],
       system: systemPrompt,
     };
     // Only for models known to take it. The Claude 5 family answers a request

package/src/providers/interface.ts

@@ -35,6 +35,19 @@ export interface ILLMProvider {
     options?: GenerateOptions
   ): Promise<LLMResponse>;
 
+  /**
+   * Optional STREAMING generate: same contract as generate(), and calls
+   * `onDelta` with each chunk of content as the model writes it, so a caller
+   * can show progress during a long generation. Callers should fall back to
+   * generate() when a provider doesn't implement it.
+   */
+  generateStream?(
+    userPrompt: string,
+    systemPrompt: string,
+    onDelta: (text: string) => void,
+    options?: GenerateOptions
+  ): Promise<LLMResponse>;
+
   /**
    * Optional NATIVE (structured) tool-calling. Providers backed by a real tools
    * API (OpenAI functions, Anthropic tool blocks) implement this so the agent

package/src/providers/openai.ts

@@ -18,14 +18,28 @@ import type {
   NativeChatMessage,
   NativeToolResponse,
   ReasoningEffort,
+  LLMImage,
 } from '../types';
 import { LLM_DEFAULTS } from '../types';
 import { LLMProviderError } from '../errors';
 import type { LLMErrorCode } from '../errors';
 
+type OpenAIContentPart =
+  | { type: 'text'; text: string }
+  | { type: 'image_url'; image_url: { url: string } };
+
 interface OpenAIMessage {
   role: 'system' | 'user' | 'assistant';
-  content: string;
+  content: string | OpenAIContentPart[];
+}
+
+/** The user message: plain text, or text plus images for vision models. */
+function userContent(prompt: string, images?: LLMImage[]): string | OpenAIContentPart[] {
+  if (!images?.length) return prompt;
+  return [
+    { type: 'text', text: prompt },
+    ...images.map((img) => ({ type: 'image_url' as const, image_url: { url: `data:${img.mediaType};base64,${img.data}` } })),
+  ];
 }
 
 interface OpenAIRequest {
@@ -268,6 +282,17 @@ export class OpenAIProvider implements ILLMProvider {
     sent: { effort?: ReasoningEffort; maxTokens: number },
     signal: AbortSignal,
   ): Promise<T> {
+    const response = await this.postAdaptivelyRaw(model, build, sent, signal);
+    return (await response.json()) as T;
+  }
+
+  /** postAdaptively, returning the successful Response unread (for streaming). */
+  private async postAdaptivelyRaw(
+    model: string,
+    build: (q: ModelQuirks) => object,
+    sent: { effort?: ReasoningEffort; maxTokens: number },
+    signal: AbortSignal,
+  ): Promise<Response> {
     const q = this.quirksFor(model);
     for (let attempt = 0; ; attempt++) {
       const body = JSON.stringify(build(q));
@@ -280,7 +305,7 @@ export class OpenAIProvider implements ILLMProvider {
         body,
         signal,
       });
-      if (response.ok) return (await response.json()) as T;
+      if (response.ok) return response;
       const errorData = (await response.json().catch(() => ({
         error: { message: 'Unknown error' },
       }))) as OpenAIErrorResponse;
@@ -295,6 +320,95 @@ export class OpenAIProvider implements ILLMProvider {
     }
   }
 
+  async generateStream(
+    userPrompt: string,
+    systemPrompt: string,
+    onDelta: (text: string) => void,
+    options?: GenerateOptions
+  ): Promise<LLMResponse> {
+    if (!this.isAvailable()) {
+      throw new LLMProviderError('OpenAI API key not configured', 'AUTHENTICATION_ERROR', false, undefined, this.name);
+    }
+    const model = options?.model ?? this.config.defaultModel;
+    const temperature = options?.temperature ?? this.config.defaultTemperature;
+    const maxTokens = options?.maxTokens ?? this.config.defaultMaxTokens;
+    const effort = options?.reasoningEffort ?? this.config.defaultReasoningEffort;
+    const timeout = options?.timeout ?? this.config.timeoutMs;
+    const base = {
+      model,
+      messages: [
+        { role: 'system' as const, content: systemPrompt },
+        { role: 'user' as const, content: userContent(userPrompt, options?.images) },
+      ],
+      response_format: { type: 'json_object' as const },
+      stream: true,
+      stream_options: { include_usage: true },
+    };
+
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    try {
+      const response = await this.postAdaptivelyRaw(model, (q) => ({
+        ...base,
+        ...samplingParams(q, temperature, maxTokens, effort),
+      }), { effort, maxTokens }, controller.signal);
+
+      let content = '';
+      let reportedModel = model;
+      let finishReason: string | undefined;
+      let responseId: string | undefined;
+      const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
+      const reader = response.body!.getReader();
+      const decoder = new TextDecoder();
+      let buffer = '';
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        let nl: number;
+        while ((nl = buffer.indexOf('\n')) >= 0) {
+          const line = buffer.slice(0, nl).trim();
+          buffer = buffer.slice(nl + 1);
+          if (!line.startsWith('data:')) continue;
+          const payload = line.slice(5).trim();
+          if (payload === '[DONE]') continue;
+          const chunk = JSON.parse(payload) as {
+            id?: string;
+            model?: string;
+            choices?: Array<{ delta?: { content?: string | null }; finish_reason?: string | null }>;
+            usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
+          };
+          if (chunk.id) responseId = chunk.id;
+          if (chunk.model) reportedModel = chunk.model;
+          const choice = chunk.choices?.[0];
+          if (choice?.finish_reason) finishReason = choice.finish_reason;
+          const delta = choice?.delta?.content;
+          if (delta) {
+            content += delta;
+            onDelta(delta);
+          }
+          if (chunk.usage) {
+            usage.promptTokens = chunk.usage.prompt_tokens;
+            usage.completionTokens = chunk.usage.completion_tokens;
+            usage.totalTokens = chunk.usage.total_tokens;
+          }
+        }
+      }
+      clearTimeout(timeoutId);
+      if (!content) {
+        throw new LLMProviderError('Empty response from OpenAI', 'UNKNOWN_ERROR', true, undefined, this.name);
+      }
+      return { content, usage, model: reportedModel, provider: this.name, responseId, finishReason };
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof LLMProviderError) throw error;
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new LLMProviderError(`Request timed out after ${timeout}ms`, 'TIMEOUT', true, undefined, this.name);
+      }
+      throw new LLMProviderError(error instanceof Error ? error.message : 'Unknown error', 'UNKNOWN_ERROR', true, undefined, this.name);
+    }
+  }
+
   isAvailable(): boolean {
     return Boolean(this.config.apiKey);
   }
@@ -324,7 +438,7 @@ export class OpenAIProvider implements ILLMProvider {
       model,
       messages: [
         { role: 'system' as const, content: systemPrompt },
-        { role: 'user' as const, content: userPrompt },
+        { role: 'user' as const, content: userContent(userPrompt, options?.images) },
       ],
       response_format: { type: 'json_object' as const },
     };

package/src/types.ts

@@ -30,6 +30,12 @@ export interface LLMResponse {
 /**
  * Options passed through to the provider for a single generation call.
  */
+/** An image attached to a prompt: base64 data (no data: prefix) and its media type. */
+export interface LLMImage {
+  data: string;
+  mediaType: 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp';
+}
+
 export interface GenerateOptions {
   /** Model to use (provider-specific) */
   model?: string;
@@ -45,6 +51,11 @@ export interface GenerateOptions {
    * provider adds headroom for the reasoning on top.
    */
   reasoningEffort?: ReasoningEffort;
+  /**
+   * Images sent with the user prompt, for vision-capable models (e.g. to have
+   * a model look at a rendering of what it designed).
+   */
+  images?: LLMImage[];
   /** Arbitrary extra data the caller wants to thread through (e.g. domain preferences) */
   extra?: Record<string, unknown>;
 }