genai-lite 0.10.0 → 0.12.0

package/README.md

@@ -6,6 +6,7 @@ A lightweight, portable Node.js/TypeScript library providing a unified interface
 
 - 🔌 **Unified API** - Single interface for multiple AI providers
 - 🏠 **Local & Cloud Models** - Run models locally with llama.cpp or use cloud APIs
+- ⚡ **Text Streaming** - Async iterable token deltas for OpenAI, Anthropic, Gemini, Mistral, OpenRouter, and llama.cpp
 - 🖼️ **Image Generation** - First-class support for AI image generation (OpenAI, local diffusion)
 - 🔐 **Flexible API Key Management** - Bring your own key storage solution
 - 📦 **Zero Electron Dependencies** - Works in any Node.js environment
@@ -13,7 +14,7 @@ A lightweight, portable Node.js/TypeScript library providing a unified interface
 - ⚡ **Lightweight** - Minimal dependencies, focused functionality
 - 🛡️ **Provider Normalization** - Consistent responses across different AI APIs
 - 🧠 **Local Reasoning Toggle** - Turn thinking on/off for detected GGUF models (Qwen 3.5-class, Gemma 4, and more) with vendor-tuned sampling defaults
-- 🔁 **Built-in Reliability** - Automatic retries with backoff/`Retry-After`, per-request timeouts, and abort signals; image generation supports cancellation too (including server-side cancel for local diffusion)
+- 🔁 **Built-in Reliability** - Automatic retries with backoff/`Retry-After`, per-request timeouts, and abort signals — for both LLM and image generation (local diffusion additionally gets server-side cancel and is never blind-retried)
 - 🎨 **Configurable Model Presets** - Built-in presets with full customization options
 - 🎭 **Template Engine** - Sophisticated templating with conditionals and variable substitution
 - 📊 **Configurable Logging** - Debug mode, custom loggers (pino, winston), and silent mode for tests
@@ -77,6 +78,33 @@ if (response.object === 'chat.completion') {
 }
 ```
 
+### Streaming Text
+
+```typescript
+import { LLMService } from 'genai-lite';
+
+const llmService = new LLMService(async () => 'not-needed');
+
+for await (const event of llmService.streamMessage({
+  providerId: 'llamacpp',
+  modelId: 'llamacpp',
+  messages: [{ role: 'user', content: 'Say hello in five words.' }],
+  settings: { maxTokens: 32 }
+})) {
+  if (event.type === 'content_delta') {
+    process.stdout.write(event.delta);
+  } else if (event.type === 'reasoning_delta') {
+    process.stderr.write(event.delta);
+  } else if (event.type === 'complete') {
+    console.log('\nTokens:', event.response.usage?.total_tokens);
+  } else if (event.type === 'error') {
+    console.error(event.error.error.message);
+  }
+}
+```
+
+Streaming is implemented for all text providers: `openai`, `anthropic`, `gemini`, `mistral`, `openrouter`, and `llamacpp`. The final `complete.response` event contains the same normalized response shape returned by `sendMessage()`.
+
 ### Image Generation
 
 ```typescript
@@ -134,7 +162,7 @@ Comprehensive documentation is available in the **[`genai-lite-docs`](./genai-li
 - **Google Gemini** - Gemini 3 (Pro, Flash preview), Gemini 2.5, Gemma 3 & 4 (free)
 - **Mistral** - Codestral, Devstral
 - **OpenRouter** - Unified gateway to 100+ models (unknown models assumed reasoning-capable)
-- **llama.cpp** - Run any GGUF model locally (no API keys required); local reasoning toggle for detected models
+- **llama.cpp** - Run any GGUF model locally (no API keys required); streaming and local reasoning toggle for detected models
 
 ### Image Providers
 - **OpenAI Images** - gpt-image-1, dall-e-3, dall-e-2

package/dist/adapters/image/GenaiElectronImageAdapter.d.ts

@@ -35,6 +35,7 @@ export declare class GenaiElectronImageAdapter implements ImageProviderAdapter {
         settings: ResolvedImageGenerationSettings;
         apiKey: string | null;
         signal?: AbortSignal;
+        timeoutMs?: number;
     }): Promise<ImageGenerationResponse>;
     /**
      * Builds the request payload for genai-electron

package/dist/adapters/image/GenaiElectronImageAdapter.js

@@ -43,7 +43,10 @@ class GenaiElectronImageAdapter {
      * Generates images using genai-electron's async API with progress polling
      */
     async generate(config) {
-        const { request, resolvedPrompt, settings, signal } = config;
+        const { request, resolvedPrompt, settings, signal, timeoutMs } = config;
+        // Per-request override of the construction-time timeout; applies to both
+        // the POST and the poll-loop budget
+        const effectiveTimeout = timeoutMs ?? this.timeout;
         let generationId;
         try {
             if (signal?.aborted) {
@@ -58,10 +61,10 @@ class GenaiElectronImageAdapter {
                 steps: payload.steps,
             });
             // Start generation (returns immediately with ID)
-            generationId = await this.startGeneration(payload, signal);
+            generationId = await this.startGeneration(payload, signal, effectiveTimeout);
             this.logger.info(`GenaiElectron Image API: Generation started with ID: ${generationId}`);
             // Poll for completion
-            const result = await this.pollForCompletion(generationId, settings.diffusion?.onProgress, signal);
+            const result = await this.pollForCompletion(generationId, settings.diffusion?.onProgress, signal, effectiveTimeout);
             this.logger.info(`GenaiElectron Image API: Generation complete (${result.timeTaken}ms)`);
             // Convert to ImageGenerationResponse
             return this.convertToResponse(result, request);
@@ -104,7 +107,7 @@ class GenaiElectronImageAdapter {
      * so both surface as AbortError — classification comes from adapter-side
      * state (timer-fired flag vs signal.aborted), with the caller's abort winning.
      */
-    async startGeneration(payload, signal) {
+    async startGeneration(payload, signal, effectiveTimeout) {
         const url = `${this.baseURL}/v1/images/generations`;
         const controller = new AbortController();
         let timedOut = false;
@@ -120,7 +123,7 @@ class GenaiElectronImageAdapter {
         const timeoutId = setTimeout(() => {
             timedOut = true;
             controller.abort();
-        }, this.timeout);
+        }, effectiveTimeout);
         try {
             const response = await fetch(url, {
                 method: 'POST',
@@ -141,7 +144,7 @@ class GenaiElectronImageAdapter {
                     throw this.createAbortError('Image generation request was aborted before it started');
                 }
                 if (timedOut) {
-                    throw this.createTimeoutError(`Request timeout after ${this.timeout}ms (connecting to ${this.baseURL})`);
+                    throw this.createTimeoutError(`Request timeout after ${effectiveTimeout}ms (connecting to ${this.baseURL})`);
                 }
             }
             throw error;
@@ -154,7 +157,7 @@ class GenaiElectronImageAdapter {
     /**
      * Polls for generation completion with progress updates
      */
-    async pollForCompletion(generationId, onProgress, signal) {
+    async pollForCompletion(generationId, onProgress, signal, effectiveTimeout) {
         const url = `${this.baseURL}/v1/images/generations/${generationId}`;
         const startTime = Date.now();
         while (true) {
@@ -163,8 +166,8 @@ class GenaiElectronImageAdapter {
                 throw this.createAbortError(`Image generation request was aborted (ID: ${generationId})`);
             }
             // Check overall timeout
-            if (Date.now() - startTime > this.timeout) {
-                throw this.createTimeoutError(`Generation timeout after ${this.timeout}ms (ID: ${generationId})`);
+            if (Date.now() - startTime > effectiveTimeout) {
+                throw this.createTimeoutError(`Generation timeout after ${effectiveTimeout}ms (ID: ${generationId})`);
             }
             // Fetch status
             let response;
@@ -343,6 +346,17 @@ class GenaiElectronImageAdapter {
             errorCode = types_1.ADAPTER_ERROR_CODES.RATE_LIMIT_EXCEEDED;
             errorType = 'rate_limit_error';
         }
+        else if (error.code === 'NOT_FOUND') {
+            // Server JSON code from a poll GET/DELETE whose generation is gone —
+            // typically expired from the registry (default TTL 300s). Without this
+            // branch the 404 status maps to MODEL_NOT_FOUND ("model not found"),
+            // which is misleading for an expired generation.
+            errorMessage =
+                'Generation not found on the server — it likely expired from the ' +
+                    `registry before polling completed. ${error.message}`;
+            errorCode = types_1.ADAPTER_ERROR_CODES.PROVIDER_ERROR;
+            errorType = 'server_error';
+        }
         else if (error.code === 'SERVER_NOT_RUNNING') {
             errorMessage = `Image generation server is not running (connecting to ${this.baseURL})`;
             errorCode = types_1.ADAPTER_ERROR_CODES.NETWORK_ERROR;
@@ -371,6 +385,9 @@ class GenaiElectronImageAdapter {
         enhancedError.code = errorCode;
         enhancedError.type = errorType;
         enhancedError.status = mapped.status;
+        if (mapped.retryAfterMs !== undefined) {
+            enhancedError.retryAfterMs = mapped.retryAfterMs;
+        }
         enhancedError.providerId = this.id;
         enhancedError.modelId = request.modelId;
         return enhancedError;

package/dist/adapters/image/MockImageAdapter.d.ts

@@ -20,5 +20,6 @@ export declare class MockImageAdapter implements ImageProviderAdapter {
         settings: ResolvedImageGenerationSettings;
         apiKey: string | null;
         signal?: AbortSignal;
+        timeoutMs?: number;
     }): Promise<ImageGenerationResponse>;
 }

package/dist/adapters/image/OpenAIImageAdapter.d.ts

@@ -36,6 +36,7 @@ export declare class OpenAIImageAdapter implements ImageProviderAdapter {
         settings: ResolvedImageGenerationSettings;
         apiKey: string | null;
         signal?: AbortSignal;
+        timeoutMs?: number;
     }): Promise<ImageGenerationResponse>;
     /**
      * Validates prompt length for the given model

package/dist/adapters/image/OpenAIImageAdapter.js

@@ -59,7 +59,7 @@ class OpenAIImageAdapter {
      * Generates images using OpenAI's API
      */
     async generate(config) {
-        const { request, resolvedPrompt, settings, apiKey, signal } = config;
+        const { request, resolvedPrompt, settings, apiKey, signal, timeoutMs } = config;
         if (!apiKey) {
             throw new Error('OpenAI API key is required but was not provided');
         }
@@ -102,14 +102,18 @@ class OpenAIImageAdapter {
                 n: params.n,
                 isGptImageModel,
             });
-            // Make API call (per-request abort signal when provided)
-            const response = await client.images.generate(params, signal ? { signal } : undefined);
+            // Make API call (per-request abort signal and timeout override when provided)
+            const requestOptions = {
+                ...(signal && { signal }),
+                ...(timeoutMs !== undefined && { timeout: timeoutMs }),
+            };
+            const response = await client.images.generate(params, Object.keys(requestOptions).length > 0 ? requestOptions : undefined);
             if (!response.data || response.data.length === 0) {
                 throw new Error('OpenAI API returned no images in response');
             }
             this.logger.info(`OpenAI Image API call successful, generated ${response.data.length} images`);
             // Process response
-            return await this.processResponse(response, request, isGptImageModel, signal);
+            return await this.processResponse(response, request, isGptImageModel, signal, timeoutMs);
         }
         catch (error) {
             this.logger.error('OpenAI Image API error:', error);
@@ -199,7 +203,7 @@ class OpenAIImageAdapter {
     /**
      * Processes the OpenAI API response and converts to ImageGenerationResponse
      */
-    async processResponse(response, request, isGptImageModel, signal) {
+    async processResponse(response, request, isGptImageModel, signal, timeoutMs) {
         const images = [];
         for (let i = 0; i < response.data.length; i++) {
             const item = response.data[i];
@@ -222,9 +226,17 @@ class OpenAIImageAdapter {
             else {
                 // dall-e-2/dall-e-3: can be url or b64_json
                 if (item.url) {
-                    // Fetch image from URL
+                    // Fetch image from URL. The SDK timeout only covers images.generate,
+                    // so bound this fetch with the same per-request budget when one was
+                    // given (AbortSignal.timeout carries a TimeoutError reason, mapping
+                    // to REQUEST_TIMEOUT; a caller abort still maps to REQUEST_ABORTED)
                     imageUrl = item.url;
-                    const imageResponse = await fetch(item.url, signal ? { signal } : undefined);
+                    const fetchSignal = timeoutMs !== undefined
+                        ? signal
+                            ? AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)])
+                            : AbortSignal.timeout(timeoutMs)
+                        : signal;
+                    const imageResponse = await fetch(item.url, fetchSignal ? { signal: fetchSignal } : undefined);
                     if (!imageResponse.ok) {
                         throw new Error(`Failed to fetch image from URL: ${imageResponse.statusText}`);
                     }
@@ -297,6 +309,9 @@ class OpenAIImageAdapter {
         enhancedError.code = mapped.errorCode;
         enhancedError.type = mapped.errorType;
         enhancedError.status = mapped.status;
+        if (mapped.retryAfterMs !== undefined) {
+            enhancedError.retryAfterMs = mapped.retryAfterMs;
+        }
         enhancedError.providerId = this.id;
         enhancedError.modelId = request.modelId;
         return enhancedError;

package/dist/image/ImageService.d.ts

@@ -17,6 +17,7 @@ export declare class ImageService {
     private requestValidator;
     private settingsResolver;
     private modelResolver;
+    private retryOptions;
     constructor(getApiKey: ApiKeyProvider, options?: ImageServiceOptions);
     /**
      * Generates images based on the request
@@ -26,6 +27,16 @@ export declare class ImageService {
      * @returns Promise resolving to response or error
      */
     generateImage(request: ImageGenerationRequest | ImageGenerationRequestWithPreset, options?: GenerateImageOptions): Promise<ImageGenerationResponse | ImageFailureResponse>;
+    /**
+     * Builds a failure envelope from a thrown error.
+     *
+     * Adapters throw errors stamped with an ADAPTER_ERROR_CODES code plus
+     * type/status/retryAfterMs — that classification is propagated. Anything else
+     * (e.g. a failing ApiKeyProvider whose error carries a foreign `.code` like
+     * ENOENT) keeps the generic fallback so unrelated codes don't leak into the
+     * API surface.
+     */
+    private buildFailureEnvelope;
     /**
      * Gets list of supported image providers
      *

package/dist/image/ImageService.js

@@ -12,6 +12,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ImageService = void 0;
 const defaultLogger_1 = require("../logging/defaultLogger");
 const types_1 = require("../llm/clients/types");
+const withRetry_1 = require("../shared/services/withRetry");
 const config_1 = require("./config");
 const image_presets_json_1 = __importDefault(require("../config/image-presets.json"));
 const PresetManager_1 = require("../shared/services/PresetManager");
@@ -33,6 +34,7 @@ const ADAPTER_ERROR_CODE_VALUES = new Set(Object.values(types_1.ADAPTER_ERROR_CO
 class ImageService {
     constructor(getApiKey, options = {}) {
         this.getApiKey = getApiKey;
+        this.retryOptions = options.retry;
         // Initialize logger - custom logger takes precedence over logLevel
         this.logger = options.logger ?? (0, defaultLogger_1.createDefaultLogger)(options.logLevel);
         // Initialize helper services
@@ -136,44 +138,58 @@ class ImageService {
                         },
                     };
                 }
-                // Generate images
+                // Generate images. Only providers marked retryable in config are ever
+                // retried — genai-electron's POST-then-poll is not idempotent (a blind
+                // retry after the POST would start a second GPU generation), and custom
+                // adapters without a config entry get the safe default (no retries).
                 this.logger.info(`ImageService: Calling adapter for provider: ${providerId}`);
-                const response = await adapter.generate({
-                    request: fullRequest,
-                    resolvedPrompt,
-                    settings: resolvedSettings,
-                    apiKey,
+                const providerRetryable = (0, config_1.getImageProviderById)(providerId)?.retryable === true;
+                const retryOnTimeout = this.retryOptions?.retryOnTimeout ?? true;
+                const retryableCodes = new Set([
+                    types_1.ADAPTER_ERROR_CODES.RATE_LIMIT_EXCEEDED,
+                    types_1.ADAPTER_ERROR_CODES.NETWORK_ERROR,
+                    ...(retryOnTimeout ? [types_1.ADAPTER_ERROR_CODES.REQUEST_TIMEOUT] : []),
+                ]);
+                return await (0, withRetry_1.withRetry)(async () => {
+                    try {
+                        const response = await adapter.generate({
+                            request: fullRequest,
+                            resolvedPrompt,
+                            settings: resolvedSettings,
+                            apiKey,
+                            signal: options?.signal,
+                            timeoutMs: options?.timeoutMs,
+                        });
+                        this.logger.info('ImageService: Image generation completed successfully');
+                        return response;
+                    }
+                    catch (error) {
+                        this.logger.error('ImageService: Error during image generation:', error);
+                        return this.buildFailureEnvelope(error, providerId, modelId);
+                    }
+                }, (result) => {
+                    if (result.object !== 'error' || !providerRetryable) {
+                        return { retry: false };
+                    }
+                    const code = String(result.error.code);
+                    const status = result.error.status;
+                    const retry = retryableCodes.has(code) ||
+                        (code === types_1.ADAPTER_ERROR_CODES.PROVIDER_ERROR &&
+                            typeof status === 'number' &&
+                            (status === 408 || status === 409 || status >= 500));
+                    return { retry, retryAfterMs: result.error.retryAfterMs };
+                }, {
+                    ...this.retryOptions,
+                    ...(options?.maxRetries !== undefined && { maxRetries: options.maxRetries }),
                     signal: options?.signal,
+                    logger: this.logger,
+                    label: `${providerId}/${modelId}`,
                 });
-                this.logger.info('ImageService: Image generation completed successfully');
-                return response;
             }
             catch (error) {
+                // Errors thrown outside the adapter call (e.g. a failing ApiKeyProvider)
                 this.logger.error('ImageService: Error during image generation:', error);
-                // Adapters throw errors stamped with an ADAPTER_ERROR_CODES code plus
-                // type/status — propagate that classification. Anything else caught here
-                // (e.g. a failing ApiKeyProvider whose error carries a foreign `.code`
-                // like ENOENT) keeps the generic fallback so unrelated codes don't leak
-                // into the API surface.
-                const thrown = error;
-                const isAdapterError = typeof thrown?.code === 'string' && ADAPTER_ERROR_CODE_VALUES.has(thrown.code);
-                return {
-                    object: 'error',
-                    providerId: providerId,
-                    modelId: modelId,
-                    error: {
-                        message: error instanceof Error
-                            ? error.message
-                            : 'An unknown error occurred during image generation',
-                        code: isAdapterError ? thrown.code : 'PROVIDER_ERROR',
-                        type: isAdapterError && typeof thrown.type === 'string'
-                            ? thrown.type
-                            : 'server_error',
-                        ...(isAdapterError &&
-                            typeof thrown.status === 'number' && { status: thrown.status }),
-                        providerError: error,
-                    },
-                };
+                return this.buildFailureEnvelope(error, providerId, modelId);
             }
         }
         catch (error) {
@@ -192,6 +208,38 @@ class ImageService {
             };
         }
     }
+    /**
+     * Builds a failure envelope from a thrown error.
+     *
+     * Adapters throw errors stamped with an ADAPTER_ERROR_CODES code plus
+     * type/status/retryAfterMs — that classification is propagated. Anything else
+     * (e.g. a failing ApiKeyProvider whose error carries a foreign `.code` like
+     * ENOENT) keeps the generic fallback so unrelated codes don't leak into the
+     * API surface.
+     */
+    buildFailureEnvelope(error, providerId, modelId) {
+        const thrown = error;
+        const isAdapterError = typeof thrown?.code === 'string' && ADAPTER_ERROR_CODE_VALUES.has(thrown.code);
+        return {
+            object: 'error',
+            providerId,
+            modelId,
+            error: {
+                message: error instanceof Error
+                    ? error.message
+                    : 'An unknown error occurred during image generation',
+                code: isAdapterError ? thrown.code : 'PROVIDER_ERROR',
+                type: isAdapterError && typeof thrown.type === 'string'
+                    ? thrown.type
+                    : 'server_error',
+                ...(isAdapterError &&
+                    typeof thrown.status === 'number' && { status: thrown.status }),
+                ...(isAdapterError &&
+                    typeof thrown.retryAfterMs === 'number' && { retryAfterMs: thrown.retryAfterMs }),
+                providerError: error,
+            },
+        };
+    }
     /**
      * Gets list of supported image providers
      *

package/dist/image/config.js

@@ -22,6 +22,8 @@ exports.SUPPORTED_IMAGE_PROVIDERS = [
         id: 'openai-images',
         displayName: 'OpenAI Images',
         description: 'DALL-E and GPT-Image models from OpenAI',
+        // Single atomic POST per generation — safe to auto-retry transient failures
+        retryable: true,
         capabilities: {
             supportsMultipleImages: true,
             supportsB64Json: true,
@@ -128,6 +130,9 @@ exports.SUPPORTED_IMAGE_PROVIDERS = [
         id: 'genai-electron-images',
         displayName: 'Local Diffusion (genai-electron)',
         description: 'Local stable-diffusion models via genai-electron wrapper',
+        // POST-then-poll: a blind retry of a mid-poll failure would start a second
+        // GPU generation, so the service-level retry layer must never touch this
+        retryable: false,
         capabilities: {
             supportsMultipleImages: true,
             supportsB64Json: true,

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 export type { ApiKeyProvider, PresetMode } from "./types";
 export { LLMService } from "./llm/LLMService";
-export type { LLMServiceOptions, SendMessageOptions, CreateMessagesResult } from "./llm/LLMService";
+export type { LLMServiceOptions, SendMessageOptions, StreamMessageOptions, CreateMessagesResult } from "./llm/LLMService";
 export { withRetry, DEFAULT_RETRY_POLICY } from "./shared/services/withRetry";
 export type { RetryPolicy, RetryVerdict, WithRetryOptions } from "./shared/services/withRetry";
 export type { ModelPreset } from "./types/presets";

package/dist/llm/LLMService.d.ts

@@ -1,6 +1,6 @@
 import type { ApiKeyProvider, PresetMode } from '../types';
 import type { Logger, LogLevel } from '../logging/types';
-import type { LLMChatRequest, LLMChatRequestWithPreset, LLMResponse, LLMFailureResponse, ProviderInfo, ModelInfo, ApiProviderId, LLMSettings, ModelContext, LLMMessage } from "./types";
+import type { LLMChatRequest, LLMChatRequestWithPreset, LLMResponse, LLMFailureResponse, ProviderInfo, ModelInfo, ApiProviderId, LLMSettings, ModelContext, LLMMessage, LLMStreamEvent } from "./types";
 import type { ModelPreset } from "../types/presets";
 import { type RetryPolicy } from "../shared/services/withRetry";
 export type { PresetMode };
@@ -43,6 +43,18 @@ export interface SendMessageOptions {
     /** Per-request retry cap (overrides the service-level retry.maxRetries) */
     maxRetries?: number;
 }
+/**
+ * Per-call options for LLMService.streamMessage
+ */
+export interface StreamMessageOptions {
+    /**
+     * Abort signal to cancel the request (client-side - the provider may still
+     * process and bill an already-dispatched request).
+     */
+    signal?: AbortSignal;
+    /** Per-request timeout in ms (overrides the service-level timeoutMs) */
+    timeoutMs?: number;
+}
 /**
  * Result from createMessages method
  */
@@ -96,6 +108,16 @@ export declare class LLMService {
      * @returns Promise resolving to either success or failure response
      */
     sendMessage(request: LLMChatRequest | LLMChatRequestWithPreset, callOptions?: SendMessageOptions): Promise<LLMResponse | LLMFailureResponse>;
+    /**
+     * Streams a chat message from an LLM provider.
+     *
+     * The final `complete` event contains the same normalized response shape as
+     * sendMessage(). Validation/API key failures and unsupported streaming adapters
+     * are yielded as a single `error` event.
+     */
+    streamMessage(request: LLMChatRequest | LLMChatRequestWithPreset, callOptions?: StreamMessageOptions): AsyncGenerator<LLMStreamEvent>;
+    private prepareRequest;
+    private postProcessResponse;
     /**
      * Gets all configured model presets
      *