@llumiverse/core 1.3.0 → 1.4.0

package/src/Driver.ts

@@ -5,74 +5,49 @@
  */
 
 import {
-    AIModel,
-    Completion,
-    CompletionChunkObject,
-    CompletionStream,
-    DataSource,
-    DriverOptions,
-    EmbeddingsOptions,
-    EmbeddingsResult,
-    ExecutionOptions,
-    ExecutionResponse,
-    LlumiverseErrorContext,
-    Logger,
-    ModelSearchPayload,
-    PromptOptions,
-    PromptSegment,
-    Providers,
-    TrainingJob,
-    TrainingOptions,
-    TrainingPromptOptions,
-    LlumiverseError
-} from "@llumiverse/common";
-import { DefaultCompletionStream, FallbackCompletionStream } from "./CompletionStream.js";
-import { formatTextPrompt } from "./formatters/index.js";
-import { validateResult } from "./validation.js";
-
-// Helper to create logger methods that support both message-only and object-first signatures
-function createConsoleLoggerMethod(consoleMethod: (...args: unknown[]) => void): Logger['info'] {
-    return ((objOrMsg: any, msgOrNever?: any, ...args: (string | number | boolean)[]) => {
-        if (typeof objOrMsg === 'string') {
-            // Message-only: logger.info("message", ...args)
-            consoleMethod(objOrMsg, msgOrNever, ...args);
-        } else if (msgOrNever !== undefined) {
-            // Object-first: logger.info({ obj }, "message", ...args)
-            consoleMethod(msgOrNever, objOrMsg, ...args);
-        } else {
-            // Object-only: logger.info({ obj })
-            consoleMethod(objOrMsg, ...args);
-        }
-    }) as Logger['info'];
-}
-
-const ConsoleLogger: Logger = {
-    debug: createConsoleLoggerMethod(console.debug.bind(console)),
-    info: createConsoleLoggerMethod(console.info.bind(console)),
-    warn: createConsoleLoggerMethod(console.warn.bind(console)),
-    error: createConsoleLoggerMethod(console.error.bind(console)),
-}
-
-const noop = () => void 0;
-const NoopLogger: Logger = {
-    debug: noop as Logger['debug'],
-    info: noop as Logger['info'],
-    warn: noop as Logger['warn'],
-    error: noop as Logger['error'],
-}
-
-export function createLogger(logger: Logger | "console" | undefined) {
-    if (logger === "console") {
-        return ConsoleLogger;
-    } else if (logger) {
-        return logger;
-    } else {
-        return NoopLogger;
+    type AIModel,
+    type Completion,
+    type CompletionChunkObject,
+    type CompletionStream,
+    type DataSource,
+    type DriverOptions,
+    type EmbeddingsOptions,
+    type EmbeddingsResult,
+    type ExecutionOptions,
+    type ExecutionResponse,
+    LlumiverseError,
+    type LlumiverseErrorContext,
+    type Logger,
+    type ModelSearchPayload,
+    type PromptOptions,
+    type PromptSegment,
+    type Providers,
+    type TrainingJob,
+    type TrainingOptions,
+    type TrainingPromptOptions,
+} from '@llumiverse/common';
+import type { Agent } from 'undici';
+import { DefaultCompletionStream, FallbackCompletionStream } from './CompletionStream.js';
+import { formatTextPrompt } from './formatters/index.js';
+import {
+    createAgentBackedFetch,
+    createDriverHttpAgent,
+    createDriverHttpAgentScope,
+    type DriverHttpAgentScope,
+} from './http-agent.js';
+import { createLogger } from './logger.js';
+import { validateResult } from './validation.js';
+
+export { createLogger } from './logger.js';
+
+function getObjectProperty(value: unknown, key: string): unknown {
+    if (value && typeof value === 'object' && key in value) {
+        return (value as Record<string, unknown>)[key];
     }
+    return undefined;
 }
 
 export interface Driver<PromptT = unknown> {
-
     /**
      *
      * @param segments
@@ -104,7 +79,13 @@ export interface Driver<PromptT = unknown> {
     //check that it is possible to connect to the environment
     validateConnection(): Promise<boolean>;
 
-    //generate embeddings for a given text or image
+    /**
+     * Generate embeddings for one or more inputs.
+     * Inputs may be text, image, video, or audio depending on the model and
+     * provider. Returns one result item per input, each with one or more
+     * output vectors (single-vector for text/image, multi-vector for
+     * segmented video/audio or joint-multimodal models).
+     */
     generateEmbeddings(options: EmbeddingsOptions): Promise<EmbeddingsResult>;
 
     /**
@@ -117,56 +98,102 @@ export interface Driver<PromptT = unknown> {
 /**
  * To be implemented by each driver
  */
-export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOptions, PromptT = unknown> implements Driver<PromptT> {
+export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOptions, PromptT = unknown>
+    implements Driver<PromptT>
+{
     options: OptionsT;
     logger: Logger;
 
     abstract provider: Providers | string; // the provider name
 
+    private _httpAgent?: Agent;
+    private _driverFetch?: typeof fetch;
+
     constructor(opts: OptionsT) {
         this.options = opts;
         this.logger = createLogger(opts.logger);
     }
 
+    /**
+     * Lazily-created undici `Agent` driven by `options.httpTimeout`.
+     * Pools sockets for the lifetime of the driver. Subclasses can
+     * either pass this directly to an SDK that accepts a `dispatcher`
+     * option (rare), or — much more commonly — use {@link getDriverFetch}
+     * to get a fetch implementation backed by it.
+     *
+     * Released via {@link destroy}.
+     */
+    protected getHttpAgent(): Agent {
+        if (!this._httpAgent) {
+            this._httpAgent = createDriverHttpAgent(this.options.httpTimeout);
+        }
+        return this._httpAgent;
+    }
+
+    /**
+     * Fetch-compatible function backed by the driver's HTTP agent.
+     * Pass to any SDK that accepts a custom `fetch` option (OpenAI,
+     * Anthropic, `@google/genai`, Bedrock via Smithy, …) or use as a
+     * drop-in replacement for the global `fetch` in drivers that make
+     * raw HTTP calls.
+     */
+    protected getDriverFetch(): typeof fetch {
+        if (!this._driverFetch) {
+            this._driverFetch = createAgentBackedFetch(this.getHttpAgent());
+        }
+        return this._driverFetch;
+    }
+
+    public createExecutionHttpAgentScope(options: Pick<ExecutionOptions, 'httpTimeout'>): DriverHttpAgentScope {
+        return createDriverHttpAgentScope(this.options.httpTimeout, options.httpTimeout);
+    }
+
     async createTrainingPrompt(options: TrainingPromptOptions): Promise<string> {
-        const prompt = await this.createPrompt(options.segments, { result_schema: options.schema, model: options.model })
+        const prompt = await this.createPrompt(options.segments, {
+            result_schema: options.schema,
+            model: options.model,
+        });
         return JSON.stringify({
             prompt,
-            completion: typeof options.completion === 'string' ? options.completion : JSON.stringify(options.completion)
+            completion:
+                typeof options.completion === 'string' ? options.completion : JSON.stringify(options.completion),
         });
     }
 
     startTraining(_dataset: DataSource, _options: TrainingOptions): Promise<TrainingJob> {
-        throw new Error("Method not implemented.");
+        throw new Error('Method not implemented.');
     }
 
     cancelTraining(_jobId: string): Promise<TrainingJob> {
-        throw new Error("Method not implemented.");
+        throw new Error('Method not implemented.');
     }
 
     getTrainingJob(_jobId: string): Promise<TrainingJob> {
-        throw new Error("Method not implemented.");
+        throw new Error('Method not implemented.');
     }
 
     validateResult(result: Completion, options: ExecutionOptions) {
         if (!result.tool_use && !result.error && options.result_schema) {
             try {
                 result.result = validateResult(result.result, options.result_schema);
-            } catch (error: any) {
-                const errorMessage = `[${this.provider}] [${options.model}] ${error.code ? '[' + error.code + '] ' : ''}Result validation error: ${error.message}`;
+            } catch (error: unknown) {
+                const validationError = error instanceof Error ? error : new Error(String(error));
+                const rawCode = getObjectProperty(error, 'code');
+                const code = rawCode === 'json_error' || rawCode === 'validation_error' ? rawCode : undefined;
+                const errorMessage = `[${this.provider}] [${options.model}] ${code ? `[${code}] ` : ''}Result validation error: ${validationError.message}`;
                 this.logger.error({ err: error, data: result.result }, errorMessage);
                 result.error = {
-                    code: error.code || error.name,
-                    message: error.message,
+                    code: code || 'validation_error',
+                    message: validationError.message,
                     data: result.result,
-                }
+                };
             }
         }
     }
 
     async execute(segments: PromptSegment[], options: ExecutionOptions): Promise<ExecutionResponse<PromptT>> {
         const prompt = await this.createPrompt(segments, options);
-        return this._execute(prompt, options).catch((error: any) => {
+        return this._execute(prompt, options).catch((error: unknown) => {
             // Don't wrap if already a LlumiverseError
             if (LlumiverseError.isLlumiverseError(error)) {
                 throw error;
@@ -180,45 +207,63 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     }
 
     async _execute(prompt: PromptT, options: ExecutionOptions): Promise<ExecutionResponse<PromptT>> {
+        const httpScope = this.createExecutionHttpAgentScope(options);
         try {
-            const start = Date.now();
-            let result;
-
-            if (this.isImageModel(options.model)) {
-                this.logger.debug(
-                    `[${this.provider}] Executing prompt on ${options.model}, image pathway.`);
-                result = await this.requestImageGeneration(prompt, options);
-            } else {
-                this.logger.debug(
-                    `[${this.provider}] Executing prompt on ${options.model}, text pathway.`);
-                result = await this.requestTextCompletion(prompt, options);
-                this.validateResult(result, options);
-            }
-
-            const execution_time = Date.now() - start;
-            return { ...result, prompt, execution_time };
-        } catch (error) {
-            // Don't wrap if already a LlumiverseError
-            if (LlumiverseError.isLlumiverseError(error)) {
-                throw error;
-            }
-            // Log the original error for debugging
-            this.logger.error({ err: error, data: { provider: this.provider, model: options.model, operation: 'execute', prompt } }, `Error during execution in provider ${this.provider}:`);
-            throw this.formatLlumiverseError(error, {
-                provider: this.provider,
-                model: options.model,
-                operation: 'execute',
+            return await httpScope.run(async () => {
+                try {
+                    const start = Date.now();
+                    let result: Completion;
+
+                    if (this.isImageModel(options.model)) {
+                        this.logger.debug(`[${this.provider}] Executing prompt on ${options.model}, image pathway.`);
+                        result = await this.requestImageGeneration(prompt, options);
+                    } else {
+                        this.logger.debug(`[${this.provider}] Executing prompt on ${options.model}, text pathway.`);
+                        result = await this.requestTextCompletion(prompt, options);
+                        this.validateResult(result, options);
+                    }
+
+                    const execution_time = Date.now() - start;
+                    return { ...result, prompt, execution_time };
+                } catch (error) {
+                    // Don't wrap if already a LlumiverseError
+                    if (LlumiverseError.isLlumiverseError(error)) {
+                        throw error;
+                    }
+                    // Log the original error for debugging
+                    this.logger.error(
+                        {
+                            err: error,
+                            data: { provider: this.provider, model: options.model, operation: 'execute', prompt },
+                        },
+                        `Error during execution in provider ${this.provider}:`,
+                    );
+                    throw this.formatLlumiverseError(error, {
+                        provider: this.provider,
+                        model: options.model,
+                        operation: 'execute',
+                    });
+                }
             });
+        } finally {
+            await httpScope.close();
         }
     }
 
+    public formatDebugPrompt(prompt: PromptT): PromptT {
+        return prompt;
+    }
+
     protected isImageModel(_model: string): boolean {
         return false;
     }
 
     // by default no stream is supported. we block and we return all at once
     async stream(segments: PromptSegment[], options: ExecutionOptions): Promise<CompletionStream<PromptT>> {
-        this.logger.debug(options, `Executing prompt with provider ${this.provider} with options: ${JSON.stringify(options)}`);
+        this.logger.debug(
+            options,
+            `Executing prompt with provider ${this.provider} with options: ${JSON.stringify(options)}`,
+        );
         const prompt = await this.createPrompt(segments, options);
         const canStream = await this.canStream(options);
         if (canStream) {
@@ -241,7 +286,9 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     }
 
     public async createPrompt(segments: PromptSegment[], opts: PromptOptions): Promise<PromptT> {
-        return await (opts.format ? opts.format(segments, opts.result_schema) : this.formatPrompt(segments, opts));
+        return await (opts.format
+            ? (opts.format(segments, opts.result_schema) as PromptT)
+            : this.formatPrompt(segments, opts));
     }
 
     /**
@@ -280,7 +327,7 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
         _prompt: PromptT,
         _result: unknown[],
         _toolUse: unknown[] | undefined,
-        _options: ExecutionOptions
+        _options: ExecutionOptions,
     ): unknown | undefined {
         // Default implementation returns undefined - drivers can override
         return undefined;
@@ -289,90 +336,90 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     /**
      * Format an error into LlumiverseError. Override in driver implementations
      * to provide provider-specific error parsing.
-     * 
+     *
      * The default implementation uses common patterns:
      * - Status 429, 408: retryable (rate limit, timeout)
      * - Status 529: retryable (overloaded)
      * - Status 5xx: retryable (server errors)
-     * - Status 4xx (except above): not retryable (client errors)
-     * - Error messages containing "rate limit", "timeout", etc.: retryable
-     * 
+     * - High-confidence transient messages containing "rate limit", "timeout", etc.: retryable
+     * - Status 4xx (except above and transient provider quirks): not retryable (client errors)
+     *
      * @param error - The error to format
      * @param context - Context about where the error occurred
      * @returns A standardized LlumiverseError
      */
-    public formatLlumiverseError(
-        error: unknown,
-        context: LlumiverseErrorContext
-    ): LlumiverseError {
+    public formatLlumiverseError(error: unknown, context: LlumiverseErrorContext): LlumiverseError {
         // Extract status code from common locations (only if numeric)
         let code: number | undefined;
-        const rawCode = (error as any)?.status
-            || (error as any)?.statusCode
-            || (error as any)?.code;
+        const rawCode =
+            getObjectProperty(error, 'status') ||
+            getObjectProperty(error, 'statusCode') ||
+            getObjectProperty(error, 'code');
 
         if (typeof rawCode === 'number') {
             code = rawCode;
         }
 
         // Extract error name if available
-        const errorName = (error as any)?.name;
+        const rawErrorName = getObjectProperty(error, 'name');
+        const errorName = typeof rawErrorName === 'string' ? rawErrorName : undefined;
 
         // Extract message
-        const message = error instanceof Error
-            ? error.message
-            : String(error);
+        const message = error instanceof Error ? error.message : String(error);
 
         // Determine retryability
         const retryable = this.isRetryableError(code, message);
 
-        return new LlumiverseError(
-            `[${this.provider}] ${message}`,
-            retryable,
-            context,
-            error,
-            code,
-            errorName
-        );
+        return new LlumiverseError(`[${this.provider}] ${message}`, retryable, context, error, code, errorName);
     }
 
     /**
      * Determine if an error is retryable based on status code and message.
      * Can be overridden by drivers for provider-specific logic.
-     * 
+     *
      * @param statusCode - The HTTP status code (if available)
      * @param message - The error message
      * @returns True if retryable, false if not retryable, undefined if unknown
      */
     protected isRetryableError(statusCode: number | undefined, message: string): boolean | undefined {
-        // Numeric status codes
-        if (statusCode !== undefined) {
-            if (statusCode === 429 || statusCode === 408) return true; // Rate limit, timeout
-            if (statusCode === 529) return true; // Overloaded
-            if (statusCode >= 500 && statusCode < 600) return true; // Server errors
-            return false; // 4xx client errors not retryable
-        }
-
-        // Message-based detection for non-HTTP errors
         const lowerMessage = message.toLowerCase();
 
-        // Rate limit variations
+        // Provider APIs sometimes surface transient failures under misleading
+        // client status codes, so high-confidence transient message signals
+        // must be honored before the generic 4xx classification below.
+        if (lowerMessage.includes('url_rejected-rejected_client_throttled')) return true;
+        if (lowerMessage.includes('url_rejected-rejected_rate_limited')) return true;
         if (lowerMessage.includes('rate') && lowerMessage.includes('limit')) return true;
-
-        // Timeout variations (timeout, timed out, time out)
         if (lowerMessage.includes('timeout')) return true;
         if (lowerMessage.includes('timed') && lowerMessage.includes('out')) return true;
         if (lowerMessage.includes('time') && lowerMessage.includes('out')) return true;
-
-        // Resource exhausted variations
         if (lowerMessage.includes('resource') && lowerMessage.includes('exhaust')) return true;
-
-        // Other retryable patterns
-        if (lowerMessage.includes('retry')) return true;
         if (lowerMessage.includes('overload')) return true;
         if (lowerMessage.includes('throttl')) return true;
         if (lowerMessage.includes('429')) return true;
         if (lowerMessage.includes('529')) return true;
+        // A transport-level abort (request-timeout / dropped connection) or a
+        // deadline-exceeded is transient and should be retried — even when the provider
+        // surfaces it under a misleading 4xx status. A deliberate cancellation is raised
+        // as a Temporal CancelledFailure (not an LLM error), so it never reaches here.
+        if (lowerMessage.includes('aborted')) return true;
+        if (lowerMessage.includes('deadline')) return true;
+
+        // Explicit auth failures should never be retried even when they arrive without
+        // a numeric status code (for example, google-auth-library invalid_grant errors).
+        if (lowerMessage.includes('invalid_grant')) return false;
+        if (lowerMessage.includes("credential's issuer")) return false;
+
+        // Numeric status codes
+        if (statusCode !== undefined) {
+            if (statusCode === 429 || statusCode === 408) return true; // Rate limit, timeout
+            if (statusCode === 529) return true; // Overloaded
+            if (statusCode >= 500 && statusCode < 600) return true; // Server errors
+            return false; // 4xx client errors not retryable
+        }
+
+        // Message-based detection for non-HTTP errors
+        if (lowerMessage.includes('retry')) return true;
 
         // Unknown errors - let consumer decide retry strategy
         return undefined;
@@ -380,10 +427,13 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
 
     abstract requestTextCompletion(prompt: PromptT, options: ExecutionOptions): Promise<Completion>;
 
-    abstract requestTextCompletionStream(prompt: PromptT, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>>;
+    abstract requestTextCompletionStream(
+        prompt: PromptT,
+        options: ExecutionOptions,
+    ): Promise<AsyncIterable<CompletionChunkObject>>;
 
     async requestImageGeneration(_prompt: PromptT, _options: ExecutionOptions): Promise<Completion> {
-        throw new Error("Image generation not implemented.");
+        throw new Error('Image generation not implemented.');
         //Cannot be made abstract, as abstract methods are required in the derived class
     }
 
@@ -398,9 +448,15 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
 
     /**
      * Cleanup method called when the driver is evicted from the cache.
-     * Override this in driver implementations that need to release resources.
+     * Releases the lazily-created HTTP agent socket pool. Override this
+     * in driver implementations that need to release additional resources
+     * — MUST call `super.destroy()` to avoid leaking sockets.
      */
     destroy(): void {
-        // No-op by default
+        this._httpAgent?.close().catch(() => {
+            /* shutdown best-effort */
+        });
+        this._httpAgent = undefined;
+        this._driverFetch = undefined;
     }
 }

package/src/async.ts

@@ -1,50 +1,52 @@
-import type { ServerSentEvent } from "@vertesia/api-fetch-client"
-import { CompletionChunkObject } from "@llumiverse/common";
+import type { CompletionChunkObject } from '@llumiverse/common';
+import type { ServerSentEvent } from '@vertesia/api-fetch-client';
 
 export async function* asyncMap<T, R>(asyncIterable: AsyncIterable<T>, callback: (value: T, index: number) => R) {
     let i = 0;
-    for await (const val of asyncIterable)
-        yield callback(val, i++);
+    for await (const val of asyncIterable) yield callback(val, i++);
 }
 
 export function oneAsyncIterator<T>(value: T): AsyncIterable<T> {
     return {
         async *[Symbol.asyncIterator]() {
-            yield value
-        }
-    }
+            yield value;
+        },
+    };
 }
 
 /**
  * Given a ReadableStream of server sent events, tran
  */
-export function transformSSEStream(stream: ReadableStream<ServerSentEvent>, transform: (data: string) => CompletionChunkObject): ReadableStream<CompletionChunkObject> & AsyncIterable<CompletionChunkObject> {
+export function transformSSEStream(
+    stream: ReadableStream<ServerSentEvent>,
+    transform: (data: string) => CompletionChunkObject,
+): ReadableStream<CompletionChunkObject> & AsyncIterable<CompletionChunkObject> {
     // on node and bun the ReadableStream is an async iterable
-    return stream.pipeThrough(new TransformStream<ServerSentEvent, CompletionChunkObject>({
-        transform(event: ServerSentEvent, controller) {
-            if (event.type === 'event' && event.data && event.data !== '[DONE]') {
-                try {
-                    const result = transform(event.data) ?? ''
-                    controller.enqueue(result);
-                } catch (err) {
-                    // double check for the last event which is not a JSON - at this time togetherai and mistralai returns the string [DONE]
-                    // do nothing - happens if data is not a JSON - the last event data is the [DONE] string
+    return stream.pipeThrough(
+        new TransformStream<ServerSentEvent, CompletionChunkObject>({
+            transform(event: ServerSentEvent, controller) {
+                if (event.type === 'event' && event.data && event.data !== '[DONE]') {
+                    try {
+                        const result = transform(event.data) ?? '';
+                        controller.enqueue(result);
+                    } catch {
+                        // double check for the last event which is not a JSON - at this time togetherai and mistralai returns the string [DONE]
+                        // do nothing - happens if data is not a JSON - the last event data is the [DONE] string
+                    }
                 }
-            }
-        }
-    })) satisfies ReadableStream<CompletionChunkObject> & AsyncIterable<CompletionChunkObject>;
+            },
+        }),
+    ) satisfies ReadableStream<CompletionChunkObject> & AsyncIterable<CompletionChunkObject>;
 }
 
-export class EventStream<T, ReturnT = any> implements AsyncIterable<T> {
-
+export class EventStream<T, ReturnT = unknown> implements AsyncIterable<T> {
     private queue: T[] = [];
     private pending?: {
-        resolve: (result: IteratorResult<T, ReturnT | undefined>) => void,
-        reject: (err: any) => void
+        resolve: (result: IteratorResult<T, ReturnT | undefined>) => void;
+        reject: (err: unknown) => void;
     };
     private done = false;
 
-
     push(event: T) {
         if (this.done) {
             throw new Error('Cannot push to a closed stream');
@@ -92,13 +94,11 @@ export class EventStream<T, ReturnT = any> implements AsyncIterable<T> {
                 }
                 const _value = await value;
                 return { done: true, value: _value };
-            }
-        }
+            },
+        };
     }
 }
 
-
-
 /**
  * Transform an async iterator by applying a function to each value.
  * @param originalGenerator
@@ -107,7 +107,7 @@ export class EventStream<T, ReturnT = any> implements AsyncIterable<T> {
 export async function* transformAsyncIterator<T, V>(
     originalGenerator: AsyncIterable<T>,
     transform: (value: T) => V | Promise<V>,
-    initCallback?: () => V | Promise<V>
+    initCallback?: () => V | Promise<V>,
 ): AsyncIterable<V> {
     if (initCallback) {
         yield initCallback();