@llumiverse/core 1.0.0-dev.20260224.234313Z → 1.0.0

package/src/Driver.error.test.ts

@@ -0,0 +1,261 @@
+import {
+    AIModel,
+    Completion,
+    CompletionChunkObject,
+    DriverOptions,
+    EmbeddingsOptions,
+    EmbeddingsResult,
+    ExecutionOptions,
+    LlumiverseErrorContext,
+    ModelSearchPayload,
+    LlumiverseError,
+} from '@llumiverse/common';
+import { beforeEach, describe, expect, it } from 'vitest';
+import { AbstractDriver } from './Driver.js';
+
+// Simple test driver implementation
+class TestDriver extends AbstractDriver<DriverOptions, string> {
+    provider = 'test-provider';
+
+    async requestTextCompletion(_prompt: string, _options: ExecutionOptions): Promise<Completion> {
+        throw new Error('Not implemented');
+    }
+
+    async requestTextCompletionStream(_prompt: string, _options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
+        throw new Error('Not implemented');
+    }
+
+    async listModels(_params?: ModelSearchPayload): Promise<AIModel[]> {
+        return [];
+    }
+
+    async validateConnection(): Promise<boolean> {
+        return true;
+    }
+
+    async generateEmbeddings(_options: EmbeddingsOptions): Promise<EmbeddingsResult> {
+        throw new Error('Not implemented');
+    }
+}
+
+describe('AbstractDriver Error Formatting', () => {
+    let driver: TestDriver;
+    const mockContext: LlumiverseErrorContext = {
+        provider: 'test-provider',
+        model: 'test-model',
+        operation: 'execute',
+    };
+
+    beforeEach(() => {
+        driver = new TestDriver({});
+    });
+
+    describe('isRetryableError', () => {
+        describe('HTTP status codes', () => {
+            it('should mark 429 as retryable (rate limit)', () => {
+                expect(driver['isRetryableError'](429, 'Rate limit exceeded')).toBe(true);
+            });
+
+            it('should mark 408 as retryable (timeout)', () => {
+                expect(driver['isRetryableError'](408, 'Request timeout')).toBe(true);
+            });
+
+            it('should mark 529 as retryable (overloaded)', () => {
+                expect(driver['isRetryableError'](529, 'Service overloaded')).toBe(true);
+            });
+
+            it('should mark 5xx as retryable (server errors)', () => {
+                expect(driver['isRetryableError'](500, 'Internal server error')).toBe(true);
+                expect(driver['isRetryableError'](502, 'Bad gateway')).toBe(true);
+                expect(driver['isRetryableError'](503, 'Service unavailable')).toBe(true);
+                expect(driver['isRetryableError'](504, 'Gateway timeout')).toBe(true);
+            });
+
+            it('should mark 4xx as not retryable (except 429, 408)', () => {
+                expect(driver['isRetryableError'](400, 'Bad request')).toBe(false);
+                expect(driver['isRetryableError'](401, 'Unauthorized')).toBe(false);
+                expect(driver['isRetryableError'](403, 'Forbidden')).toBe(false);
+                expect(driver['isRetryableError'](404, 'Not found')).toBe(false);
+            });
+
+            it('should mark 2xx and 3xx as not retryable', () => {
+                expect(driver['isRetryableError'](200, 'OK')).toBe(false);
+                expect(driver['isRetryableError'](301, 'Moved permanently')).toBe(false);
+            });
+        });
+
+        describe('message-based detection', () => {
+            it('should detect rate limit in message', () => {
+                expect(driver['isRetryableError'](undefined, 'Rate limit exceeded')).toBe(true);
+                expect(driver['isRetryableError'](undefined, 'You have hit the rate limit')).toBe(true);
+                expect(driver['isRetryableError'](undefined, 'RATE_LIMIT_EXCEEDED')).toBe(true);
+            });
+
+            it('should detect timeout in message', () => {
+                expect(driver['isRetryableError'](undefined, 'Request timeout')).toBe(true);
+                expect(driver['isRetryableError'](undefined, 'Connection timed out')).toBe(true);
+                expect(driver['isRetryableError'](undefined, 'TIMEOUT_ERROR')).toBe(true);
+            });
+
+            it('should detect retry in message', () => {
+                expect(driver['isRetryableError'](undefined, 'Please retry later')).toBe(true);
+                expect(driver['isRetryableError'](undefined, 'Retry the request')).toBe(true);
+            });
+
+            it('should detect overload in message', () => {
+                expect(driver['isRetryableError'](undefined, 'Service overloaded')).toBe(true);
+                expect(driver['isRetryableError'](undefined, 'Server is overload')).toBe(true);
+                expect(driver['isRetryableError'](undefined, 'System overloaded')).toBe(true);
+            });
+
+            it('should detect resource exhausted in message', () => {
+                expect(driver['isRetryableError'](undefined, 'Resource exhausted')).toBe(true);
+                expect(driver['isRetryableError'](undefined, 'Resources exhausted')).toBe(true);
+            });
+
+            it('should detect throttle in message', () => {
+                expect(driver['isRetryableError'](undefined, 'Request throttled')).toBe(true);
+                expect(driver['isRetryableError'](undefined, 'Throttling exception')).toBe(true);
+            });
+
+            it('should detect status codes in message', () => {
+                expect(driver['isRetryableError'](undefined, 'Error 429: Too many requests')).toBe(true);
+                expect(driver['isRetryableError'](undefined, 'HTTP 529 error')).toBe(true);
+            });
+
+            it('should mark unknown messages as undefined (let consumer decide)', () => {
+                expect(driver['isRetryableError'](undefined, 'Invalid API key')).toBeUndefined();
+                expect(driver['isRetryableError'](undefined, 'Bad request')).toBeUndefined();
+                expect(driver['isRetryableError'](undefined, 'Model not found')).toBeUndefined();
+            });
+        });
+    });
+
+    describe('formatLlumiverseError', () => {
+        it('should format error with status code', () => {
+            const originalError = new Error('Rate limit exceeded');
+            (originalError as any).status = 429;
+
+            const formatted = driver['formatLlumiverseError'](originalError, mockContext);
+
+            expect(formatted).toBeInstanceOf(LlumiverseError);
+            expect(formatted.code).toBe(429);
+            expect(formatted.retryable).toBe(true);
+            expect(formatted.message).toContain('[test-provider]');
+            expect(formatted.message).toContain('Rate limit exceeded');
+            expect(formatted.context).toEqual(mockContext);
+            expect(formatted.originalError).toBe(originalError);
+        });
+
+        it('should extract status from statusCode property', () => {
+            const originalError = new Error('Server error');
+            (originalError as any).statusCode = 500;
+
+            const formatted = driver['formatLlumiverseError'](originalError, mockContext);
+
+            expect(formatted.code).toBe(500);
+            expect(formatted.retryable).toBe(true);
+        });
+
+        it('should extract status from code property', () => {
+            const originalError = new Error('Timeout');
+            (originalError as any).code = 408;
+
+            const formatted = driver['formatLlumiverseError'](originalError, mockContext);
+
+            expect(formatted.code).toBe(408);
+            expect(formatted.retryable).toBe(true);
+        });
+
+        it('should use undefined when no status code found', () => {
+            const originalError = new Error('Generic error');
+
+            const formatted = driver['formatLlumiverseError'](originalError, mockContext);
+
+            expect(formatted.code).toBeUndefined();
+            expect(formatted.retryable).toBeUndefined(); // Unknown retryability
+        });
+
+        it('should handle non-Error objects', () => {
+            const originalError = 'String error message';
+
+            const formatted = driver['formatLlumiverseError'](originalError, mockContext);
+
+            expect(formatted.message).toContain('String error message');
+            expect(formatted.originalError).toBe(originalError);
+        });
+
+        it('should preserve provider in message', () => {
+            const error = new Error('Test error');
+
+            const formatted = driver['formatLlumiverseError'](error, mockContext);
+
+            expect(formatted.message).toMatch(/^\[test-provider\]/);
+        });
+
+        it('should determine retryability based on status and message', () => {
+            // Retryable by status
+            const retryableError = new Error('Error');
+            (retryableError as any).status = 429;
+            const formatted1 = driver['formatLlumiverseError'](retryableError, mockContext);
+            expect(formatted1.retryable).toBe(true);
+
+            // Not retryable by status
+            const nonRetryableError = new Error('Error');
+            (nonRetryableError as any).status = 400;
+            const formatted2 = driver['formatLlumiverseError'](nonRetryableError, mockContext);
+            expect(formatted2.retryable).toBe(false);
+
+            // Retryable by message
+            const messageRetryable = new Error('Rate limit exceeded');
+            const formatted3 = driver['formatLlumiverseError'](messageRetryable, mockContext);
+            expect(formatted3.retryable).toBe(true);
+        });
+    });
+
+    describe('driver override capability', () => {
+        class CustomDriver extends TestDriver {
+            public formatLlumiverseError(
+                error: unknown,
+                context: LlumiverseErrorContext
+            ): LlumiverseError {
+                // Custom logic: check for specific error type
+                if ((error as any).type === 'custom_retryable') {
+                    return new LlumiverseError(
+                        `[${this.provider}] Custom retryable error`,
+                        true,
+                        context,
+                        error,
+                        undefined,
+                        'CUSTOM_ERROR'
+                    );
+                }
+                // Fall back to default
+                return super.formatLlumiverseError(error, context);
+            }
+        }
+
+        it('should allow drivers to override error formatting', () => {
+            const customDriver = new CustomDriver({});
+            const customError = { type: 'custom_retryable', message: 'Custom error' };
+
+            const formatted = customDriver['formatLlumiverseError'](customError, mockContext);
+
+            expect(formatted.name).toBe('CUSTOM_ERROR');
+            expect(formatted.code).toBeUndefined();
+            expect(formatted.retryable).toBe(true);
+            expect(formatted.message).toContain('Custom retryable error');
+        });
+
+        it('should fall back to default for non-custom errors', () => {
+            const customDriver = new CustomDriver({});
+            const regularError = new Error('Regular error');
+            (regularError as any).status = 500;
+
+            const formatted = customDriver['formatLlumiverseError'](regularError, mockContext);
+
+            expect(formatted.code).toBe(500);
+            expect(formatted.retryable).toBe(true);
+        });
+    });
+});

package/src/Driver.ts

@@ -4,8 +4,6 @@
  * (eg: OpenAI, HuggingFace, etc.)
  */
 
-import { DefaultCompletionStream, FallbackCompletionStream } from "./CompletionStream.js";
-import { formatTextPrompt } from "./formatters/index.js";
 import {
     AIModel,
     Completion,
@@ -17,15 +15,19 @@ import {
     EmbeddingsResult,
     ExecutionOptions,
     ExecutionResponse,
+    LlumiverseErrorContext,
     Logger,
-    Modalities,
     ModelSearchPayload,
     PromptOptions,
     PromptSegment,
+    Providers,
     TrainingJob,
     TrainingOptions,
-    TrainingPromptOptions
+    TrainingPromptOptions,
+    LlumiverseError
 } from "@llumiverse/common";
+import { DefaultCompletionStream, FallbackCompletionStream } from "./CompletionStream.js";
+import { formatTextPrompt } from "./formatters/index.js";
 import { validateResult } from "./validation.js";
 
 // Helper to create logger methods that support both message-only and object-first signatures
@@ -119,7 +121,7 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     options: OptionsT;
     logger: Logger;
 
-    abstract provider: string; // the provider name
+    abstract provider: Providers | string; // the provider name
 
     constructor(opts: OptionsT) {
         this.options = opts;
@@ -165,8 +167,15 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     async execute(segments: PromptSegment[], options: ExecutionOptions): Promise<ExecutionResponse<PromptT>> {
         const prompt = await this.createPrompt(segments, options);
         return this._execute(prompt, options).catch((error: any) => {
-            (error as any).prompt = prompt;
-            throw error;
+            // Don't wrap if already a LlumiverseError
+            if (LlumiverseError.isLlumiverseError(error)) {
+                throw error;
+            }
+            throw this.formatLlumiverseError(error, {
+                provider: this.provider,
+                model: options.model,
+                operation: 'execute',
+            });
         });
     }
 
@@ -189,8 +198,17 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
             const execution_time = Date.now() - start;
             return { ...result, prompt, execution_time };
         } catch (error) {
-            (error as any).prompt = prompt;
-            throw error;
+            // Don't wrap if already a LlumiverseError
+            if (LlumiverseError.isLlumiverseError(error)) {
+                throw error;
+            }
+            // Log the original error for debugging
+            this.logger.error({ err: error, data: { provider: this.provider, model: options.model, operation: 'execute', prompt } }, `Error during execution in provider ${this.provider}:`);
+            throw this.formatLlumiverseError(error, {
+                provider: this.provider,
+                model: options.model,
+                operation: 'execute',
+            });
         }
     }
 
@@ -200,9 +218,10 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
 
     // by default no stream is supported. we block and we return all at once
     async stream(segments: PromptSegment[], options: ExecutionOptions): Promise<CompletionStream<PromptT>> {
+        this.logger.info(options, `Executing prompt with provider ${this.provider} with options: ${JSON.stringify(options)}`);
         const prompt = await this.createPrompt(segments, options);
         const canStream = await this.canStream(options);
-        if (options.output_modality === Modalities.text && canStream) {
+        if (canStream) {
             return new DefaultCompletionStream(this, prompt, options);
         } else if (this.isImageModel(options.model)) {
             return new FallbackCompletionStream(this, prompt, options);
@@ -267,6 +286,98 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
         return undefined;
     }
 
+    /**
+     * Format an error into LlumiverseError. Override in driver implementations
+     * to provide provider-specific error parsing.
+     * 
+     * The default implementation uses common patterns:
+     * - Status 429, 408: retryable (rate limit, timeout)
+     * - Status 529: retryable (overloaded)
+     * - Status 5xx: retryable (server errors)
+     * - Status 4xx (except above): not retryable (client errors)
+     * - Error messages containing "rate limit", "timeout", etc.: retryable
+     * 
+     * @param error - The error to format
+     * @param context - Context about where the error occurred
+     * @returns A standardized LlumiverseError
+     */
+    public formatLlumiverseError(
+        error: unknown,
+        context: LlumiverseErrorContext
+    ): LlumiverseError {
+        // Extract status code from common locations (only if numeric)
+        let code: number | undefined;
+        const rawCode = (error as any)?.status
+            || (error as any)?.statusCode
+            || (error as any)?.code;
+
+        if (typeof rawCode === 'number') {
+            code = rawCode;
+        }
+
+        // Extract error name if available
+        const errorName = (error as any)?.name;
+
+        // Extract message
+        const message = error instanceof Error
+            ? error.message
+            : String(error);
+
+        // Determine retryability
+        const retryable = this.isRetryableError(code, message);
+
+        return new LlumiverseError(
+            `[${this.provider}] ${message}`,
+            retryable,
+            context,
+            error,
+            code,
+            errorName
+        );
+    }
+
+    /**
+     * Determine if an error is retryable based on status code and message.
+     * Can be overridden by drivers for provider-specific logic.
+     * 
+     * @param statusCode - The HTTP status code (if available)
+     * @param message - The error message
+     * @returns True if retryable, false if not retryable, undefined if unknown
+     */
+    protected isRetryableError(statusCode: number | undefined, message: string): boolean | undefined {
+        // Numeric status codes
+        if (statusCode !== undefined) {
+            if (statusCode === 429 || statusCode === 408) return true; // Rate limit, timeout
+            if (statusCode === 529) return true; // Overloaded
+            if (statusCode >= 500 && statusCode < 600) return true; // Server errors
+            return false; // 4xx client errors not retryable
+        }
+
+        // Message-based detection for non-HTTP errors
+        const lowerMessage = message.toLowerCase();
+
+        // Rate limit variations
+        if (lowerMessage.includes('rate') && lowerMessage.includes('limit')) return true;
+
+        // Timeout variations (timeout, timed out, time out)
+        if (lowerMessage.includes('timeout')) return true;
+        if (lowerMessage.includes('timed') && lowerMessage.includes('out')) return true;
+        if (lowerMessage.includes('time') && lowerMessage.includes('out')) return true;
+
+        // Resource exhausted variations
+        if (lowerMessage.includes('resource') && lowerMessage.includes('exhaust')) return true;
+
+        // Other retryable patterns
+        if (lowerMessage.includes('retry')) return true;
+        if (lowerMessage.includes('overload')) return true;
+        if (lowerMessage.includes('throttl')) return true;
+        if (lowerMessage.includes('429')) return true;
+        if (lowerMessage.includes('529')) return true;
+
+        // Unknown errors - let consumer decide retry strategy
+        return undefined;
+    }
+
     abstract requestTextCompletion(prompt: PromptT, options: ExecutionOptions): Promise<Completion>;
 
     abstract requestTextCompletionStream(prompt: PromptT, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>>;

package/src/conversation-utils.ts

@@ -529,3 +529,69 @@ function truncateLargeTextInternal(obj: unknown, maxChars: number): unknown {
 
     return obj;
 }
+
+const HEARTBEAT_OPEN_TAG = '<heartbeat>';
+const HEARTBEAT_CLOSE_TAG = '</heartbeat>';
+const HEARTBEAT_PLACEHOLDER = '[Heartbeat removed from conversation history]';
+
+/**
+ * Strip heartbeat status messages from conversation to reduce context bloat.
+ *
+ * Heartbeat messages are periodic workstream status updates injected by the
+ * workstream management system. They are wrapped with `<heartbeat>...</heartbeat>`
+ * tags at the point of injection.
+ *
+ * This function recursively walks the conversation and replaces any string
+ * wrapped in heartbeat tags with a short placeholder.
+ *
+ * @param obj The conversation object to strip heartbeats from
+ * @param options Optional settings for turn-based stripping (default keepForTurns: 1)
+ * @returns A new object with old heartbeat messages replaced
+ */
+export function stripHeartbeatsFromConversation(obj: unknown, options?: StripOptions): unknown {
+    const { keepForTurns = 1 } = options ?? {};
+    const currentTurn = options?.currentTurn ?? getConversationMeta(obj).turnNumber;
+
+    // If keepForTurns is Infinity, never strip
+    if (keepForTurns === Infinity) {
+        return obj;
+    }
+
+    // Keep heartbeats if we haven't exceeded the turn threshold
+    if (keepForTurns > 0 && currentTurn < keepForTurns) {
+        return obj;
+    }
+
+    return stripHeartbeatsInternal(obj);
+}
+
+function stripHeartbeatsInternal(obj: unknown): unknown {
+    if (obj === null || obj === undefined) return obj;
+
+    // Replace heartbeat-tagged strings with placeholder
+    if (typeof obj === 'string') {
+        if (obj.startsWith(HEARTBEAT_OPEN_TAG) && obj.endsWith(HEARTBEAT_CLOSE_TAG)) {
+            return HEARTBEAT_PLACEHOLDER;
+        }
+        return obj;
+    }
+
+    if (Array.isArray(obj)) {
+        return obj.map(item => stripHeartbeatsInternal(item));
+    }
+
+    if (typeof obj === 'object') {
+        const result: Record<string, unknown> = {};
+        for (const [key, value] of Object.entries(obj)) {
+            // Preserve metadata
+            if (key === META_KEY) {
+                result[key] = value;
+            } else {
+                result[key] = stripHeartbeatsInternal(value);
+            }
+        }
+        return result;
+    }
+
+    return obj;
+}

package/src/index.ts

@@ -1,5 +1,5 @@
+export * from "@llumiverse/common";
+export * from "./conversation-utils.js";
 export * from "./Driver.js";
 export * from "./json.js";
 export * from "./stream.js";
-export * from "./conversation-utils.js";
-export * from "@llumiverse/common";

package/src/validation.ts

@@ -47,7 +47,7 @@ function parseCompletionAsJson(data: CompletionResult[]) {
 }
 
 
-export function validateResult(data: CompletionResult[], schema: Object): CompletionResult[] {
+export function validateResult(data: CompletionResult[], schema: object): CompletionResult[] {
     let json;
     if (Array.isArray(data)) {
         const jsonResults = data.filter(r => r.type === "json");
@@ -68,7 +68,7 @@ export function validateResult(data: CompletionResult[], schema: Object): Comple
     const valid = validate(json);
 
     if (!valid && validate.errors) {
-        let errors = [];
+        const errors = [];
 
         for (const e of validate.errors) {
             const path = e.instancePath.split("/").slice(1);