@vibe-lang/runtime 0.2.5

package/src/runtime/ai/test/formatters.test.ts

@@ -0,0 +1,485 @@
+// Formatter tests
+
+import { describe, test, expect } from 'bun:test';
+import {
+  buildSystemMessage,
+  buildContextMessage,
+  buildPromptMessage,
+  buildMessages,
+  buildToolSystemMessage,
+  extractTextContent,
+  extractUsage,
+} from '../formatters';
+import type { ToolSchema } from '../../tools/types';
+
+describe('buildSystemMessage', () => {
+  test('returns a system message string', () => {
+    const message = buildSystemMessage();
+    expect(typeof message).toBe('string');
+    expect(message.length).toBeGreaterThan(0);
+    expect(message).toContain('Vibe');
+  });
+});
+
+describe('buildContextMessage', () => {
+  test('returns null for empty context', () => {
+    expect(buildContextMessage('')).toBeNull();
+    expect(buildContextMessage('   ')).toBeNull();
+  });
+
+  test('wraps context with header', () => {
+    const message = buildContextMessage('Variable x = 5');
+    expect(message).toContain('context');
+    expect(message).toContain('Variable x = 5');
+  });
+});
+
+describe('buildPromptMessage', () => {
+  test('returns prompt as-is', () => {
+    // Type instructions are now handled by return tools, not prompt modification
+    expect(buildPromptMessage('Hello')).toBe('Hello');
+    expect(buildPromptMessage('Calculate 2+2')).toBe('Calculate 2+2');
+  });
+});
+
+describe('buildToolSystemMessage', () => {
+  test('returns null for empty tools array', () => {
+    expect(buildToolSystemMessage([])).toBeNull();
+  });
+
+  test('formats single tool with description', () => {
+    const tools: ToolSchema[] = [
+      {
+        name: 'getWeather',
+        description: 'Get weather for a city',
+        parameters: [
+          { name: 'city', type: { type: 'string' }, required: true },
+        ],
+      },
+    ];
+
+    const message = buildToolSystemMessage(tools);
+
+    expect(message).toBe(
+`You have access to the following tools:
+- getWeather(city: string)
+    Get weather for a city
+
+Call tools when needed to complete the task.`
+    );
+  });
+
+  test('full system message with multiple user-defined tools', () => {
+    // This represents what tools defined in Vibe would look like:
+    // tool add(a: number, b: number): number @description "Add two numbers" @param a "First number" @param b "Second number"
+    // tool multiply(x: number, y: number): number @description "Multiply two numbers"
+    // tool fetchData(url: text): json @description "Fetch JSON from URL" @param url "The URL to fetch"
+    const tools: ToolSchema[] = [
+      {
+        name: 'add',
+        description: 'Add two numbers',
+        parameters: [
+          { name: 'a', type: { type: 'number' }, description: 'First number', required: true },
+          { name: 'b', type: { type: 'number' }, description: 'Second number', required: true },
+        ],
+        returns: { type: 'number' },
+      },
+      {
+        name: 'multiply',
+        description: 'Multiply two numbers',
+        parameters: [
+          { name: 'x', type: { type: 'number' }, required: true },
+          { name: 'y', type: { type: 'number' }, required: true },
+        ],
+        returns: { type: 'number' },
+      },
+      {
+        name: 'fetchData',
+        description: 'Fetch JSON from URL',
+        parameters: [
+          { name: 'url', type: { type: 'string' }, description: 'The URL to fetch', required: true },
+        ],
+        returns: { type: 'object' },
+      },
+    ];
+
+    const message = buildToolSystemMessage(tools);
+
+    expect(message).toBe(
+`You have access to the following tools:
+- add(a: number, b: number)
+    Add two numbers
+    Parameters:
+      a: First number
+      b: Second number
+- multiply(x: number, y: number)
+    Multiply two numbers
+- fetchData(url: string)
+    Fetch JSON from URL
+    Parameters:
+      url: The URL to fetch
+
+Call tools when needed to complete the task.`
+    );
+  });
+
+  test('full message array with tools, context, and prompt', () => {
+    const tools: ToolSchema[] = [
+      {
+        name: 'calculate',
+        description: 'Perform arithmetic',
+        parameters: [
+          { name: 'expression', type: { type: 'string' }, description: 'Math expression', required: true },
+        ],
+      },
+      {
+        name: 'storeResult',
+        description: 'Store a calculation result',
+        parameters: [
+          { name: 'key', type: { type: 'string' }, required: true },
+          { name: 'value', type: { type: 'number' }, required: true },
+        ],
+      },
+    ];
+
+    const contextText = `<entry> (current scope)
+  - x (number): 10
+  - y (number): 20`;
+
+    const messages = buildMessages(
+      'Calculate x + y and store it',
+      contextText,
+      tools
+    );
+
+    // Verify complete message structure
+    expect(messages).toHaveLength(4);
+
+    // Message 0: Base system message
+    expect(messages[0]).toEqual({
+      role: 'system',
+      content: `You are an AI assistant integrated into the Vibe programming language runtime.
+Your responses will be used programmatically in the execution flow.
+Be concise and precise. Follow any type constraints exactly.
+When context is provided, use it to inform your response.`,
+    });
+
+    // Message 1: Tool system message
+    expect(messages[1]).toEqual({
+      role: 'system',
+      content: `You have access to the following tools:
+- calculate(expression: string)
+    Perform arithmetic
+    Parameters:
+      expression: Math expression
+- storeResult(key: string, value: number)
+    Store a calculation result
+
+Call tools when needed to complete the task.`,
+    });
+
+    // Message 2: Context message
+    expect(messages[2]).toEqual({
+      role: 'user',
+      content: `Here is the current program context:
+
+<entry> (current scope)
+  - x (number): 10
+  - y (number): 20`,
+    });
+
+    // Message 3: Prompt
+    expect(messages[3]).toEqual({
+      role: 'user',
+      content: 'Calculate x + y and store it',
+    });
+  });
+
+  test('formats tool without description', () => {
+    const tools: ToolSchema[] = [
+      {
+        name: 'now',
+        parameters: [],
+      },
+    ];
+
+    const message = buildToolSystemMessage(tools);
+
+    expect(message).toBe(
+`You have access to the following tools:
+- now()
+
+Call tools when needed to complete the task.`
+    );
+  });
+
+  test('formats multiple tools with various parameters', () => {
+    const tools: ToolSchema[] = [
+      {
+        name: 'add',
+        description: 'Add two numbers',
+        parameters: [
+          { name: 'a', type: { type: 'number' }, required: true },
+          { name: 'b', type: { type: 'number' }, required: true },
+        ],
+      },
+      {
+        name: 'greet',
+        description: 'Greet someone',
+        parameters: [
+          { name: 'name', type: { type: 'string' }, required: true },
+        ],
+      },
+      {
+        name: 'now',
+        description: 'Get current timestamp',
+        parameters: [],
+      },
+    ];
+
+    const message = buildToolSystemMessage(tools);
+
+    expect(message).toBe(
+`You have access to the following tools:
+- add(a: number, b: number)
+    Add two numbers
+- greet(name: string)
+    Greet someone
+- now()
+    Get current timestamp
+
+Call tools when needed to complete the task.`
+    );
+  });
+
+  test('formats tool with many parameters', () => {
+    const tools: ToolSchema[] = [
+      {
+        name: 'sendEmail',
+        description: 'Send an email',
+        parameters: [
+          { name: 'to', type: { type: 'string' }, required: true },
+          { name: 'subject', type: { type: 'string' }, required: true },
+          { name: 'body', type: { type: 'string' }, required: true },
+          { name: 'cc', type: { type: 'string' }, required: false },
+        ],
+      },
+    ];
+
+    const message = buildToolSystemMessage(tools);
+
+    expect(message).toBe(
+`You have access to the following tools:
+- sendEmail(to: string, subject: string, body: string, cc: string)
+    Send an email
+
+Call tools when needed to complete the task.`
+    );
+  });
+
+  test('formats tool with complex nested object parameter', () => {
+    const tools: ToolSchema[] = [
+      {
+        name: 'createOrder',
+        description: 'Create a new order',
+        parameters: [
+          {
+            name: 'customer',
+            type: {
+              type: 'object',
+              properties: {
+                name: { type: 'string' },
+                email: { type: 'string' },
+                address: {
+                  type: 'object',
+                  properties: {
+                    street: { type: 'string' },
+                    city: { type: 'string' },
+                    zip: { type: 'string' },
+                  },
+                },
+              },
+            },
+            description: 'Customer information',
+            required: true,
+          },
+          {
+            name: 'items',
+            type: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  productId: { type: 'string' },
+                  quantity: { type: 'number' },
+                  price: { type: 'number' },
+                },
+              },
+            },
+            description: 'Order items',
+            required: true,
+          },
+        ],
+      },
+    ];
+
+    const message = buildToolSystemMessage(tools);
+
+    expect(message).toBe(
+`You have access to the following tools:
+- createOrder(customer: {name: string, email: string, address: {street: string, city: string, zip: string}}, items: {productId: string, quantity: number, price: number}[])
+    Create a new order
+    Parameters:
+      customer: Customer information
+      items: Order items
+
+Call tools when needed to complete the task.`
+    );
+  });
+});
+
+describe('buildMessages', () => {
+  test('builds messages with system and prompt', () => {
+    const messages = buildMessages('Hello', '');
+
+    expect(messages).toHaveLength(2);
+    expect(messages[0].role).toBe('system');
+    expect(messages[1].role).toBe('user');
+    expect(messages[1].content).toBe('Hello');
+  });
+
+  test('includes context message when provided', () => {
+    const messages = buildMessages('Hello', 'x = 5');
+
+    expect(messages).toHaveLength(3);
+    expect(messages[0].role).toBe('system');
+    expect(messages[1].role).toBe('user');
+    expect(messages[1].content).toContain('x = 5');
+    expect(messages[2].role).toBe('user');
+    expect(messages[2].content).toBe('Hello');
+  });
+
+  test('includes tool system message when tools provided', () => {
+    const tools: ToolSchema[] = [
+      {
+        name: 'add',
+        description: 'Add two numbers',
+        parameters: [
+          { name: 'a', type: { type: 'number' }, required: true },
+          { name: 'b', type: { type: 'number' }, required: true },
+        ],
+      },
+    ];
+
+    const messages = buildMessages('Calculate 2+3', '', tools);
+
+    // Should have: system, tool system, prompt
+    expect(messages).toHaveLength(3);
+    expect(messages[0].role).toBe('system');
+    expect(messages[1].role).toBe('system');
+    expect(messages[1].content).toContain('You have access to the following tools:');
+    expect(messages[1].content).toContain('- add(a: number, b: number)');
+    expect(messages[1].content).toContain('Add two numbers');
+    expect(messages[2].role).toBe('user');
+    expect(messages[2].content).toBe('Calculate 2+3');
+  });
+
+  test('includes all message types: system, tools, context, prompt', () => {
+    const tools: ToolSchema[] = [
+      {
+        name: 'getWeather',
+        description: 'Get weather',
+        parameters: [{ name: 'city', type: { type: 'string' }, required: true }],
+      },
+    ];
+
+    const messages = buildMessages(
+      'What is the weather?',
+      'location = Seattle',
+      tools
+    );
+
+    // Should have: system, tool system, context, prompt
+    expect(messages).toHaveLength(4);
+    expect(messages[0].role).toBe('system');
+    expect(messages[0].content).toContain('Vibe');
+    expect(messages[1].role).toBe('system');
+    expect(messages[1].content).toContain('getWeather(city: string)');
+    expect(messages[1].content).toContain('Get weather');
+    expect(messages[2].role).toBe('user');
+    expect(messages[2].content).toContain('location = Seattle');
+    expect(messages[3].role).toBe('user');
+    expect(messages[3].content).toBe('What is the weather?');
+  });
+});
+
+describe('extractTextContent', () => {
+  test('extracts from Anthropic format', () => {
+    const response = {
+      content: [{ type: 'text', text: 'Hello from Claude' }],
+    };
+    expect(extractTextContent(response)).toBe('Hello from Claude');
+  });
+
+  test('extracts from OpenAI format', () => {
+    const response = {
+      choices: [{ message: { content: 'Hello from GPT' } }],
+    };
+    expect(extractTextContent(response)).toBe('Hello from GPT');
+  });
+
+  test('extracts from Google format', () => {
+    const response = {
+      candidates: [
+        {
+          content: {
+            parts: [{ text: 'Hello from Gemini' }],
+          },
+        },
+      ],
+    };
+    expect(extractTextContent(response)).toBe('Hello from Gemini');
+  });
+
+  test('throws for unknown format', () => {
+    expect(() => extractTextContent({})).toThrow();
+    expect(() => extractTextContent({ unknown: 'format' })).toThrow();
+  });
+});
+
+describe('extractUsage', () => {
+  test('extracts from Anthropic format', () => {
+    const response = {
+      usage: { input_tokens: 10, output_tokens: 20 },
+    };
+    expect(extractUsage(response)).toEqual({
+      inputTokens: 10,
+      outputTokens: 20,
+    });
+  });
+
+  test('extracts from OpenAI format', () => {
+    const response = {
+      usage: { prompt_tokens: 15, completion_tokens: 25 },
+    };
+    expect(extractUsage(response)).toEqual({
+      inputTokens: 15,
+      outputTokens: 25,
+    });
+  });
+
+  test('extracts from Google format', () => {
+    const response = {
+      usageMetadata: { promptTokenCount: 8, candidatesTokenCount: 12 },
+    };
+    expect(extractUsage(response)).toEqual({
+      inputTokens: 8,
+      outputTokens: 12,
+    });
+  });
+
+  test('returns undefined for missing usage', () => {
+    expect(extractUsage({})).toBeUndefined();
+    expect(extractUsage({ content: 'text' })).toBeUndefined();
+  });
+});

package/src/runtime/ai/test/retry.test.ts

@@ -0,0 +1,137 @@
+// Retry logic tests
+
+import { describe, test, expect, mock } from 'bun:test';
+import {
+  isRetryableError,
+  calculateDelay,
+  withRetry,
+  createAIErrorFromResponse,
+} from '../retry';
+import { AIError } from '../types';
+
+describe('isRetryableError', () => {
+  test('returns true for AIError with isRetryable flag', () => {
+    const error = new AIError('Rate limited', 429, true);
+    expect(isRetryableError(error)).toBe(true);
+  });
+
+  test('returns false for AIError without isRetryable flag', () => {
+    const error = new AIError('Bad request', 400, false);
+    expect(isRetryableError(error)).toBe(false);
+  });
+
+  test('returns true for network-related errors', () => {
+    expect(isRetryableError(new Error('network error'))).toBe(true);
+    expect(isRetryableError(new Error('ECONNRESET'))).toBe(true);
+    expect(isRetryableError(new Error('timeout'))).toBe(true);
+    expect(isRetryableError(new Error('socket hang up'))).toBe(true);
+  });
+
+  test('returns false for non-retryable errors', () => {
+    expect(isRetryableError(new Error('Unknown error'))).toBe(false);
+    expect(isRetryableError(new Error('Invalid API key'))).toBe(false);
+  });
+});
+
+describe('calculateDelay', () => {
+  test('returns delay with exponential growth', () => {
+    const delay0 = calculateDelay(0, 1000, 30000);
+    const delay1 = calculateDelay(1, 1000, 30000);
+    const delay2 = calculateDelay(2, 1000, 30000);
+
+    // Each should be roughly double the previous (with jitter)
+    // delay0 is base * 2^0 * jitter = 1000 * 1 * (0.5-1.0) = 500-1000
+    expect(delay0).toBeGreaterThanOrEqual(500);
+    expect(delay0).toBeLessThanOrEqual(1000);
+
+    // delay1 is base * 2^1 * jitter = 1000 * 2 * (0.5-1.0) = 1000-2000
+    expect(delay1).toBeGreaterThanOrEqual(1000);
+    expect(delay1).toBeLessThanOrEqual(2000);
+
+    // delay2 is base * 2^2 * jitter = 1000 * 4 * (0.5-1.0) = 2000-4000
+    expect(delay2).toBeGreaterThanOrEqual(2000);
+    expect(delay2).toBeLessThanOrEqual(4000);
+  });
+
+  test('caps delay at maxDelay', () => {
+    const delay = calculateDelay(10, 1000, 5000);
+    // At attempt 10, base delay would be 1000 * 2^10 = 1024000
+    // But capped at 5000 with jitter = 2500-5000
+    expect(delay).toBeGreaterThanOrEqual(2500);
+    expect(delay).toBeLessThanOrEqual(5000);
+  });
+});
+
+describe('createAIErrorFromResponse', () => {
+  test('creates retryable error for 429', () => {
+    const error = createAIErrorFromResponse(429, 'Rate limited');
+    expect(error.statusCode).toBe(429);
+    expect(error.isRetryable).toBe(true);
+    expect(error.message).toContain('Rate limited');
+  });
+
+  test('creates retryable error for 5xx', () => {
+    const error500 = createAIErrorFromResponse(500, 'Internal error');
+    expect(error500.isRetryable).toBe(true);
+
+    const error503 = createAIErrorFromResponse(503, 'Service unavailable');
+    expect(error503.isRetryable).toBe(true);
+  });
+
+  test('creates non-retryable error for 4xx (except 429)', () => {
+    const error400 = createAIErrorFromResponse(400, 'Bad request');
+    expect(error400.isRetryable).toBe(false);
+
+    const error401 = createAIErrorFromResponse(401, 'Unauthorized');
+    expect(error401.isRetryable).toBe(false);
+  });
+});
+
+describe('withRetry', () => {
+  test('returns result on first success', async () => {
+    const fn = mock().mockResolvedValue('success');
+    const result = await withRetry(fn, { maxRetries: 3 });
+
+    expect(result).toBe('success');
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  test('retries on retryable error', async () => {
+    const fn = mock()
+      .mockRejectedValueOnce(new AIError('Rate limited', 429, true))
+      .mockResolvedValue('success');
+
+    const result = await withRetry(fn, { maxRetries: 3, baseDelayMs: 10 });
+
+    expect(result).toBe('success');
+    expect(fn).toHaveBeenCalledTimes(2);
+  });
+
+  test('does not retry on non-retryable error', async () => {
+    const fn = mock().mockRejectedValue(new AIError('Bad request', 400, false));
+
+    await expect(withRetry(fn, { maxRetries: 3 })).rejects.toThrow('Bad request');
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  test('throws after max retries', async () => {
+    const fn = mock().mockRejectedValue(new AIError('Rate limited', 429, true));
+
+    await expect(
+      withRetry(fn, { maxRetries: 2, baseDelayMs: 10 })
+    ).rejects.toThrow('Rate limited');
+
+    expect(fn).toHaveBeenCalledTimes(3); // Initial + 2 retries
+  });
+
+  test('retries on network errors', async () => {
+    const fn = mock()
+      .mockRejectedValueOnce(new Error('network error'))
+      .mockResolvedValue('success');
+
+    const result = await withRetry(fn, { maxRetries: 3, baseDelayMs: 10 });
+
+    expect(result).toBe('success');
+    expect(fn).toHaveBeenCalledTimes(2);
+  });
+});