@vibe-lang/runtime 0.2.5

package/tests/integration/basic-ai/basic-ai.integration.test.ts

@@ -0,0 +1,166 @@
+// Basic AI Integration Tests
+// Tests real API calls with OpenAI, Anthropic, and Google for all return types
+
+import { describe, test, expect } from 'bun:test';
+import { Runtime, formatAIInteractions } from '../../../src/runtime';
+import { createRealAIProvider } from '../../../src/runtime/ai-provider';
+import { parse } from '../../../src/parser/parse';
+
+// API Keys from environment
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
+const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY;
+
+// Provider configurations
+const providers = [
+  {
+    name: 'OpenAI',
+    hasKey: !!OPENAI_API_KEY,
+    modelConfig: `
+model testModel = {
+  name: "gpt-5-mini",
+  apiKey: "${OPENAI_API_KEY}",
+  url: "https://api.openai.com/v1",
+  provider: "openai"
+}
+`,
+  },
+  {
+    name: 'Anthropic',
+    hasKey: !!ANTHROPIC_API_KEY,
+    modelConfig: `
+model testModel = {
+  name: "claude-haiku-4-5",
+  apiKey: "${ANTHROPIC_API_KEY}",
+  url: "https://api.anthropic.com",
+  provider: "anthropic"
+}
+`,
+  },
+  {
+    name: 'Google',
+    hasKey: !!GOOGLE_API_KEY,
+    modelConfig: `
+model testModel = {
+  name: "gemini-3-flash-preview",
+  apiKey: "${GOOGLE_API_KEY}",
+  provider: "google"
+}
+`,
+  },
+];
+
+// Shared Vibe code for each test case
+const testCases = [
+  {
+    name: 'returns text response',
+    vibeCode: `let result: text = vibe "Reply with exactly: PONG" testModel default`,
+    assert: (runtime: Runtime) => {
+      const result = runtime.getValue('result') as string;
+      expect(typeof result).toBe('string');
+      expect(result.toUpperCase()).toContain('PONG');
+    },
+  },
+  {
+    name: 'returns number response',
+    vibeCode: `let result: number = vibe "What is 2 + 2? Reply with just the number." testModel default`,
+    assert: (runtime: Runtime) => {
+      const result = runtime.getValue('result');
+      expect(typeof result).toBe('number');
+      expect(result).toBe(4);
+    },
+  },
+  {
+    name: 'returns boolean response',
+    vibeCode: `let result: boolean = vibe "Is 5 greater than 3? Reply with true or false only." testModel default`,
+    assert: (runtime: Runtime) => {
+      const result = runtime.getValue('result');
+      expect(typeof result).toBe('boolean');
+      expect(result).toBe(true);
+    },
+  },
+  {
+    name: 'returns json response',
+    vibeCode: `let result: json = vibe "Return a JSON object with name set to Alice and age set to 30" testModel default`,
+    assert: (runtime: Runtime) => {
+      const result = runtime.getValue('result') as Record<string, unknown>;
+      expect(typeof result).toBe('object');
+      expect(result).not.toBeNull();
+      expect(result.name).toBe('Alice');
+      expect(result.age).toBe(30);
+    },
+  },
+  {
+    name: 'returns text[] response',
+    vibeCode: `let result: text[] = vibe "Return a JSON array of 3 colors: red, green, blue" testModel default`,
+    assert: (runtime: Runtime) => {
+      const result = runtime.getValue('result') as string[];
+      expect(Array.isArray(result)).toBe(true);
+      expect(result).toHaveLength(3);
+      expect(result).toContain('red');
+      expect(result).toContain('green');
+      expect(result).toContain('blue');
+    },
+  },
+  {
+    name: 'returns number[] response',
+    vibeCode: `let result: number[] = vibe "Return a JSON array of the first 5 prime numbers" testModel default`,
+    assert: (runtime: Runtime) => {
+      const result = runtime.getValue('result') as number[];
+      expect(Array.isArray(result)).toBe(true);
+      expect(result).toHaveLength(5);
+      expect(result).toEqual([2, 3, 5, 7, 11]);
+    },
+  },
+  {
+    name: 'returns boolean[] response',
+    vibeCode: `let result: boolean[] = vibe "Return a JSON array: [true, false, true]" testModel default`,
+    assert: (runtime: Runtime) => {
+      const result = runtime.getValue('result') as boolean[];
+      expect(Array.isArray(result)).toBe(true);
+      expect(result).toEqual([true, false, true]);
+    },
+  },
+  {
+    name: 'returns json[] response',
+    vibeCode: `let result: json[] = do "Return a JSON array with 2 objects: first with name Alice and age 30, second with name Bob and age 25" testModel default`,
+    assert: (runtime: Runtime) => {
+      const result = runtime.getValue('result') as Array<Record<string, unknown>>;
+      expect(Array.isArray(result)).toBe(true);
+      expect(result).toHaveLength(2);
+      expect(result[0].name).toBe('Alice');
+      expect(result[0].age).toBe(30);
+      expect(result[1].name).toBe('Bob');
+      expect(result[1].age).toBe(25);
+    },
+  },
+];
+
+async function runVibe(modelConfig: string, vibeCode: string, logAi = true): Promise<Runtime> {
+  const program = parse(modelConfig + vibeCode);
+  const runtime = new Runtime(
+    program,
+    createRealAIProvider(() => runtime.getState()),
+    { logAiInteractions: logAi }
+  );
+  await runtime.run();
+
+  if (logAi) {
+    const interactions = runtime.getAIInteractions();
+    console.log('\n' + formatAIInteractions(interactions));
+  }
+
+  return runtime;
+}
+
+// Generate tests for each provider
+for (const provider of providers) {
+  describe.skipIf(!provider.hasKey)(`${provider.name} Integration - Return Types`, () => {
+    for (const testCase of testCases) {
+      test(testCase.name, async () => {
+        const runtime = await runVibe(provider.modelConfig, testCase.vibeCode);
+        testCase.assert(runtime);
+      }, 30000);
+    }
+  });
+}

package/tests/integration/basic-ai/basic-ai.vibe

@@ -0,0 +1,12 @@
+// Basic AI Integration Test
+// Tests the most fundamental AI operation: a simple prompt
+
+model testModel = {
+  name: "gpt-5-mini",
+  apiKey: OPENAI_API_KEY,
+  url: "https://api.openai.com/v1",
+  provider: "openai"
+}
+
+// Test: Basic text response - model should reply with exactly "PONG"
+let greeting = vibe "Reply with exactly: PONG" testModel default

package/tests/integration/bug-fix/bug-fix.integration.test.ts

@@ -0,0 +1,201 @@
+// Bug Fix Integration Test
+// Tests that AI can use tools to find and fix bugs in real code
+
+import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
+import { Runtime, formatAIInteractions } from '../../../src/runtime';
+import { createRealAIProvider } from '../../../src/runtime/ai-provider';
+import { parse } from '../../../src/parser/parse';
+import * as fs from 'fs';
+import * as path from 'path';
+
+// API Keys from environment
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
+const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY;
+
+// Test directory setup
+const BASE_WORKSPACE = path.join(__dirname, '.test-workspace');
+
+// The buggy code - sumArray has an off-by-one error (starts at index 1 instead of 0)
+const BUGGY_CODE = `// Array utilities module
+
+export function sumArray(numbers: number[]): number {
+  // Bug: loop starts at index 1, skipping the first element
+  let total = 0;
+  for (let i = 1; i < numbers.length; i++) {
+    total += numbers[i];
+  }
+  return total;
+}
+
+export function findMax(numbers: number[]): number {
+  if (numbers.length === 0) {
+    return 0;
+  }
+  // This function is correct - starts at 1 because max is initialized with numbers[0]
+  let max = numbers[0];
+  for (let j = 1; j < numbers.length; j++) {
+    if (numbers[j] > max) {
+      max = numbers[j];
+    }
+  }
+  return max;
+}
+`;
+
+// Generate Vibe program with provider-specific model config
+function createVibeProgram(modelConfig: string): string {
+  return `
+import { readFile, edit } from "system/tools"
+
+${modelConfig}
+
+let result: text = vibe "There's a file 'buggy-code.ts' with array utility functions.
+
+A test is failing: sumArray([1, 2, 3, 4, 5]) returns 14 but should return 15.
+
+Read the file, find the bug, and fix it. After fixing, respond with 'FIXED'." fixer default
+
+result
+`;
+}
+
+// Core test function that runs the bug fix test for a given provider
+async function runBugFixTest(
+  providerName: string,
+  modelConfig: string,
+  workspace: string
+): Promise<void> {
+  const buggyFile = path.join(workspace, 'buggy-code.ts');
+
+  // Ensure workspace exists and reset the buggy file
+  fs.mkdirSync(workspace, { recursive: true });
+  fs.writeFileSync(buggyFile, BUGGY_CODE);
+
+  // Verify the code is buggy
+  const buggyModule = await import(`${buggyFile}?t=${Date.now()}`);
+  const buggyResult = buggyModule.sumArray([1, 2, 3, 4, 5]);
+  console.log(`\n[${providerName}] Buggy sumArray([1,2,3,4,5]) = ${buggyResult} (expected 15)`);
+  expect(buggyResult).toBe(14);
+
+  // Run the Vibe program
+  const program = parse(createVibeProgram(modelConfig));
+  const runtime = new Runtime(
+    program,
+    createRealAIProvider(() => runtime.getState()),
+    { logAiInteractions: true, rootDir: workspace }
+  );
+
+  await runtime.run();
+
+  // Log AI interactions
+  const state = runtime.getState();
+  console.log('\n' + formatAIInteractions(state.aiInteractions));
+
+  // Verify tool calls were made
+  const hasToolCalls = state.callStack.some(frame =>
+    frame.orderedEntries.some(entry =>
+      entry.kind === 'prompt' && entry.toolCalls && entry.toolCalls.length > 0
+    )
+  );
+  console.log(`\n[${providerName}] Tool calls made: ${hasToolCalls}`);
+  expect(hasToolCalls).toBe(true);
+
+  // Verify the fix
+  const fixedCode = fs.readFileSync(buggyFile, 'utf-8');
+  console.log(`\n[${providerName}] Fixed code:\n`, fixedCode);
+  expect(fixedCode).toContain('i = 0');
+
+  // Test the fixed code
+  const fixedModule = await import(`${buggyFile}?t=${Date.now()}`);
+  const fixedResult = fixedModule.sumArray([1, 2, 3, 4, 5]);
+  console.log(`[${providerName}] Fixed sumArray([1,2,3,4,5]) = ${fixedResult}`);
+  expect(fixedResult).toBe(15);
+}
+
+// Test findMax still works after fix
+async function testFindMaxAfterFix(workspace: string): Promise<void> {
+  const buggyFile = path.join(workspace, 'buggy-code.ts');
+  const fixedModule = await import(`${buggyFile}?t=${Date.now()}`);
+  expect(fixedModule.findMax([3, 1, 4, 1, 5, 9, 2, 6])).toBe(9);
+  expect(fixedModule.findMax([1])).toBe(1);
+  expect(fixedModule.findMax([])).toBe(0);
+}
+
+// Cleanup workspace
+function cleanupWorkspace(workspace: string): void {
+  if (fs.existsSync(workspace)) {
+    fs.rmSync(workspace, { recursive: true });
+  }
+}
+
+// OpenAI Tests
+describe.skipIf(!OPENAI_API_KEY)('Bug Fix Integration - OpenAI', () => {
+  const workspace = path.join(BASE_WORKSPACE, 'openai');
+  const modelConfig = `
+model fixer = {
+  name: "gpt-5-mini",
+  apiKey: "${OPENAI_API_KEY}",
+  url: "https://api.openai.com/v1",
+  provider: "openai",
+  tools: [readFile, edit]
+}
+`;
+
+  afterAll(() => cleanupWorkspace(workspace));
+
+  test('AI diagnoses and fixes bug using tools', async () => {
+    await runBugFixTest('OpenAI', modelConfig, workspace);
+  }, 120000);
+
+  test('findMax function still works after fix', async () => {
+    await testFindMaxAfterFix(workspace);
+  }, 10000);
+});
+
+// Anthropic Tests
+describe.skipIf(!ANTHROPIC_API_KEY)('Bug Fix Integration - Anthropic', () => {
+  const workspace = path.join(BASE_WORKSPACE, 'anthropic');
+  const modelConfig = `
+model fixer = {
+  name: "claude-haiku-4-5",
+  apiKey: "${ANTHROPIC_API_KEY}",
+  url: "https://api.anthropic.com",
+  provider: "anthropic",
+  tools: [readFile, edit]
+}
+`;
+
+  afterAll(() => cleanupWorkspace(workspace));
+
+  test('AI diagnoses and fixes bug using tools', async () => {
+    await runBugFixTest('Anthropic', modelConfig, workspace);
+  }, 120000);
+
+  test('findMax function still works after fix', async () => {
+    await testFindMaxAfterFix(workspace);
+  }, 10000);
+});
+
+// Google Tests
+describe.skipIf(!GOOGLE_API_KEY)('Bug Fix Integration - Google', () => {
+  const workspace = path.join(BASE_WORKSPACE, 'google');
+  const modelConfig = `
+model fixer = {
+  name: "gemini-3-flash-preview",
+  apiKey: "${GOOGLE_API_KEY}",
+  provider: "google",
+  tools: [readFile, edit]
+}
+`;
+
+  afterAll(() => cleanupWorkspace(workspace));
+
+  test('AI diagnoses and fixes bug using tools', async () => {
+    await runBugFixTest('Google', modelConfig, workspace);
+  }, 120000);
+
+  test('findMax function still works after fix', async () => {
+    await testFindMaxAfterFix(workspace);
+  }, 10000);
+});

package/tests/integration/bug-fix/buggy-code.ts

@@ -0,0 +1,22 @@
+// Array utilities module
+
+export function sumArray(numbers: number[]): number {
+  let total = 0;
+  for (let i = 1; i < numbers.length; i++) {
+    total += numbers[i];
+  }
+  return total;
+}
+
+export function findMax(numbers: number[]): number {
+  if (numbers.length === 0) {
+    return 0;
+  }
+  let max = numbers[0];
+  for (let i = 1; i < numbers.length; i++) {
+    if (numbers[i] > max) {
+      max = numbers[i];
+    }
+  }
+  return max;
+}

package/tests/integration/bug-fix/fix-bug.vibe

@@ -0,0 +1,21 @@
+// Bug Fix Script
+// AI diagnoses and fixes bugs - figures out which tools to use on its own
+
+model fixer = {
+  name: "claude-haiku-4-5",
+  apiKey: env("ANTHROPIC_API_KEY"),
+  url: "https://api.anthropic.com",
+  provider: "anthropic"
+}
+
+let code = readFile("buggy-code.ts")
+
+let result: text = vibe "Here's a TypeScript file 'buggy-code.ts':
+
+{code}
+
+A test is failing: sumArray([1, 2, 3, 4, 5]) returns 14 but should return 15.
+
+Fix the bug. After fixing, respond with 'FIXED'." fixer default
+
+result

package/tests/integration/compress/compress.integration.test.ts

@@ -0,0 +1,206 @@
+// Compress Context Mode Integration Test
+// Tests that compress mode calls AI to summarize loop context
+
+import { describe, test, expect } from 'bun:test';
+import { Runtime } from '../../../src/runtime';
+import { createRealAIProvider } from '../../../src/runtime/ai-provider';
+import { parse } from '../../../src/parser/parse';
+import { buildLocalContext, formatContextForAI } from '../../../src/runtime/context';
+
+// Use Google's Gemini Flash for cheap/fast summarization
+const GOOGLE_API_KEY = process.env.GOOGLE_API_KEY;
+const shouldRun = !!GOOGLE_API_KEY;
+
+describe.skipIf(!shouldRun)('Compress Integration', () => {
+  test(
+    'compress summarizes loop iterations',
+    async () => {
+      const program = parse(`
+        model m = {
+          name: "gemini-3-flash-preview",
+          apiKey: "${GOOGLE_API_KEY}",
+          provider: "google"
+        }
+
+        // Simple loop that compress will summarize
+        let results: number[] = []
+        for i in [1, 2, 3, 4, 5] {
+          let squared = ts(i) { return i * i }
+          results.push(squared)
+        } compress
+
+        results
+      `);
+
+      const runtime = new Runtime(
+        program,
+        createRealAIProvider(() => runtime.getState()),
+        { logAiInteractions: true }
+      );
+
+      const result = await runtime.run();
+
+      // Verify the loop executed correctly
+      expect(result).toEqual([1, 4, 9, 16, 25]);
+
+      // Get final context
+      const state = runtime.getState();
+      const context = buildLocalContext(state);
+      const formatted = formatContextForAI(context);
+
+      console.log('\n=== FINAL CONTEXT ===');
+      console.log(formatted.text);
+
+      // Verify there's a summary in the context
+      const summaryEntry = context.find(e => e.kind === 'summary');
+      expect(summaryEntry).toBeDefined();
+      console.log('\n=== SUMMARY ===');
+      console.log((summaryEntry as { text: string }).text);
+
+      // The summary should exist and be non-empty
+      expect((summaryEntry as { text: string }).text.length).toBeGreaterThan(0);
+    },
+    60000
+  );
+
+  test(
+    'compress with custom prompt',
+    async () => {
+      const program = parse(`
+        model m = {
+          name: "gemini-3-flash-preview",
+          apiKey: "${GOOGLE_API_KEY}",
+          provider: "google"
+        }
+
+        let items: text[] = []
+        for word in ["apple", "banana", "cherry"] {
+          let upper: text = ts(word) { return word.toUpperCase() }
+          items.push(upper)
+        } compress("List the fruits that were processed")
+
+        items
+      `);
+
+      const runtime = new Runtime(
+        program,
+        createRealAIProvider(() => runtime.getState()),
+        { logAiInteractions: true }
+      );
+
+      const result = await runtime.run();
+
+      // Verify the loop executed correctly
+      expect(result).toEqual(['APPLE', 'BANANA', 'CHERRY']);
+
+      // Get final context and verify summary
+      const state = runtime.getState();
+      const context = buildLocalContext(state);
+      const summaryEntry = context.find(e => e.kind === 'summary');
+
+      expect(summaryEntry).toBeDefined();
+      const summaryText = (summaryEntry as { text: string }).text.toLowerCase();
+      console.log('\n=== SUMMARY ===');
+      console.log((summaryEntry as { text: string }).text);
+
+      // Summary should mention the fruits
+      expect(
+        summaryText.includes('apple') ||
+        summaryText.includes('banana') ||
+        summaryText.includes('cherry') ||
+        summaryText.includes('fruit')
+      ).toBe(true);
+    },
+    60000
+  );
+
+  test(
+    'compress with explicit model',
+    async () => {
+      const program = parse(`
+        model main = {
+          name: "gemini-3-flash-preview",
+          apiKey: "${GOOGLE_API_KEY}",
+          provider: "google"
+        }
+
+        model summarizer = {
+          name: "gemini-3-flash-preview",
+          apiKey: "${GOOGLE_API_KEY}",
+          provider: "google"
+        }
+
+        let total = 0
+        for n in [10, 20, 30] {
+          total = total + n
+        } compress(summarizer)
+
+        total
+      `);
+
+      const runtime = new Runtime(
+        program,
+        createRealAIProvider(() => runtime.getState()),
+        { logAiInteractions: true }
+      );
+
+      const result = await runtime.run();
+
+      // Verify the loop executed correctly
+      expect(result).toBe(60);
+
+      // Verify summary exists
+      const state = runtime.getState();
+      const context = buildLocalContext(state);
+      const summaryEntry = context.find(e => e.kind === 'summary');
+
+      expect(summaryEntry).toBeDefined();
+      console.log('\n=== SUMMARY ===');
+      console.log((summaryEntry as { text: string }).text);
+    },
+    60000
+  );
+
+  test(
+    'while loop with compress',
+    async () => {
+      const program = parse(`
+        model m = {
+          name: "gemini-3-flash-preview",
+          apiKey: "${GOOGLE_API_KEY}",
+          provider: "google"
+        }
+
+        let count = 0
+        let sum = 0
+        while (count < 4) {
+          count = count + 1
+          sum = sum + count
+        } compress
+
+        sum
+      `);
+
+      const runtime = new Runtime(
+        program,
+        createRealAIProvider(() => runtime.getState()),
+        { logAiInteractions: true }
+      );
+
+      const result = await runtime.run();
+
+      // sum = 1 + 2 + 3 + 4 = 10
+      expect(result).toBe(10);
+
+      // Verify summary exists
+      const state = runtime.getState();
+      const context = buildLocalContext(state);
+      const summaryEntry = context.find(e => e.kind === 'summary');
+
+      expect(summaryEntry).toBeDefined();
+      console.log('\n=== SUMMARY ===');
+      console.log((summaryEntry as { text: string }).text);
+    },
+    60000
+  );
+});