ai-functions 2.0.2 → 2.1.3

package/test/blog-generation.test.js

@@ -0,0 +1,390 @@
+/**
+ * Blog Post Generation Live Tests
+ *
+ * These tests run LIVE against real AI providers by default.
+ * They verify the complete blog generation workflow:
+ *
+ * ```ts
+ * const titles = await list`10 blog post titles about ${topic}`
+ * const posts = titles.map(title => write`a blog post starting with "# ${title}"`)
+ * ```
+ *
+ * Tests cover:
+ * - Real API calls to OpenAI, Anthropic, etc.
+ * - Action/event storage in the database
+ * - Both realtime and batch execution modes
+ * - Multiple providers
+ *
+ * Run with:
+ * ```bash
+ * pnpm test blog-generation.live
+ * ```
+ *
+ * Skip live tests (CI without API keys):
+ * ```bash
+ * SKIP_LIVE_TESTS=true pnpm test blog-generation.live
+ * ```
+ *
+ * @packageDocumentation
+ */
+import { describe, it, expect, beforeEach, afterEach, beforeAll } from 'vitest';
+import { configure, resetContext, withContext, getProvider, getModel, } from '../src/context.js';
+import { createBatch } from '../src/batch-queue.js';
+import { generateObject, generateText } from '../src/generate.js';
+// Database provider for action/event storage
+import { createMemoryProvider } from '../../ai-database/src/memory-provider.js';
+// Batch storage
+import '../src/batch/memory.js';
+import { configureMemoryAdapter, clearBatches, getBatches } from '../src/batch/memory.js';
+// ============================================================================
+// Configuration
+// ============================================================================
+const SKIP_LIVE = process.env.SKIP_LIVE_TESTS === 'true';
+const describeLive = SKIP_LIVE ? describe.skip : describe;
+// Detect available providers
+const hasOpenAI = !!process.env.OPENAI_API_KEY;
+const hasAnthropic = !!process.env.ANTHROPIC_API_KEY;
+// Provider configs
+const PROVIDERS = {
+    openai: { model: 'gpt-4o-mini', available: hasOpenAI },
+    anthropic: { model: 'claude-sonnet-4-20250514', available: hasAnthropic },
+};
+// Get first available provider
+const defaultProvider = Object.entries(PROVIDERS).find(([, cfg]) => cfg.available)?.[0] || 'openai';
+const defaultModel = PROVIDERS[defaultProvider]?.model || 'gpt-4o-mini';
+// ============================================================================
+// Database Setup
+// ============================================================================
+let db;
+let capturedEvents = [];
+// ============================================================================
+// Test Helpers
+// ============================================================================
+async function createAction(data) {
+    return db.createAction({
+        actor: 'test:live',
+        action: data.action,
+        object: data.object,
+        objectData: data.objectData,
+        total: data.total,
+    });
+}
+async function generateTitles(topic, count) {
+    const action = await createAction({
+        action: 'generate',
+        object: 'BlogTitles',
+        objectData: { topic, count },
+        total: 1,
+    });
+    await db.updateAction(action.id, { status: 'active' });
+    const result = await generateObject({
+        model: getModel(),
+        schema: { titles: [`${count} blog post titles about ${topic}`] },
+        prompt: `Generate exactly ${count} creative blog post titles about "${topic}".`,
+    });
+    const titles = result.object.titles;
+    await db.updateAction(action.id, {
+        status: 'completed',
+        progress: 1,
+        result: { titles },
+    });
+    return { titles, action };
+}
+async function generatePost(title) {
+    const result = await generateText({
+        model: getModel(),
+        prompt: `Write a blog post starting with "# ${title}"\n\nInclude an introduction, 2-3 sections, and conclusion. Be concise.`,
+        maxTokens: 800,
+    });
+    return result.text;
+}
+async function generatePosts(titles, mode = 'realtime') {
+    const action = await createAction({
+        action: 'generate',
+        object: 'BlogPosts',
+        objectData: { titles, mode },
+        total: titles.length,
+    });
+    await db.updateAction(action.id, { status: 'active' });
+    const posts = [];
+    if (mode === 'batch') {
+        const batch = createBatch({
+            provider: getProvider(),
+            model: getModel(),
+            metadata: { actionId: action.id },
+        });
+        titles.forEach((title, i) => {
+            batch.add(`Write a blog post starting with "# ${title}"\n\nBe concise.`, { customId: `post-${i}`, metadata: { title } });
+        });
+        const { completion } = await batch.submit();
+        const results = await completion;
+        for (const result of results) {
+            posts.push(result.status === 'completed' ? result.result : `[Failed]`);
+            await db.updateAction(action.id, { progress: posts.length });
+        }
+    }
+    else {
+        for (let i = 0; i < titles.length; i++) {
+            const post = await generatePost(titles[i]);
+            posts.push(post);
+            await db.updateAction(action.id, { progress: i + 1 });
+        }
+    }
+    await db.updateAction(action.id, { status: 'completed', result: { count: posts.length } });
+    return { posts, action };
+}
+// ============================================================================
+// Tests
+// ============================================================================
+describeLive('Blog Generation (Live)', () => {
+    beforeAll(() => {
+        console.log('\n🔴 LIVE TEST MODE');
+        console.log(`   Default provider: ${defaultProvider}`);
+        console.log(`   OpenAI available: ${hasOpenAI}`);
+        console.log(`   Anthropic available: ${hasAnthropic}\n`);
+    });
+    beforeEach(() => {
+        resetContext();
+        db = createMemoryProvider();
+        capturedEvents = [];
+        db.on('*', (e) => capturedEvents.push(e));
+        configure({ provider: defaultProvider, model: defaultModel, batchMode: 'immediate' });
+        clearBatches();
+        configureMemoryAdapter({});
+    });
+    afterEach(() => {
+        resetContext();
+        clearBatches();
+        db.clear();
+    });
+    // ==========================================================================
+    // Core Pattern Tests
+    // ==========================================================================
+    describe('Core Pattern: list → write', () => {
+        it('generates titles from a topic', async () => {
+            const { titles, action } = await generateTitles('building AI products', 3);
+            expect(titles).toHaveLength(3);
+            titles.forEach((t) => expect(typeof t).toBe('string'));
+            // Verify action tracking
+            expect(action.status).toBe('completed');
+            expect(action.result).toEqual({ titles });
+        }, 30000);
+        it('generates a blog post from a title', async () => {
+            const title = 'The Future of AI Development';
+            const post = await generatePost(title);
+            expect(post).toContain(`# ${title}`);
+            expect(post.length).toBeGreaterThan(200);
+        }, 30000);
+        it('generates multiple posts from titles', async () => {
+            const { titles } = await generateTitles('startup growth', 2);
+            const { posts, action } = await generatePosts(titles);
+            expect(posts).toHaveLength(2);
+            posts.forEach((post, i) => {
+                expect(post).toContain(`# ${titles[i]}`);
+            });
+            expect(action.progress).toBe(2);
+            expect(action.total).toBe(2);
+        }, 60000);
+    });
+    // ==========================================================================
+    // Action & Event Storage
+    // ==========================================================================
+    describe('Action & Event Storage', () => {
+        it('creates actions with verb conjugation', async () => {
+            const { action } = await generateTitles('test topic', 2);
+            expect(action.action).toBe('generate');
+            expect(action.act).toBe('generates');
+            expect(action.activity).toBe('generating');
+            expect(action.actor).toBe('test:live');
+        }, 30000);
+        it('tracks action lifecycle timestamps', async () => {
+            const { action } = await generateTitles('test', 2);
+            const final = await db.getAction(action.id);
+            expect(final?.createdAt).toBeInstanceOf(Date);
+            expect(final?.startedAt).toBeInstanceOf(Date);
+            expect(final?.completedAt).toBeInstanceOf(Date);
+            expect(final?.startedAt.getTime()).toBeGreaterThanOrEqual(final?.createdAt.getTime());
+            expect(final?.completedAt.getTime()).toBeGreaterThanOrEqual(final?.startedAt.getTime());
+        }, 30000);
+        it('emits events for state transitions', async () => {
+            await generateTitles('test', 2);
+            const actionEvents = capturedEvents.filter((e) => e.event.startsWith('Action.'));
+            const eventTypes = actionEvents.map((e) => e.event);
+            expect(eventTypes).toContain('Action.created');
+            expect(eventTypes).toContain('Action.started');
+            expect(eventTypes).toContain('Action.completed');
+        }, 30000);
+        it('stores action result data', async () => {
+            const { titles, action } = await generateTitles('AI development', 3);
+            const stored = await db.getAction(action.id);
+            expect(stored?.result).toEqual({ titles });
+            expect(stored?.objectData).toEqual({ topic: 'AI development', count: 3 });
+        }, 30000);
+        it('queries actions by status', async () => {
+            await generateTitles('topic 1', 2);
+            await generateTitles('topic 2', 2);
+            const completed = await db.listActions({ status: 'completed' });
+            expect(completed.length).toBe(2);
+        }, 60000);
+        it('queries events by type pattern', async () => {
+            await generateTitles('test', 2);
+            const created = await db.listEvents({ event: 'Action.created' });
+            const allAction = await db.listEvents({ event: 'Action.*' });
+            expect(created.length).toBeGreaterThanOrEqual(1);
+            expect(allAction.length).toBeGreaterThanOrEqual(3); // created, started, completed
+        }, 30000);
+    });
+    // ==========================================================================
+    // Realtime Execution
+    // ==========================================================================
+    describe('Realtime Execution', () => {
+        beforeEach(() => {
+            configure({ batchMode: 'immediate' });
+        });
+        it('executes requests immediately', async () => {
+            const start = Date.now();
+            const { titles } = await generateTitles('quick test', 2);
+            const elapsed = Date.now() - start;
+            expect(titles).toHaveLength(2);
+            // Should complete in reasonable time (not batched/delayed)
+            expect(elapsed).toBeLessThan(30000);
+        }, 30000);
+        it('tracks progress during sequential generation', async () => {
+            const titles = ['Post One', 'Post Two'];
+            const { posts, action } = await generatePosts(titles, 'realtime');
+            expect(posts).toHaveLength(2);
+            const final = await db.getAction(action.id);
+            expect(final?.progress).toBe(2);
+            expect(final?.total).toBe(2);
+        }, 60000);
+    });
+    // ==========================================================================
+    // Batch Execution
+    // ==========================================================================
+    describe('Batch Execution', () => {
+        beforeEach(() => {
+            configure({ batchMode: 'deferred' });
+        });
+        it('creates and submits batch jobs', async () => {
+            const titles = ['Batch Post 1', 'Batch Post 2'];
+            const { posts, action } = await generatePosts(titles, 'batch');
+            expect(posts).toHaveLength(2);
+            // Verify batch was stored
+            const batches = getBatches();
+            expect(batches.size).toBe(1);
+            const [, batch] = [...batches.entries()][0];
+            expect(batch.items).toHaveLength(2);
+            expect(batch.status).toBe('completed');
+        }, 90000);
+        it('stores batch metadata', async () => {
+            const batch = createBatch({
+                provider: getProvider(),
+                model: getModel(),
+                metadata: { task: 'test-batch', timestamp: Date.now() },
+            });
+            batch.add('Write a test post', { customId: 'test-1' });
+            const { job } = await batch.submit();
+            await batch.wait();
+            const stored = getBatches().get(job.id);
+            expect(stored?.options.metadata?.task).toBe('test-batch');
+        }, 30000);
+    });
+    // ==========================================================================
+    // Multi-Provider Tests
+    // ==========================================================================
+    describe('Multi-Provider', () => {
+        it.skipIf(!hasOpenAI)('generates with OpenAI', async () => {
+            configure({ provider: 'openai', model: 'gpt-4o-mini' });
+            const { titles } = await generateTitles('OpenAI test', 2);
+            expect(titles).toHaveLength(2);
+            expect(getProvider()).toBe('openai');
+        }, 30000);
+        it.skipIf(!hasAnthropic)('generates with Anthropic', async () => {
+            configure({ provider: 'anthropic', model: 'claude-sonnet-4-20250514' });
+            const { titles } = await generateTitles('Anthropic test', 2);
+            expect(titles).toHaveLength(2);
+            expect(getProvider()).toBe('anthropic');
+        }, 30000);
+        it.skipIf(!hasOpenAI || !hasAnthropic)('switches providers mid-workflow', async () => {
+            // Generate titles with OpenAI
+            configure({ provider: 'openai', model: 'gpt-4o-mini' });
+            const { titles } = await generateTitles('cross-provider test', 2);
+            // Generate posts with Anthropic
+            const posts = await withContext({ provider: 'anthropic', model: 'claude-sonnet-4-20250514' }, async () => {
+                expect(getProvider()).toBe('anthropic');
+                return Promise.all(titles.slice(0, 1).map(generatePost));
+            });
+            expect(posts).toHaveLength(1);
+            expect(getProvider()).toBe('openai'); // restored
+        }, 60000);
+        it.skipIf(!hasOpenAI || !hasAnthropic)('runs providers in parallel', async () => {
+            const [openaiResult, anthropicResult] = await Promise.all([
+                withContext({ provider: 'openai', model: 'gpt-4o-mini' }, () => generateTitles('OpenAI parallel', 2)),
+                withContext({ provider: 'anthropic', model: 'claude-sonnet-4-20250514' }, () => generateTitles('Anthropic parallel', 2)),
+            ]);
+            expect(openaiResult.titles).toHaveLength(2);
+            expect(anthropicResult.titles).toHaveLength(2);
+        }, 60000);
+    });
+    // ==========================================================================
+    // Error Handling
+    // ==========================================================================
+    describe('Error Handling', () => {
+        it('tracks failed actions', async () => {
+            const action = await createAction({
+                action: 'generate',
+                object: 'FailTest',
+                total: 1,
+            });
+            await db.updateAction(action.id, { status: 'active' });
+            // Simulate failure
+            await db.updateAction(action.id, {
+                status: 'failed',
+                error: 'Simulated failure',
+            });
+            const failed = await db.listActions({ status: 'failed' });
+            expect(failed).toHaveLength(1);
+            expect(failed[0].error).toBe('Simulated failure');
+            // Verify failure event
+            const failEvents = capturedEvents.filter((e) => e.event === 'Action.failed');
+            expect(failEvents.length).toBe(1);
+        });
+        it('handles invalid model gracefully', async () => {
+            configure({ model: 'invalid-model-xyz' });
+            const action = await createAction({
+                action: 'generate',
+                object: 'InvalidModelTest',
+                total: 1,
+            });
+            await db.updateAction(action.id, { status: 'active' });
+            try {
+                await generateObject({
+                    model: 'invalid-model-xyz',
+                    schema: { test: 'test' },
+                    prompt: 'test',
+                });
+            }
+            catch (e) {
+                await db.updateAction(action.id, {
+                    status: 'failed',
+                    error: e instanceof Error ? e.message : 'Unknown error',
+                });
+            }
+            const final = await db.getAction(action.id);
+            expect(final?.status).toBe('failed');
+            expect(final?.error).toBeDefined();
+        }, 30000);
+    });
+    // ==========================================================================
+    // Database Statistics
+    // ==========================================================================
+    describe('Database Statistics', () => {
+        it('tracks aggregate stats', async () => {
+            await generateTitles('stats test 1', 2);
+            await generateTitles('stats test 2', 2);
+            const stats = db.stats();
+            expect(stats.actions.completed).toBe(2);
+            expect(stats.events).toBeGreaterThan(0);
+        }, 60000);
+    });
+});