ai-functions 0.3.0 → 0.4.0

package/test/blog-generation.test.ts

@@ -0,0 +1,510 @@
+/**
+ * Blog Post Generation Live Tests
+ *
+ * These tests run LIVE against real AI providers by default.
+ * They verify the complete blog generation workflow:
+ *
+ * ```ts
+ * const titles = await list`10 blog post titles about ${topic}`
+ * const posts = titles.map(title => write`a blog post starting with "# ${title}"`)
+ * ```
+ *
+ * Tests cover:
+ * - Real API calls to OpenAI, Anthropic, etc.
+ * - Action/event storage in the database
+ * - Both realtime and batch execution modes
+ * - Multiple providers
+ *
+ * Run with:
+ * ```bash
+ * pnpm test blog-generation.live
+ * ```
+ *
+ * Skip live tests (CI without API keys):
+ * ```bash
+ * SKIP_LIVE_TESTS=true pnpm test blog-generation.live
+ * ```
+ *
+ * @packageDocumentation
+ */
+
+import { describe, it, expect, beforeEach, afterEach, beforeAll } from 'vitest'
+import {
+  configure,
+  resetContext,
+  withContext,
+  getProvider,
+  getModel,
+  getBatchMode,
+} from '../src/context.js'
+import { createBatch, withBatch, type BatchProvider } from '../src/batch-queue.js'
+import { generateObject, generateText } from '../src/generate.js'
+
+// Database provider for action/event storage
+import { createMemoryProvider, type MemoryProvider, type Action, type Event } from '../../ai-database/src/memory-provider.js'
+
+// Batch storage
+import '../src/batch/memory.js'
+import { configureMemoryAdapter, clearBatches, getBatches } from '../src/batch/memory.js'
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+const SKIP_LIVE = process.env.SKIP_LIVE_TESTS === 'true'
+const describeLive = SKIP_LIVE ? describe.skip : describe
+
+// Detect available providers
+const hasOpenAI = !!process.env.OPENAI_API_KEY
+const hasAnthropic = !!process.env.ANTHROPIC_API_KEY
+
+// Provider configs
+const PROVIDERS: Record<string, { model: string; available: boolean }> = {
+  openai: { model: 'gpt-4o-mini', available: hasOpenAI },
+  anthropic: { model: 'claude-sonnet-4-20250514', available: hasAnthropic },
+}
+
+// Get first available provider
+const defaultProvider = Object.entries(PROVIDERS).find(([, cfg]) => cfg.available)?.[0] || 'openai'
+const defaultModel = PROVIDERS[defaultProvider]?.model || 'gpt-4o-mini'
+
+// ============================================================================
+// Database Setup
+// ============================================================================
+
+let db: MemoryProvider
+let capturedEvents: Event[] = []
+
+// ============================================================================
+// Test Helpers
+// ============================================================================
+
+async function createAction(data: {
+  action: string
+  object: string
+  objectData?: Record<string, unknown>
+  total?: number
+}): Promise<Action> {
+  return db.createAction({
+    actor: 'test:live',
+    action: data.action,
+    object: data.object,
+    objectData: data.objectData,
+    total: data.total,
+  })
+}
+
+async function generateTitles(topic: string, count: number): Promise<{ titles: string[]; action: Action }> {
+  const action = await createAction({
+    action: 'generate',
+    object: 'BlogTitles',
+    objectData: { topic, count },
+    total: 1,
+  })
+
+  await db.updateAction(action.id, { status: 'active' })
+
+  const result = await generateObject({
+    model: getModel(),
+    schema: { titles: [`${count} blog post titles about ${topic}`] },
+    prompt: `Generate exactly ${count} creative blog post titles about "${topic}".`,
+  })
+
+  const titles = (result.object as { titles: string[] }).titles
+
+  await db.updateAction(action.id, {
+    status: 'completed',
+    progress: 1,
+    result: { titles },
+  })
+
+  return { titles, action }
+}
+
+async function generatePost(title: string): Promise<string> {
+  const result = await generateText({
+    model: getModel(),
+    prompt: `Write a blog post starting with "# ${title}"\n\nInclude an introduction, 2-3 sections, and conclusion. Be concise.`,
+    maxTokens: 800,
+  })
+  return result.text
+}
+
+async function generatePosts(
+  titles: string[],
+  mode: 'realtime' | 'batch' = 'realtime'
+): Promise<{ posts: string[]; action: Action }> {
+  const action = await createAction({
+    action: 'generate',
+    object: 'BlogPosts',
+    objectData: { titles, mode },
+    total: titles.length,
+  })
+
+  await db.updateAction(action.id, { status: 'active' })
+
+  const posts: string[] = []
+
+  if (mode === 'batch') {
+    const batch = createBatch({
+      provider: getProvider(),
+      model: getModel(),
+      metadata: { actionId: action.id },
+    })
+
+    titles.forEach((title, i) => {
+      batch.add(
+        `Write a blog post starting with "# ${title}"\n\nBe concise.`,
+        { customId: `post-${i}`, metadata: { title } }
+      )
+    })
+
+    const { completion } = await batch.submit()
+    const results = await completion
+
+    for (const result of results) {
+      posts.push(result.status === 'completed' ? (result.result as string) : `[Failed]`)
+      await db.updateAction(action.id, { progress: posts.length })
+    }
+  } else {
+    for (let i = 0; i < titles.length; i++) {
+      const post = await generatePost(titles[i])
+      posts.push(post)
+      await db.updateAction(action.id, { progress: i + 1 })
+    }
+  }
+
+  await db.updateAction(action.id, { status: 'completed', result: { count: posts.length } })
+
+  return { posts, action }
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+describeLive('Blog Generation (Live)', () => {
+  beforeAll(() => {
+    console.log('\n🔴 LIVE TEST MODE')
+    console.log(`   Default provider: ${defaultProvider}`)
+    console.log(`   OpenAI available: ${hasOpenAI}`)
+    console.log(`   Anthropic available: ${hasAnthropic}\n`)
+  })
+
+  beforeEach(() => {
+    resetContext()
+    db = createMemoryProvider()
+    capturedEvents = []
+    db.on('*', (e) => capturedEvents.push(e))
+    configure({ provider: defaultProvider as BatchProvider, model: defaultModel, batchMode: 'immediate' })
+    clearBatches()
+    configureMemoryAdapter({})
+  })
+
+  afterEach(() => {
+    resetContext()
+    clearBatches()
+    db.clear()
+  })
+
+  // ==========================================================================
+  // Core Pattern Tests
+  // ==========================================================================
+
+  describe('Core Pattern: list → write', () => {
+    it('generates titles from a topic', async () => {
+      const { titles, action } = await generateTitles('building AI products', 3)
+
+      expect(titles).toHaveLength(3)
+      titles.forEach((t) => expect(typeof t).toBe('string'))
+
+      // Verify action tracking
+      expect(action.status).toBe('completed')
+      expect(action.result).toEqual({ titles })
+    }, 30000)
+
+    it('generates a blog post from a title', async () => {
+      const title = 'The Future of AI Development'
+      const post = await generatePost(title)
+
+      expect(post).toContain(`# ${title}`)
+      expect(post.length).toBeGreaterThan(200)
+    }, 30000)
+
+    it('generates multiple posts from titles', async () => {
+      const { titles } = await generateTitles('startup growth', 2)
+      const { posts, action } = await generatePosts(titles)
+
+      expect(posts).toHaveLength(2)
+      posts.forEach((post, i) => {
+        expect(post).toContain(`# ${titles[i]}`)
+      })
+
+      expect(action.progress).toBe(2)
+      expect(action.total).toBe(2)
+    }, 60000)
+  })
+
+  // ==========================================================================
+  // Action & Event Storage
+  // ==========================================================================
+
+  describe('Action & Event Storage', () => {
+    it('creates actions with verb conjugation', async () => {
+      const { action } = await generateTitles('test topic', 2)
+
+      expect(action.action).toBe('generate')
+      expect(action.act).toBe('generates')
+      expect(action.activity).toBe('generating')
+      expect(action.actor).toBe('test:live')
+    }, 30000)
+
+    it('tracks action lifecycle timestamps', async () => {
+      const { action } = await generateTitles('test', 2)
+
+      const final = await db.getAction(action.id)
+      expect(final?.createdAt).toBeInstanceOf(Date)
+      expect(final?.startedAt).toBeInstanceOf(Date)
+      expect(final?.completedAt).toBeInstanceOf(Date)
+      expect(final?.startedAt!.getTime()).toBeGreaterThanOrEqual(final?.createdAt!.getTime()!)
+      expect(final?.completedAt!.getTime()).toBeGreaterThanOrEqual(final?.startedAt!.getTime()!)
+    }, 30000)
+
+    it('emits events for state transitions', async () => {
+      await generateTitles('test', 2)
+
+      const actionEvents = capturedEvents.filter((e) => e.event.startsWith('Action.'))
+      const eventTypes = actionEvents.map((e) => e.event)
+
+      expect(eventTypes).toContain('Action.created')
+      expect(eventTypes).toContain('Action.started')
+      expect(eventTypes).toContain('Action.completed')
+    }, 30000)
+
+    it('stores action result data', async () => {
+      const { titles, action } = await generateTitles('AI development', 3)
+
+      const stored = await db.getAction(action.id)
+      expect(stored?.result).toEqual({ titles })
+      expect(stored?.objectData).toEqual({ topic: 'AI development', count: 3 })
+    }, 30000)
+
+    it('queries actions by status', async () => {
+      await generateTitles('topic 1', 2)
+      await generateTitles('topic 2', 2)
+
+      const completed = await db.listActions({ status: 'completed' })
+      expect(completed.length).toBe(2)
+    }, 60000)
+
+    it('queries events by type pattern', async () => {
+      await generateTitles('test', 2)
+
+      const created = await db.listEvents({ event: 'Action.created' })
+      const allAction = await db.listEvents({ event: 'Action.*' })
+
+      expect(created.length).toBeGreaterThanOrEqual(1)
+      expect(allAction.length).toBeGreaterThanOrEqual(3) // created, started, completed
+    }, 30000)
+  })
+
+  // ==========================================================================
+  // Realtime Execution
+  // ==========================================================================
+
+  describe('Realtime Execution', () => {
+    beforeEach(() => {
+      configure({ batchMode: 'immediate' })
+    })
+
+    it('executes requests immediately', async () => {
+      const start = Date.now()
+      const { titles } = await generateTitles('quick test', 2)
+      const elapsed = Date.now() - start
+
+      expect(titles).toHaveLength(2)
+      // Should complete in reasonable time (not batched/delayed)
+      expect(elapsed).toBeLessThan(30000)
+    }, 30000)
+
+    it('tracks progress during sequential generation', async () => {
+      const titles = ['Post One', 'Post Two']
+      const { posts, action } = await generatePosts(titles, 'realtime')
+
+      expect(posts).toHaveLength(2)
+
+      const final = await db.getAction(action.id)
+      expect(final?.progress).toBe(2)
+      expect(final?.total).toBe(2)
+    }, 60000)
+  })
+
+  // ==========================================================================
+  // Batch Execution
+  // ==========================================================================
+
+  describe('Batch Execution', () => {
+    beforeEach(() => {
+      configure({ batchMode: 'deferred' })
+    })
+
+    it('creates and submits batch jobs', async () => {
+      const titles = ['Batch Post 1', 'Batch Post 2']
+      const { posts, action } = await generatePosts(titles, 'batch')
+
+      expect(posts).toHaveLength(2)
+
+      // Verify batch was stored
+      const batches = getBatches()
+      expect(batches.size).toBe(1)
+
+      const [, batch] = [...batches.entries()][0]
+      expect(batch.items).toHaveLength(2)
+      expect(batch.status).toBe('completed')
+    }, 90000)
+
+    it('stores batch metadata', async () => {
+      const batch = createBatch({
+        provider: getProvider(),
+        model: getModel(),
+        metadata: { task: 'test-batch', timestamp: Date.now() },
+      })
+
+      batch.add('Write a test post', { customId: 'test-1' })
+      const { job } = await batch.submit()
+      await batch.wait()
+
+      const stored = getBatches().get(job.id)
+      expect(stored?.options.metadata?.task).toBe('test-batch')
+    }, 30000)
+  })
+
+  // ==========================================================================
+  // Multi-Provider Tests
+  // ==========================================================================
+
+  describe('Multi-Provider', () => {
+    it.skipIf(!hasOpenAI)('generates with OpenAI', async () => {
+      configure({ provider: 'openai', model: 'gpt-4o-mini' })
+
+      const { titles } = await generateTitles('OpenAI test', 2)
+      expect(titles).toHaveLength(2)
+      expect(getProvider()).toBe('openai')
+    }, 30000)
+
+    it.skipIf(!hasAnthropic)('generates with Anthropic', async () => {
+      configure({ provider: 'anthropic', model: 'claude-sonnet-4-20250514' })
+
+      const { titles } = await generateTitles('Anthropic test', 2)
+      expect(titles).toHaveLength(2)
+      expect(getProvider()).toBe('anthropic')
+    }, 30000)
+
+    it.skipIf(!hasOpenAI || !hasAnthropic)('switches providers mid-workflow', async () => {
+      // Generate titles with OpenAI
+      configure({ provider: 'openai', model: 'gpt-4o-mini' })
+      const { titles } = await generateTitles('cross-provider test', 2)
+
+      // Generate posts with Anthropic
+      const posts = await withContext(
+        { provider: 'anthropic', model: 'claude-sonnet-4-20250514' },
+        async () => {
+          expect(getProvider()).toBe('anthropic')
+          return Promise.all(titles.slice(0, 1).map(generatePost))
+        }
+      )
+
+      expect(posts).toHaveLength(1)
+      expect(getProvider()).toBe('openai') // restored
+    }, 60000)
+
+    it.skipIf(!hasOpenAI || !hasAnthropic)('runs providers in parallel', async () => {
+      const [openaiResult, anthropicResult] = await Promise.all([
+        withContext({ provider: 'openai', model: 'gpt-4o-mini' }, () =>
+          generateTitles('OpenAI parallel', 2)
+        ),
+        withContext({ provider: 'anthropic', model: 'claude-sonnet-4-20250514' }, () =>
+          generateTitles('Anthropic parallel', 2)
+        ),
+      ])
+
+      expect(openaiResult.titles).toHaveLength(2)
+      expect(anthropicResult.titles).toHaveLength(2)
+    }, 60000)
+  })
+
+  // ==========================================================================
+  // Error Handling
+  // ==========================================================================
+
+  describe('Error Handling', () => {
+    it('tracks failed actions', async () => {
+      const action = await createAction({
+        action: 'generate',
+        object: 'FailTest',
+        total: 1,
+      })
+
+      await db.updateAction(action.id, { status: 'active' })
+
+      // Simulate failure
+      await db.updateAction(action.id, {
+        status: 'failed',
+        error: 'Simulated failure',
+      })
+
+      const failed = await db.listActions({ status: 'failed' })
+      expect(failed).toHaveLength(1)
+      expect(failed[0].error).toBe('Simulated failure')
+
+      // Verify failure event
+      const failEvents = capturedEvents.filter((e) => e.event === 'Action.failed')
+      expect(failEvents.length).toBe(1)
+    })
+
+    it('handles invalid model gracefully', async () => {
+      configure({ model: 'invalid-model-xyz' })
+
+      const action = await createAction({
+        action: 'generate',
+        object: 'InvalidModelTest',
+        total: 1,
+      })
+
+      await db.updateAction(action.id, { status: 'active' })
+
+      try {
+        await generateObject({
+          model: 'invalid-model-xyz',
+          schema: { test: 'test' },
+          prompt: 'test',
+        })
+      } catch (e) {
+        await db.updateAction(action.id, {
+          status: 'failed',
+          error: e instanceof Error ? e.message : 'Unknown error',
+        })
+      }
+
+      const final = await db.getAction(action.id)
+      expect(final?.status).toBe('failed')
+      expect(final?.error).toBeDefined()
+    }, 30000)
+  })
+
+  // ==========================================================================
+  // Database Statistics
+  // ==========================================================================
+
+  describe('Database Statistics', () => {
+    it('tracks aggregate stats', async () => {
+      await generateTitles('stats test 1', 2)
+      await generateTitles('stats test 2', 2)
+
+      const stats = db.stats()
+
+      expect(stats.actions.completed).toBe(2)
+      expect(stats.events).toBeGreaterThan(0)
+    }, 60000)
+  })
+})