ai-functions 2.1.3 → 2.4.0

package/test/retry-policy-integration.test.ts

@@ -0,0 +1,117 @@
+/**
+ * Integration tests: retry/CB/fallback machinery reads policy data from
+ * `language-models`. Validates that the per-model `ModelPolicy` flows through
+ * the resilience classes correctly.
+ */
+
+import { describe, it, expect, vi } from 'vitest'
+import { RetryPolicy, CircuitBreaker, FallbackChain } from '../src/retry.js'
+import { tiersForModel, modelSupportsTier, modelPolicyFor } from '../src/index.js'
+
+describe('RetryPolicy.forModel', () => {
+  it('uses frontier-provider settings for sonnet', async () => {
+    const policy = RetryPolicy.forModel('sonnet')
+    let attempts = 0
+    const op = vi.fn(async () => {
+      attempts++
+      if (attempts < 2) {
+        const err = new Error('rate limit')
+        ;(err as Error & { status?: number }).status = 429
+        throw err
+      }
+      return 'ok'
+    })
+    const result = await policy.execute(() => op())
+    expect(result).toBe('ok')
+    expect(attempts).toBe(2)
+  })
+
+  it('forModel respects per-call overrides', async () => {
+    const policy = RetryPolicy.forModel('sonnet', { maxRetries: 0 })
+    let attempts = 0
+    const op = async () => {
+      attempts++
+      const err = new Error('rate limit')
+      ;(err as Error & { status?: number }).status = 429
+      throw err
+    }
+    await expect(policy.execute(() => op())).rejects.toThrow()
+    expect(attempts).toBe(1) // No retries
+  })
+})
+
+describe('CircuitBreaker.forModel', () => {
+  it('uses frontier-provider failure threshold for sonnet', async () => {
+    const breaker = CircuitBreaker.forModel('sonnet')
+    // Frontier threshold is 8 — eight failures should still report state
+    // before opening on the 8th.
+    for (let i = 0; i < 7; i++) {
+      try {
+        await breaker.execute(async () => {
+          throw new Error('fail')
+        })
+      } catch {
+        // expected
+      }
+    }
+    // Still closed (threshold is 8)
+    expect(breaker.state).toBe('closed')
+  })
+
+  it('forModel respects per-call overrides', async () => {
+    const breaker = CircuitBreaker.forModel('sonnet', { failureThreshold: 1 })
+    try {
+      await breaker.execute(async () => {
+        throw new Error('fail')
+      })
+    } catch {
+      // expected
+    }
+    expect(breaker.state).toBe('open')
+  })
+})
+
+describe('FallbackChain.forModel', () => {
+  it('builds a chain from policy.fallbackChain plus the primary model id', async () => {
+    const seen: string[] = []
+    const chain = FallbackChain.forModel<string, void>('sonnet', async (modelId) => {
+      seen.push(modelId)
+      throw new Error('fail')
+    })
+    await expect(chain.execute()).rejects.toThrow('All fallback models failed')
+    // First call is the primary alias, rest are fallbacks
+    expect(seen[0]).toBe('anthropic/claude-sonnet-4.5')
+    expect(seen.length).toBeGreaterThan(1)
+  })
+
+  it('returns first successful model result', async () => {
+    let calls = 0
+    const chain = FallbackChain.forModel<string, void>('sonnet', async (modelId) => {
+      calls++
+      if (calls === 1) throw new Error('first fails')
+      return modelId
+    })
+    const result = await chain.execute()
+    expect(calls).toBe(2)
+    expect(result).toBeTruthy()
+  })
+})
+
+describe('tier helpers', () => {
+  it('tiersForModel returns expected tiers', () => {
+    expect(tiersForModel('sonnet')).toContain('immediate')
+    expect(tiersForModel('sonnet')).toContain('batch')
+    expect(tiersForModel('gpt-4o')).toContain('flex')
+  })
+
+  it('modelSupportsTier checks eligibility', () => {
+    expect(modelSupportsTier('gpt-4o', 'flex')).toBe(true)
+    expect(modelSupportsTier('sonnet', 'flex')).toBe(false)
+    expect(modelSupportsTier('sonnet', 'immediate')).toBe(true)
+  })
+
+  it('modelPolicyFor is re-exported', () => {
+    const p = modelPolicyFor('sonnet')
+    expect(p.$type).toBe('ModelPolicy')
+  })
+})

package/test/sandbox-execution.test.ts

@@ -0,0 +1,155 @@
+/**
+ * Verification for routing ALL dynamic code execution through ai-evaluate's
+ * V8-isolate sandbox (Cloudflare Dynamic Workers, Miniflare fallback in Node).
+ *
+ * Two distinct paths are exercised:
+ *
+ *  - Path A — `type: 'code'` is DETERMINISTIC. A `handler` is a direct call; an
+ *    inline `code` body runs in the sandbox. NO model is ever consulted. We spy
+ *    on the model entry points and assert zero calls, and assert identical
+ *    output across repeated calls.
+ *
+ *  - Path B — `generateAndRunCode` is the NON-deterministic generate → run →
+ *    test → return capability. The model AUTHORS the code; we mock that author
+ *    step, but the run + test + return plumbing executes against the REAL
+ *    Miniflare sandbox (no live Worker, no model).
+ *
+ * What is mocked: ONLY the model-author step in Path B (`generateObject` from
+ * `./generate.js`). The sandbox itself (Miniflare) is real. Path A mocks
+ * nothing — it only spies to prove the model is never touched.
+ */
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+
+// Spy targets for Path A: prove no model is consulted on the deterministic path.
+import * as generateModule from '../src/generate.js'
+
+describe('Path A — type:code is deterministic (no model, no network)', () => {
+  let generateObjectSpy: ReturnType<typeof vi.spyOn>
+  let generateTextSpy: ReturnType<typeof vi.spyOn>
+
+  beforeEach(() => {
+    generateObjectSpy = vi.spyOn(generateModule, 'generateObject')
+    generateTextSpy = vi.spyOn(generateModule, 'generateText')
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('handler functions are a direct call — deterministic, no model', async () => {
+    const { defineFunction } = await import('../src/function-registry.js')
+    const calculateTax = defineFunction<number, { amount: number; rate: number }>({
+      type: 'code',
+      name: 'calculateTax',
+      args: { amount: 'Amount (number)', rate: 'Rate (number)' },
+      handler: ({ amount, rate }) => amount * rate,
+    })
+
+    const a = await calculateTax.call({ amount: 100, rate: 0.2 })
+    const b = await calculateTax.call({ amount: 100, rate: 0.2 })
+
+    expect(a).toBe(20)
+    expect(b).toBe(20) // identical across repeated calls
+    expect(generateObjectSpy).not.toHaveBeenCalled()
+    expect(generateTextSpy).not.toHaveBeenCalled()
+  })
+
+  it('inline code bodies run in the sandbox — deterministic, no model', async () => {
+    const { defineFunction } = await import('../src/function-registry.js')
+    const sum = defineFunction<number, { items: number[] }>({
+      type: 'code',
+      name: 'sum',
+      args: { items: ['Numbers'] },
+      language: 'typescript',
+      code: 'return args.items.reduce((a, b) => a + b, 0)',
+    })
+
+    const a = await sum.call({ items: [1, 2, 3, 4] })
+    const b = await sum.call({ items: [1, 2, 3, 4] })
+
+    expect(a).toBe(10)
+    expect(b).toBe(10) // identical across repeated calls — fully deterministic
+    expect(generateObjectSpy).not.toHaveBeenCalled()
+    expect(generateTextSpy).not.toHaveBeenCalled()
+  })
+
+  it('an inline code body that throws surfaces the sandbox error', async () => {
+    const { defineFunction } = await import('../src/function-registry.js')
+    // Use an explicit statement body (contains `return`) so the runtime throw
+    // is reached rather than being mis-wrapped as a `return (expr)`.
+    const boom = defineFunction<number, Record<string, never>>({
+      type: 'code',
+      name: 'boom',
+      args: {},
+      language: 'typescript',
+      code: "if (true) { throw new Error('kaboom') }\nreturn 0",
+    })
+
+    await expect(boom.call({})).rejects.toThrow(/kaboom/)
+    expect(generateObjectSpy).not.toHaveBeenCalled()
+  })
+}, 60000)
+
+// Path B mocks ONLY the model-author step; the run+test runs in real Miniflare.
+vi.mock('../src/generate.js', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('../src/generate.js')>()
+  return { ...actual }
+})
+
+describe('Path B — generateAndRunCode: generate → run → test → return', () => {
+  beforeEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('runs MODEL-AUTHORED code in the real sandbox and returns the computed result', async () => {
+    const gen = await import('../src/generate.js')
+
+    // Mock ONLY the model-author step. Everything downstream (run, test,
+    // return) executes against the REAL Miniflare sandbox.
+    const authoredModule = `export function calculateTax(args) {\n  return args.amount * args.rate;\n}`
+    const authoredTests = `describe('calculateTax', () => {\n  it('multiplies amount by rate', () => {\n    expect(calculateTax({ amount: 100, rate: 0.2 })).toBe(20);\n  });\n});`
+
+    const spy = vi.spyOn(gen, 'generateObject').mockResolvedValue({
+      object: { code: authoredModule, tests: authoredTests },
+    } as Awaited<ReturnType<typeof gen.generateObject>>)
+
+    const { generateAndRunCode } = await import('../src/function-registry.js')
+
+    const result = await generateAndRunCode<number, { amount: number; rate: number }>(
+      {
+        name: 'calculateTax',
+        description: 'Calculate tax owed',
+        args: { amount: '(number)', rate: '(number)' },
+        returnType: '(number)',
+      },
+      { amount: 100, rate: 0.2 }
+    )
+
+    // The model was consulted exactly once (the author step).
+    expect(spy).toHaveBeenCalledTimes(1)
+
+    // The RESULT was actually computed by running the authored code (not just
+    // returned as source).
+    expect(result.value).toBe(20)
+    expect(result.code).toContain('function calculateTax')
+
+    // Tests ran in the same sandbox and passed.
+    expect(result.testResults).toBeDefined()
+    expect(result.testResults!.failed).toBe(0)
+    expect(result.testResults!.passed).toBeGreaterThanOrEqual(1)
+  })
+
+  it('surfaces a sandbox failure when authored code throws at runtime', async () => {
+    const gen = await import('../src/generate.js')
+    vi.spyOn(gen, 'generateObject').mockResolvedValue({
+      object: { code: `export function bad(args) { throw new Error('runtime boom'); }` },
+    } as Awaited<ReturnType<typeof gen.generateObject>>)
+
+    const { generateAndRunCode } = await import('../src/function-registry.js')
+
+    await expect(
+      generateAndRunCode({ name: 'bad', args: { x: '(number)' }, includeTests: false }, { x: 1 })
+    ).rejects.toThrow(/runtime boom/)
+  })
+}, 60000)

package/test/schema.test.ts

@@ -2,6 +2,7 @@
  * Tests for schema conversion
  *
  * These are pure unit tests - no AI calls needed.
+ * Uses type-safe instanceof checks instead of accessing internal _def property.
  */
 
 import { describe, it, expect } from 'vitest'
@@ -12,54 +13,67 @@ describe('schema', () => {
   describe('string types', () => {
     it('converts simple string description to z.string()', () => {
       const result = schema('User name')
-      expect(result._def.typeName).toBe('ZodString')
-      expect(result._def.description).toBe('User name')
+      expect(result instanceof z.ZodString).toBe(true)
+      expect(result.description).toBe('User name')
     })
 
     it('converts (number) hint to z.number()', () => {
       const result = schema('User age (number)')
-      expect(result._def.typeName).toBe('ZodNumber')
-      expect(result._def.description).toBe('User age')
+      expect(result instanceof z.ZodNumber).toBe(true)
+      expect(result.description).toBe('User age')
     })
 
     it('converts (boolean) hint to z.boolean()', () => {
       const result = schema('Is active (boolean)')
-      expect(result._def.typeName).toBe('ZodBoolean')
-      expect(result._def.description).toBe('Is active')
+      expect(result instanceof z.ZodBoolean).toBe(true)
+      expect(result.description).toBe('Is active')
     })
 
     it('converts (integer) hint to z.number().int()', () => {
       const result = schema('Item count (integer)')
-      expect(result._def.typeName).toBe('ZodNumber')
-      expect(result._def.checks?.some((c: { kind: string }) => c.kind === 'int')).toBe(true)
+      expect(result instanceof z.ZodNumber).toBe(true)
+      // Verify it's an integer by testing validation behavior
+      expect(result.safeParse(5).success).toBe(true)
+      expect(result.safeParse(5.5).success).toBe(false)
     })
 
     it('converts (date) hint to z.string().datetime()', () => {
       const result = schema('Created at (date)')
-      expect(result._def.typeName).toBe('ZodString')
-      expect(result._def.checks?.some((c: { kind: string }) => c.kind === 'datetime')).toBe(true)
+      expect(result instanceof z.ZodString).toBe(true)
+      // Verify it validates datetime format
+      expect(result.safeParse('2024-01-15T10:30:00Z').success).toBe(true)
+      expect(result.safeParse('not-a-date').success).toBe(false)
     })
   })
 
   describe('enum types', () => {
     it('converts pipe-separated values to z.enum()', () => {
       const result = schema('pending | done | cancelled')
-      expect(result._def.typeName).toBe('ZodEnum')
-      expect(result._def.values).toEqual(['pending', 'done', 'cancelled'])
+      expect(result instanceof z.ZodEnum).toBe(true)
+      // Verify enum values through validation
+      expect(result.safeParse('pending').success).toBe(true)
+      expect(result.safeParse('done').success).toBe(true)
+      expect(result.safeParse('cancelled').success).toBe(true)
+      expect(result.safeParse('invalid').success).toBe(false)
     })
 
     it('handles spaces around pipe', () => {
       const result = schema('yes | no | maybe')
-      expect(result._def.values).toEqual(['yes', 'no', 'maybe'])
+      expect(result instanceof z.ZodEnum).toBe(true)
+      expect(result.safeParse('yes').success).toBe(true)
+      expect(result.safeParse('no').success).toBe(true)
+      expect(result.safeParse('maybe').success).toBe(true)
     })
   })
 
   describe('array types', () => {
     it('converts [string] to z.array(z.string())', () => {
       const result = schema(['List of items'])
-      expect(result._def.typeName).toBe('ZodArray')
-      expect(result._def.type._def.typeName).toBe('ZodString')
-      expect(result._def.description).toBe('List of items')
+      expect(result instanceof z.ZodArray).toBe(true)
+      expect(result.description).toBe('List of items')
+      // Verify array of strings validation
+      expect(result.safeParse(['a', 'b', 'c']).success).toBe(true)
+      expect(result.safeParse([1, 2, 3]).success).toBe(false)
     })
   })
 
@@ -69,7 +83,10 @@ describe('schema', () => {
         name: 'User name',
         age: 'Age (number)',
       })
-      expect(result._def.typeName).toBe('ZodObject')
+      expect(result instanceof z.ZodObject).toBe(true)
+      // Verify object validation
+      expect(result.safeParse({ name: 'John', age: 30 }).success).toBe(true)
+      expect(result.safeParse({ name: 'John', age: 'thirty' }).success).toBe(false)
     })
 
     it('handles nested objects', () => {
@@ -81,7 +98,16 @@ describe('schema', () => {
           },
         },
       })
-      expect(result._def.typeName).toBe('ZodObject')
+      expect(result instanceof z.ZodObject).toBe(true)
+      // Verify nested object validation
+      expect(
+        result.safeParse({
+          user: {
+            name: 'John',
+            profile: { bio: 'A developer' },
+          },
+        }).success
+      ).toBe(true)
     })
 
     it('handles mixed types in object', () => {
@@ -92,7 +118,17 @@ describe('schema', () => {
         status: 'pending | done',
         tags: ['Tags'],
       })
-      expect(result._def.typeName).toBe('ZodObject')
+      expect(result instanceof z.ZodObject).toBe(true)
+      // Verify mixed type validation
+      expect(
+        result.safeParse({
+          name: 'Test',
+          count: 5,
+          active: true,
+          status: 'pending',
+          tags: ['tag1', 'tag2'],
+        }).success
+      ).toBe(true)
     })
   })