ai-functions 0.3.0 → 0.4.0

package/test/embeddings.test.ts

@@ -0,0 +1,284 @@
+/**
+ * Tests for embedding utilities
+ *
+ * Pure unit tests for utility functions that don't require AI API calls.
+ * Gateway-dependent tests are skipped if no gateway is configured.
+ */
+
+import { describe, it, expect } from 'vitest'
+import {
+  cosineSimilarity,
+  findSimilar,
+  pairwiseSimilarity,
+  clusterBySimilarity,
+  averageEmbeddings,
+  normalizeEmbedding,
+} from '../src/index.js'
+
+describe('cosineSimilarity', () => {
+  it('returns 1 for identical vectors', () => {
+    const a = [1, 0, 0]
+    const b = [1, 0, 0]
+    expect(cosineSimilarity(a, b)).toBeCloseTo(1)
+  })
+
+  it('returns 0 for orthogonal vectors', () => {
+    const a = [1, 0, 0]
+    const b = [0, 1, 0]
+    expect(cosineSimilarity(a, b)).toBeCloseTo(0)
+  })
+
+  it('returns -1 for opposite vectors', () => {
+    const a = [1, 0, 0]
+    const b = [-1, 0, 0]
+    expect(cosineSimilarity(a, b)).toBeCloseTo(-1)
+  })
+
+  it('handles non-normalized vectors', () => {
+    const a = [2, 0, 0]
+    const b = [5, 0, 0]
+    expect(cosineSimilarity(a, b)).toBeCloseTo(1)
+  })
+
+  it('works with higher dimensional vectors', () => {
+    const a = [1, 2, 3, 4, 5]
+    const b = [1, 2, 3, 4, 5]
+    expect(cosineSimilarity(a, b)).toBeCloseTo(1)
+  })
+})
+
+describe('findSimilar', () => {
+  const embeddings = [
+    [1, 0, 0],
+    [0.9, 0.1, 0],
+    [0, 1, 0],
+    [0, 0, 1],
+    [-1, 0, 0],
+  ]
+  const items = ['A', 'B', 'C', 'D', 'E']
+
+  it('finds most similar items', () => {
+    const query = [1, 0, 0]
+    const results = findSimilar(query, embeddings, items, { topK: 3 })
+
+    expect(results).toHaveLength(3)
+    expect(results[0].item).toBe('A') // Exact match
+    expect(results[0].score).toBeCloseTo(1)
+    expect(results[1].item).toBe('B') // Very similar
+  })
+
+  it('respects topK parameter', () => {
+    const query = [1, 0, 0]
+    const results = findSimilar(query, embeddings, items, { topK: 2 })
+
+    expect(results).toHaveLength(2)
+  })
+
+  it('filters by minScore', () => {
+    const query = [1, 0, 0]
+    const results = findSimilar(query, embeddings, items, { minScore: 0.5 })
+
+    // Only A and B should have score >= 0.5
+    expect(results.every(r => r.score >= 0.5)).toBe(true)
+    expect(results).toHaveLength(2)
+  })
+
+  it('returns index in results', () => {
+    const query = [0, 1, 0]
+    const results = findSimilar(query, embeddings, items, { topK: 1 })
+
+    expect(results[0].item).toBe('C')
+    expect(results[0].index).toBe(2)
+  })
+
+  it('handles empty embeddings', () => {
+    const query = [1, 0, 0]
+    const results = findSimilar(query, [], [], { topK: 5 })
+
+    expect(results).toHaveLength(0)
+  })
+})
+
+describe('pairwiseSimilarity', () => {
+  it('creates a symmetric matrix', () => {
+    const embeddings = [
+      [1, 0, 0],
+      [0, 1, 0],
+      [0, 0, 1],
+    ]
+
+    const matrix = pairwiseSimilarity(embeddings)
+
+    expect(matrix).toHaveLength(3)
+    expect(matrix[0]).toHaveLength(3)
+
+    // Check symmetry
+    for (let i = 0; i < 3; i++) {
+      for (let j = 0; j < 3; j++) {
+        expect(matrix[i][j]).toBeCloseTo(matrix[j][i])
+      }
+    }
+  })
+
+  it('has 1s on the diagonal', () => {
+    const embeddings = [
+      [1, 2, 3],
+      [4, 5, 6],
+      [7, 8, 9],
+    ]
+
+    const matrix = pairwiseSimilarity(embeddings)
+
+    for (let i = 0; i < 3; i++) {
+      expect(matrix[i][i]).toBeCloseTo(1)
+    }
+  })
+
+  it('correctly computes similarity for orthogonal vectors', () => {
+    const embeddings = [
+      [1, 0, 0],
+      [0, 1, 0],
+    ]
+
+    const matrix = pairwiseSimilarity(embeddings)
+
+    expect(matrix[0][1]).toBeCloseTo(0)
+    expect(matrix[1][0]).toBeCloseTo(0)
+  })
+})
+
+describe('clusterBySimilarity', () => {
+  it('groups similar items together', () => {
+    const embeddings = [
+      [1, 0, 0],
+      [0.95, 0.05, 0],
+      [0, 1, 0],
+      [0.05, 0.95, 0],
+    ]
+    const items = ['A', 'A-like', 'B', 'B-like']
+
+    const clusters = clusterBySimilarity(embeddings, items, { threshold: 0.9 })
+
+    // Should create 2 clusters
+    expect(clusters).toHaveLength(2)
+
+    // Find the cluster with A
+    const clusterA = clusters.find(c => c.includes('A'))
+    expect(clusterA).toContain('A-like')
+
+    // Find the cluster with B
+    const clusterB = clusters.find(c => c.includes('B'))
+    expect(clusterB).toContain('B-like')
+  })
+
+  it('creates single-item clusters for dissimilar items', () => {
+    const embeddings = [
+      [1, 0, 0],
+      [0, 1, 0],
+      [0, 0, 1],
+    ]
+    const items = ['X', 'Y', 'Z']
+
+    const clusters = clusterBySimilarity(embeddings, items, { threshold: 0.9 })
+
+    // Each item should be in its own cluster
+    expect(clusters).toHaveLength(3)
+    expect(clusters.every(c => c.length === 1)).toBe(true)
+  })
+
+  it('puts all items in one cluster at low threshold', () => {
+    const embeddings = [
+      [1, 0.1, 0.1],
+      [0.9, 0.2, 0.1],
+      [0.8, 0.3, 0.1],
+    ]
+    const items = ['A', 'B', 'C']
+
+    const clusters = clusterBySimilarity(embeddings, items, { threshold: 0.5 })
+
+    // All items similar enough to be in one cluster
+    expect(clusters).toHaveLength(1)
+    expect(clusters[0]).toHaveLength(3)
+  })
+})
+
+describe('averageEmbeddings', () => {
+  it('averages multiple embeddings', () => {
+    const embeddings = [
+      [1, 0, 0],
+      [0, 1, 0],
+      [0, 0, 1],
+    ]
+
+    const avg = averageEmbeddings(embeddings)
+
+    expect(avg).toHaveLength(3)
+    expect(avg[0]).toBeCloseTo(1 / 3)
+    expect(avg[1]).toBeCloseTo(1 / 3)
+    expect(avg[2]).toBeCloseTo(1 / 3)
+  })
+
+  it('returns empty array for empty input', () => {
+    const avg = averageEmbeddings([])
+    expect(avg).toEqual([])
+  })
+
+  it('returns same vector for single input', () => {
+    const embedding = [1, 2, 3]
+    const avg = averageEmbeddings([embedding])
+
+    expect(avg).toEqual([1, 2, 3])
+  })
+})
+
+describe('normalizeEmbedding', () => {
+  it('normalizes a vector to unit length', () => {
+    const embedding = [3, 4, 0] // length = 5
+    const normalized = normalizeEmbedding(embedding)
+
+    expect(normalized[0]).toBeCloseTo(0.6)
+    expect(normalized[1]).toBeCloseTo(0.8)
+    expect(normalized[2]).toBeCloseTo(0)
+
+    // Check magnitude is 1
+    const magnitude = Math.sqrt(
+      normalized.reduce((sum, val) => sum + val * val, 0)
+    )
+    expect(magnitude).toBeCloseTo(1)
+  })
+
+  it('handles already normalized vectors', () => {
+    const embedding = [1, 0, 0]
+    const normalized = normalizeEmbedding(embedding)
+
+    expect(normalized).toEqual([1, 0, 0])
+  })
+
+  it('handles zero vector', () => {
+    const embedding = [0, 0, 0]
+    const normalized = normalizeEmbedding(embedding)
+
+    expect(normalized).toEqual([0, 0, 0])
+  })
+})
+
+// Skip API-dependent tests if no gateway
+const hasGateway = !!process.env.AI_GATEWAY_URL || !!process.env.CF_ACCOUNT_ID
+
+describe.skipIf(!hasGateway)('embedText and embedTexts', () => {
+  // These tests would require actual API calls
+  // They're here as placeholders for when the gateway is available
+
+  it.skip('embeds a single text', async () => {
+    // const { embedText } = await import('../src/index.js')
+    // const { embedding } = await embedText('hello world')
+    // expect(Array.isArray(embedding)).toBe(true)
+    // expect(embedding.length).toBeGreaterThan(0)
+  })
+
+  it.skip('embeds multiple texts', async () => {
+    // const { embedTexts } = await import('../src/index.js')
+    // const { embeddings } = await embedTexts(['doc1', 'doc2', 'doc3'])
+    // expect(embeddings).toHaveLength(3)
+  })
+})

package/test/evals/define-function.eval.test.ts

@@ -0,0 +1,379 @@
+/**
+ * defineFunction Eval Suite
+ *
+ * Tests the defineFunction API with real AI calls across multiple models.
+ * Ensures that defined functions work correctly with generative, agentic, and code types.
+ *
+ * Run with: pnpm test -- test/evals/define-function.eval.test.ts
+ * Run specific model: MODEL=sonnet pnpm test -- test/evals/define-function.eval.test.ts
+ */
+
+import { describe, it, expect, beforeEach } from 'vitest'
+import { define, defineFunction, functions } from '../../src/index.js'
+import { EVAL_MODELS, type EvalModel, type ModelTier } from '../../src/eval/models.js'
+
+// Configuration
+const EVAL_TIERS: ModelTier[] = (process.env.EVAL_TIERS?.split(',') as ModelTier[]) || ['fast']
+const SINGLE_MODEL = process.env.MODEL
+
+// Get models to test
+function getTestModels(): EvalModel[] {
+  if (SINGLE_MODEL) {
+    const model = EVAL_MODELS.find(m => m.id === SINGLE_MODEL || m.name.toLowerCase().includes(SINGLE_MODEL.toLowerCase()))
+    return model ? [model] : EVAL_MODELS.filter(m => m.tier === 'fast').slice(0, 1)
+  }
+  return EVAL_MODELS.filter(m => EVAL_TIERS.includes(m.tier))
+}
+
+// Skip if no API access
+const hasAPI = !!process.env.AI_GATEWAY_URL || !!process.env.ANTHROPIC_API_KEY || !!process.env.OPENAI_API_KEY
+
+// Test timeout for AI calls
+const AI_TIMEOUT = 30000
+
+describe.skipIf(!hasAPI)('defineFunction Evals', () => {
+  const models = getTestModels()
+
+  beforeEach(() => {
+    functions.clear()
+  })
+
+  // ==========================================================================
+  // Generative Functions
+  // ==========================================================================
+  describe('define.generative()', () => {
+    describe('string output functions', () => {
+      for (const model of models) {
+        describe(`[${model.name}]`, () => {
+          it('executes greeting function', async () => {
+            const greet = define.generative({
+              name: 'greet',
+              args: { name: 'Name to greet' },
+              output: 'string',
+              promptTemplate: 'Say hello to {{name}} in a friendly way',
+              model: model.id,
+            })
+
+            const result = await greet.call({ name: 'World' })
+            expect(typeof result).toBe('string')
+            expect((result as string).toLowerCase()).toMatch(/hello|hi|hey|greet/)
+          }, AI_TIMEOUT)
+
+          it('executes translation function', async () => {
+            const translate = define.generative({
+              name: 'translate',
+              args: {
+                text: 'Text to translate',
+                targetLang: 'Target language',
+              },
+              output: 'string',
+              promptTemplate: 'Translate "{{text}}" to {{targetLang}}',
+              model: model.id,
+            })
+
+            const result = await translate.call({ text: 'Hello', targetLang: 'Spanish' })
+            expect(typeof result).toBe('string')
+            expect((result as string).toLowerCase()).toMatch(/hola|buenos/)
+          }, AI_TIMEOUT)
+
+          it('executes summarization function', async () => {
+            const summarizeText = define.generative({
+              name: 'summarizeText',
+              args: { text: 'Text to summarize' },
+              output: 'string',
+              promptTemplate: 'Summarize the following in one sentence: {{text}}',
+              model: model.id,
+            })
+
+            const longText = 'TypeScript is a strongly typed programming language that builds on JavaScript, giving you better tooling at any scale. TypeScript adds additional syntax to JavaScript to support a tighter integration with your editor. Catch errors early in your editor.'
+
+            const result = await summarizeText.call({ text: longText })
+            expect(typeof result).toBe('string')
+            expect((result as string).length).toBeGreaterThan(10)
+            expect((result as string).length).toBeLessThan(longText.length)
+          }, AI_TIMEOUT)
+        })
+      }
+    })
+
+    describe('object output functions', () => {
+      for (const model of models) {
+        describe(`[${model.name}]`, () => {
+          it('executes sentiment analysis function', async () => {
+            const analyzeSentiment = define.generative({
+              name: 'analyzeSentiment',
+              args: { text: 'Text to analyze' },
+              output: 'object',
+              returnType: {
+                sentiment: 'positive | negative | neutral',
+                confidence: 'Confidence score 0-1 (number)',
+                reasoning: 'Brief explanation',
+              },
+              promptTemplate: 'Analyze the sentiment of: {{text}}',
+              model: model.id,
+            })
+
+            const result = await analyzeSentiment.call({ text: 'I absolutely love this product!' }) as {
+              sentiment: string
+              confidence: number
+              reasoning: string
+            }
+
+            expect(result).toBeDefined()
+            expect(['positive', 'negative', 'neutral']).toContain(result.sentiment)
+            expect(typeof result.confidence).toBe('number')
+            expect(result.confidence).toBeGreaterThanOrEqual(0)
+            expect(result.confidence).toBeLessThanOrEqual(1)
+          }, AI_TIMEOUT)
+
+          it('executes entity extraction function', async () => {
+            const extractEntities = define.generative({
+              name: 'extractEntities',
+              args: { text: 'Text to extract entities from' },
+              output: 'object',
+              returnType: {
+                people: ['Names of people mentioned'],
+                organizations: ['Names of organizations mentioned'],
+                locations: ['Location names mentioned'],
+              },
+              promptTemplate: 'Extract named entities from: {{text}}',
+              model: model.id,
+            })
+
+            const result = await extractEntities.call({
+              text: 'John Smith from Google met with Jane Doe at the New York office to discuss the project.',
+            }) as {
+              people: string[]
+              organizations: string[]
+              locations: string[]
+            }
+
+            expect(result).toBeDefined()
+            expect(Array.isArray(result.people)).toBe(true)
+            expect(Array.isArray(result.organizations)).toBe(true)
+            expect(Array.isArray(result.locations)).toBe(true)
+
+            // Should find at least some entities
+            const totalEntities = result.people.length + result.organizations.length + result.locations.length
+            expect(totalEntities).toBeGreaterThanOrEqual(2)
+          }, AI_TIMEOUT)
+
+          it('executes classification function', async () => {
+            const classifyTicket = define.generative({
+              name: 'classifyTicket',
+              args: { ticket: 'Support ticket text' },
+              output: 'object',
+              returnType: {
+                category: 'billing | technical | account | shipping',
+                priority: 'low | medium | high',
+                suggestedAction: 'Brief suggested action',
+              },
+              promptTemplate: 'Classify this support ticket: {{ticket}}',
+              model: model.id,
+            })
+
+            const result = await classifyTicket.call({
+              ticket: 'My credit card was charged twice for the same order and I need a refund immediately!',
+            }) as {
+              category: string
+              priority: string
+              suggestedAction: string
+            }
+
+            expect(result).toBeDefined()
+            expect(['billing', 'technical', 'account', 'shipping']).toContain(result.category)
+            expect(['low', 'medium', 'high']).toContain(result.priority)
+            expect(typeof result.suggestedAction).toBe('string')
+          }, AI_TIMEOUT)
+        })
+      }
+    })
+  })
+
+  // ==========================================================================
+  // Code Generation Functions
+  // ==========================================================================
+  describe('define.code()', () => {
+    for (const model of models) {
+      describe(`[${model.name}]`, () => {
+        it('generates TypeScript function', async () => {
+          const generateTs = define.code({
+            name: 'generateTypeScript',
+            args: { spec: 'Function specification' },
+            language: 'typescript',
+            model: model.id,
+          })
+
+          const result = await generateTs.call({
+            spec: 'A function that checks if a string is a valid URL',
+          })
+
+          expect(typeof result).toBe('string')
+          expect((result as string).length).toBeGreaterThan(20)
+          // Should contain TypeScript/JavaScript-like syntax
+          expect(result as string).toMatch(/function|const|=>|return/)
+        }, AI_TIMEOUT)
+
+        it('generates Python function', async () => {
+          const generatePy = define.code({
+            name: 'generatePython',
+            args: { spec: 'Function specification' },
+            language: 'python',
+            model: model.id,
+          })
+
+          const result = await generatePy.call({
+            spec: 'A function that calculates the factorial of a number',
+          })
+
+          expect(typeof result).toBe('string')
+          expect((result as string).length).toBeGreaterThan(20)
+          // Should contain Python-like syntax
+          expect(result as string).toMatch(/def|return|if|else/)
+        }, AI_TIMEOUT)
+
+        it('generates SQL query', async () => {
+          const generateSql = define.code({
+            name: 'generateSql',
+            args: { spec: 'Query specification' },
+            language: 'sql',
+            model: model.id,
+          })
+
+          const result = await generateSql.call({
+            spec: 'Select all users who signed up in the last 30 days',
+          })
+
+          expect(typeof result).toBe('string')
+          expect((result as string).toUpperCase()).toMatch(/SELECT|FROM|WHERE/)
+        }, AI_TIMEOUT)
+      })
+    }
+  })
+
+  // ==========================================================================
+  // Function Registration and Tool Generation
+  // ==========================================================================
+  describe('function registry and tools', () => {
+    it('registers functions in the registry', () => {
+      const fn = define.generative({
+        name: 'testRegistration',
+        args: { input: 'Test input' },
+        output: 'string',
+      })
+
+      expect(functions.has('testRegistration')).toBe(true)
+      expect(functions.get('testRegistration')).toBeDefined()
+      expect(functions.list()).toContain('testRegistration')
+    })
+
+    it('generates valid tool schema', () => {
+      const fn = define.generative({
+        name: 'processData',
+        description: 'Process input data and return results',
+        args: {
+          data: 'The data to process',
+          format: 'Output format (json | csv | xml)',
+        },
+        output: 'string',
+      })
+
+      const tool = fn.asTool()
+
+      expect(tool.name).toBe('processData')
+      expect(tool.description).toBe('Process input data and return results')
+      expect(tool.parameters.type).toBe('object')
+      expect(tool.parameters.properties).toHaveProperty('data')
+      expect(tool.parameters.properties).toHaveProperty('format')
+      expect(tool.parameters.required).toContain('data')
+      expect(tool.parameters.required).toContain('format')
+    })
+
+    it('clears all registered functions', () => {
+      define.generative({ name: 'fn1', args: {}, output: 'string' })
+      define.generative({ name: 'fn2', args: {}, output: 'string' })
+      define.generative({ name: 'fn3', args: {}, output: 'string' })
+
+      expect(functions.list().length).toBeGreaterThanOrEqual(3)
+
+      functions.clear()
+
+      expect(functions.list()).toEqual([])
+    })
+  })
+
+  // ==========================================================================
+  // Complex Multi-Step Functions
+  // ==========================================================================
+  describe('complex functions', () => {
+    for (const model of models) {
+      describe(`[${model.name}]`, () => {
+        it('executes multi-output analysis function', async () => {
+          const analyzeArticle = define.generative({
+            name: 'analyzeArticle',
+            args: { article: 'Article text to analyze' },
+            output: 'object',
+            returnType: {
+              title: 'Generated title for the article',
+              summary: 'Brief summary (2-3 sentences)',
+              keyPoints: ['Main points from the article'],
+              readingTime: 'Estimated reading time in minutes (number)',
+            },
+            promptTemplate: 'Analyze this article and provide structured output: {{article}}',
+            model: model.id,
+          })
+
+          const article = `
+            Machine learning has revolutionized the tech industry. Companies are using AI to automate tasks,
+            improve customer service, and make better predictions. From healthcare to finance, the applications
+            are endless. However, there are concerns about job displacement and ethical use of AI.
+          `
+
+          const result = await analyzeArticle.call({ article }) as {
+            title: string
+            summary: string
+            keyPoints: string[]
+            readingTime: number
+          }
+
+          expect(result).toBeDefined()
+          expect(typeof result.title).toBe('string')
+          expect(result.title.length).toBeGreaterThan(5)
+          expect(typeof result.summary).toBe('string')
+          expect(result.summary.length).toBeGreaterThan(20)
+          expect(Array.isArray(result.keyPoints)).toBe(true)
+          expect(result.keyPoints.length).toBeGreaterThanOrEqual(1)
+          expect(typeof result.readingTime).toBe('number')
+        }, AI_TIMEOUT)
+
+        it('executes math problem solver', async () => {
+          const solveMath = define.generative({
+            name: 'solveMath',
+            args: { problem: 'Math problem to solve' },
+            output: 'object',
+            returnType: {
+              answer: 'The numeric answer (number)',
+              steps: ['Step by step solution'],
+              formula: 'Mathematical formula used (if applicable)',
+            },
+            promptTemplate: 'Solve this math problem step by step: {{problem}}',
+            model: model.id,
+          })
+
+          const result = await solveMath.call({
+            problem: 'What is 15% of 200?',
+          }) as {
+            answer: number
+            steps: string[]
+            formula: string
+          }
+
+          expect(result).toBeDefined()
+          expect(typeof result.answer).toBe('number')
+          expect(Math.abs(result.answer - 30)).toBeLessThan(1) // Allow small floating point difference
+          expect(Array.isArray(result.steps)).toBe(true)
+        }, AI_TIMEOUT)
+      })
+    }
+  })
+})