@elevasis/core 0.1.0

package/src/execution/engine/llm/adapters/__tests__/google.integration.test.ts

@@ -0,0 +1,376 @@
+/**
+ * Google Adapter Integration Test
+ *
+ * Purpose: Verify Google adapter works with real API calls
+ * Pattern: External API Integration (Pattern 4)
+ * Runs in CI: NO (requires GOOGLE_AI_STUDIO_KEY)
+ *
+ * Prerequisites:
+ * - GOOGLE_AI_STUDIO_KEY in .env.development
+ *
+ * Run locally:
+ *   cd packages/core
+ *   pnpm vitest run google.integration
+ *
+ * Note: Tests use models defined in GoogleModel type (model-info.ts)
+ *
+ * Token Limits: Using 8000 tokens (recommended for gemini-3-flash-preview)
+ * Gemini 3 uses internal "thinking" tokens that consume output budget,
+ * so we need generous limits to avoid truncated JSON responses.
+ *
+ * Retry Logic: Tests use retry(3) due to API instability with the preview model.
+ * The Gemini 3 Flash preview API can occasionally return truncated JSON.
+ */
+
+import { describe, it, expect, beforeAll } from 'vitest'
+import { createLLMAdapter } from '../server/adapter-factory'
+import type { ModelConfig } from '../../model-info'
+import type { LLMGenerateRequest } from '../../types'
+
+// Skip if GOOGLE_AI_STUDIO_KEY not available (CI environments)
+const SKIP_TESTS = !process.env.GOOGLE_AI_STUDIO_KEY
+
+describe.skipIf(SKIP_TESTS)('Google Adapter Integration', () => {
+  let apiKey: string
+
+  beforeAll(() => {
+    apiKey = process.env.GOOGLE_AI_STUDIO_KEY!
+  })
+
+  const createRequest = (prompt: string): LLMGenerateRequest => ({
+    messages: [
+      {
+        role: 'system',
+        content: 'You are a helpful assistant. Always respond with valid JSON matching the schema.'
+      },
+      { role: 'user', content: prompt }
+    ],
+    responseSchema: {
+      type: 'object',
+      properties: {
+        answer: { type: 'string' },
+        confidence: { type: 'number' }
+      },
+      required: ['answer', 'confidence'],
+      additionalProperties: false
+    },
+    maxOutputTokens: 8000
+  })
+
+  describe('Gemini 3 Flash via Google AI Studio', () => {
+    it('generates structured output with accurate token counts', { timeout: 60000, retry: 3 }, async () => {
+      const config: ModelConfig = {
+        provider: 'google',
+        model: 'gemini-3-flash-preview',
+        apiKey,
+        temperature: 1, // Gemini 3 requires temperature 1
+        maxOutputTokens: 8000
+      }
+
+      const adapter = createLLMAdapter(config)
+      const request = createRequest('What is the capital of France? Be concise.')
+
+      const response = await adapter.generate<{ answer: string; confidence: number }>(request)
+
+      // Verify structured output
+      expect(response.output).toBeDefined()
+      expect(response.output.answer).toBeDefined()
+      expect(typeof response.output.answer).toBe('string')
+      expect(response.output.answer.toLowerCase()).toContain('paris')
+      expect(typeof response.output.confidence).toBe('number')
+
+      // Verify usage metadata
+      expect(response.usage).toBeDefined()
+      expect(response.usage?.inputTokens).toBeGreaterThan(0)
+      expect(response.usage?.outputTokens).toBeGreaterThan(0)
+      // Note: totalTokens may include thinking tokens, so it can be >= input + output
+      expect(response.usage?.totalTokens).toBeGreaterThanOrEqual(
+        (response.usage?.inputTokens || 0) + (response.usage?.outputTokens || 0)
+      )
+    })
+
+    it('works with thinking level option', { timeout: 60000, retry: 3 }, async () => {
+      const config: ModelConfig = {
+        provider: 'google',
+        model: 'gemini-3-flash-preview',
+        apiKey,
+        temperature: 1,
+        maxOutputTokens: 8000,
+        modelOptions: {
+          thinkingLevel: 'low' // Use low for faster test execution
+        }
+      }
+
+      const adapter = createLLMAdapter(config)
+      const request = createRequest('What is 2 + 2?')
+
+      const response = await adapter.generate<{ answer: string; confidence: number }>(request)
+
+      expect(response.output).toBeDefined()
+      expect(response.output.answer).toBeDefined()
+      // Should contain "4" in some form
+      expect(response.output.answer).toMatch(/4|four/i)
+    })
+
+    it('handles schema with const keyword (z.literal() from Zod)', { timeout: 60000, retry: 3 }, async () => {
+      // This test verifies the fix for Gemini schema compatibility
+      // Gemini doesn't support "const" keyword, so we transform it to "enum" + "type: string"
+      const config: ModelConfig = {
+        provider: 'google',
+        model: 'gemini-3-flash-preview',
+        apiKey,
+        temperature: 1,
+        maxOutputTokens: 8000
+      }
+
+      const adapter = createLLMAdapter(config)
+
+      // Schema that mimics agent action output with const (from z.literal())
+      // This is the pattern that was previously failing with Gemini
+      const request: LLMGenerateRequest = {
+        messages: [
+          {
+            role: 'system',
+            content:
+              'You are an agent that decides on actions. Respond with a valid action matching the schema. Choose "complete" action and provide reasoning.'
+          },
+          { role: 'user', content: 'The task is done. What action should we take?' }
+        ],
+        responseSchema: {
+          type: 'object',
+          properties: {
+            reasoning: { type: 'string' },
+            action: {
+              anyOf: [
+                {
+                  type: 'object',
+                  properties: {
+                    type: { const: 'complete' } // Transformed to: type: "string", enum: ["complete"]
+                  },
+                  required: ['type']
+                },
+                {
+                  type: 'object',
+                  properties: {
+                    type: { const: 'tool-call' }, // Transformed to: type: "string", enum: ["tool-call"]
+                    name: { type: 'string' },
+                    // Note: Object properties must be non-empty for Gemini
+                    input: {
+                      type: 'object',
+                      properties: {
+                        value: { type: 'string' }
+                      }
+                    }
+                  },
+                  required: ['type', 'name', 'input']
+                }
+              ]
+            }
+          },
+          required: ['reasoning', 'action']
+        },
+        maxOutputTokens: 8000
+      }
+
+      // This call should NOT fail with "Unknown name const" error anymore
+      const response = await adapter.generate<{ reasoning: string; action: { type: string } }>(request)
+
+      expect(response.output).toBeDefined()
+      expect(response.output.reasoning).toBeDefined()
+      expect(typeof response.output.reasoning).toBe('string')
+      expect(response.output.action).toBeDefined()
+      expect(response.output.action.type).toBe('complete')
+    })
+
+    /**
+     * Tests for empty object schema handling
+     * Google requires properties to be non-empty for OBJECT type
+     * These tests verify our transformation handles these cases correctly
+     *
+     * SKIP REASON: The gemini-3-flash-preview model is unstable with additionalProperties schemas.
+     * It generates very long responses (10-20KB) that frequently get truncated mid-JSON,
+     * causing JSON.parse to fail. This triggers the circuit breaker after multiple retries.
+     * The schema transformation logic is verified by the unit tests in google-adapter.test.ts.
+     * These integration tests can be re-enabled when using a stable Gemini model.
+     */
+    describe.skip('Empty Object Schema Handling', () => {
+      it(
+        'handles object with additionalProperties but no properties (memoryOps.set pattern)',
+        { timeout: 60000, retry: 3 },
+        async () => {
+          // This is the pattern used by memoryOps.set: { type: 'object', additionalProperties: true }
+          // Google rejects: "properties: should be non-empty for OBJECT type"
+          const config: ModelConfig = {
+            provider: 'google',
+            model: 'gemini-3-flash-preview',
+            apiKey,
+            temperature: 1,
+            maxOutputTokens: 8000
+          }
+
+          const adapter = createLLMAdapter(config)
+
+          const request: LLMGenerateRequest = {
+            messages: [
+              { role: 'system', content: 'Return a simple key-value object.' },
+              { role: 'user', content: 'Give me a config with name="test" and value=123' }
+            ],
+            responseSchema: {
+              type: 'object',
+              properties: {
+                data: {
+                  type: 'object',
+                  additionalProperties: true // No properties defined - dynamic keys
+                }
+              },
+              required: ['data']
+            },
+            maxOutputTokens: 8000
+          }
+
+          const response = await adapter.generate<{ data: Record<string, unknown> }>(request)
+
+          expect(response.output).toBeDefined()
+          expect(response.output.data).toBeDefined()
+          expect(typeof response.output.data).toBe('object')
+        }
+      )
+
+      it(
+        'handles z.record() pattern (responseSchema input in generate_content)',
+        { timeout: 60000, retry: 3 },
+        async () => {
+          // This is the pattern from generate_content tool input:
+          // responseSchema: z.record(z.string(), z.unknown())
+          // Which becomes: { type: 'object', additionalProperties: true }
+          const config: ModelConfig = {
+            provider: 'google',
+            model: 'gemini-3-flash-preview',
+            apiKey,
+            temperature: 1,
+            maxOutputTokens: 8000
+          }
+
+          const adapter = createLLMAdapter(config)
+
+          const request: LLMGenerateRequest = {
+            messages: [
+              { role: 'system', content: 'You are generating a JSON schema for structured output.' },
+              { role: 'user', content: 'Create a schema with properties: name (string) and age (number)' }
+            ],
+            responseSchema: {
+              type: 'object',
+              properties: {
+                schema: {
+                  type: 'object',
+                  additionalProperties: true // Dynamic schema structure
+                }
+              },
+              required: ['schema']
+            },
+            maxOutputTokens: 8000
+          }
+
+          const response = await adapter.generate<{ schema: Record<string, unknown> }>(request)
+
+          expect(response.output).toBeDefined()
+          expect(response.output.schema).toBeDefined()
+        }
+      )
+
+      it('handles empty object type with no additionalProperties', { timeout: 60000, retry: 3 }, async () => {
+        // Pure empty object: { type: 'object', properties: {} }
+        // Must add placeholder property
+        const config: ModelConfig = {
+          provider: 'google',
+          model: 'gemini-3-flash-preview',
+          apiKey,
+          temperature: 1,
+          maxOutputTokens: 8000
+        }
+
+        const adapter = createLLMAdapter(config)
+
+        const request: LLMGenerateRequest = {
+          messages: [
+            { role: 'system', content: 'Return a result with an empty metadata object.' },
+            { role: 'user', content: 'Give me a result with status=ok and empty metadata' }
+          ],
+          responseSchema: {
+            type: 'object',
+            properties: {
+              status: { type: 'string' },
+              metadata: {
+                type: 'object',
+                properties: {} // Empty properties - needs placeholder
+              }
+            },
+            required: ['status', 'metadata']
+          },
+          maxOutputTokens: 8000
+        }
+
+        const response = await adapter.generate<{ status: string; metadata: object }>(request)
+
+        expect(response.output).toBeDefined()
+        expect(response.output.status).toBe('ok')
+        expect(response.output.metadata).toBeDefined()
+      })
+
+      it('handles agent iteration schema with memoryOps.set pattern', { timeout: 60000, retry: 3 }, async () => {
+        // This is the actual pattern from agent iteration schema
+        // memoryOps: { set: { type: 'object', additionalProperties: true } }
+        const config: ModelConfig = {
+          provider: 'google',
+          model: 'gemini-3-flash-preview',
+          apiKey,
+          temperature: 1,
+          maxOutputTokens: 8000
+        }
+
+        const adapter = createLLMAdapter(config)
+
+        const request: LLMGenerateRequest = {
+          messages: [
+            { role: 'system', content: 'You are an agent. Respond with reasoning and memory operations.' },
+            { role: 'user', content: 'Store the value "hello" under key "greeting" in memory.' }
+          ],
+          responseSchema: {
+            type: 'object',
+            properties: {
+              reasoning: { type: 'string' },
+              memoryOps: {
+                type: 'object',
+                properties: {
+                  set: {
+                    type: 'object',
+                    additionalProperties: true // Dynamic key-value pairs
+                  },
+                  delete: {
+                    type: 'array',
+                    items: { type: 'string' }
+                  }
+                },
+                additionalProperties: false
+              }
+            },
+            required: ['reasoning']
+          },
+          maxOutputTokens: 8000
+        }
+
+        const response = await adapter.generate<{
+          reasoning: string
+          memoryOps?: { set?: Record<string, unknown>; delete?: string[] }
+        }>(request)
+
+        expect(response.output).toBeDefined()
+        expect(response.output.reasoning).toBeDefined()
+        // memoryOps.set should work with dynamic keys
+        if (response.output.memoryOps?.set) {
+          expect(typeof response.output.memoryOps.set).toBe('object')
+        }
+      })
+    })
+  })
+})