@elevasis/core 0.7.1 → 0.8.0

package/src/operations/observability/__tests__/openrouter-cost-flow.test.ts

@@ -1,297 +1,297 @@
-/**
- * OpenRouter Cost Flow Integration Tests
- * Verifies that actual cost from OpenRouter flows through the full observability pipeline:
- * OpenRouterAdapter → UniversalLLMAdapter → AIUsageCollector → Database
- */
-
-import { describe, it, expect, vi, beforeEach } from 'vitest'
-import { AIUsageCollector } from '../ai-usage-collector'
-import { UniversalLLMAdapter } from '../../../execution/engine/llm/adapters/universal-adapter'
-import type { LLMAdapter, LLMGenerateRequest, LLMGenerateResponse } from '../../../execution/engine/llm/types'
-import type { LLMModel } from '../../../execution/engine/llm/model-info'
-
-describe('OpenRouter Cost Flow', () => {
-  describe('AIUsageCollector', () => {
-    it('uses actual cost when provided by adapter', () => {
-      const collector = new AIUsageCollector()
-
-      collector.record({
-        model: 'openrouter/anthropic/claude-sonnet-4.5',
-        inputTokens: 100,
-        outputTokens: 50,
-        latencyMs: 1500,
-        cost: 0.00125 // Actual cost from OpenRouter
-      })
-
-      const summary = collector.getSummary()
-
-      expect(summary.totalCostUsd).toBe(0.00125)
-      expect(summary.calls[0].costUsd).toBe(0.00125)
-    })
-
-    it('falls back to calculateCost when cost not provided', () => {
-      const collector = new AIUsageCollector()
-
-      collector.record({
-        model: 'gpt-5-mini',
-        inputTokens: 100,
-        outputTokens: 50,
-        latencyMs: 1000
-        // No cost field - should use calculateCost()
-      })
-
-      const summary = collector.getSummary()
-
-      // Should have calculated a cost (not zero, not undefined)
-      expect(summary.totalCostUsd).toBeGreaterThan(0)
-      expect(summary.calls[0].costUsd).toBeGreaterThan(0)
-    })
-
-    it('handles mixed providers - some with actual cost, some calculated', () => {
-      const collector = new AIUsageCollector()
-
-      // OpenRouter call with actual cost
-      collector.record({
-        model: 'openrouter/anthropic/claude-sonnet-4.5',
-        inputTokens: 100,
-        outputTokens: 50,
-        latencyMs: 1500,
-        cost: 0.00125
-      })
-
-      // OpenAI call without cost (will be calculated)
-      collector.record({
-        model: 'gpt-5-mini',
-        inputTokens: 200,
-        outputTokens: 100,
-        latencyMs: 800
-      })
-
-      const summary = collector.getSummary()
-
-      expect(summary.callCount).toBe(2)
-      expect(summary.calls[0].costUsd).toBe(0.00125) // Actual
-      expect(summary.calls[1].costUsd).toBeGreaterThan(0) // Calculated
-      expect(summary.totalCostUsd).toBeGreaterThan(0.00125) // Sum of both
-    })
-
-    it('uses actual cost of zero when explicitly provided', () => {
-      const collector = new AIUsageCollector()
-
-      // Edge case: OpenRouter returns cost of 0 (e.g., free tier, cached response)
-      collector.record({
-        model: 'openrouter/anthropic/claude-sonnet-4.5',
-        inputTokens: 100,
-        outputTokens: 50,
-        latencyMs: 500,
-        cost: 0 // Explicitly zero (not undefined)
-      })
-
-      const summary = collector.getSummary()
-
-      // Should use the explicit zero, not fall back to calculated cost
-      expect(summary.totalCostUsd).toBe(0)
-      expect(summary.calls[0].costUsd).toBe(0)
-    })
-  })
-
-  describe('UniversalLLMAdapter Cost Passthrough', () => {
-    let mockBaseAdapter: LLMAdapter
-    let collector: AIUsageCollector
-
-    const baseRequest: LLMGenerateRequest = {
-      messages: [{ role: 'user', content: 'Hello' }],
-      responseSchema: { type: 'object', properties: { response: { type: 'string' } } },
-      maxOutputTokens: 1000
-    }
-
-    beforeEach(() => {
-      vi.clearAllMocks()
-      collector = new AIUsageCollector()
-    })
-
-    it('passes actual cost from response to collector', async () => {
-      // Mock adapter that returns response with cost (like OpenRouter)
-      mockBaseAdapter = {
-        generate: vi.fn().mockResolvedValue({
-          output: { response: 'Hello!' },
-          usage: {
-            inputTokens: 15,
-            outputTokens: 25,
-            totalTokens: 40
-          },
-          cost: 0.00089 // Actual cost from OpenRouter
-        } as LLMGenerateResponse)
-      }
-
-      const universalAdapter = new UniversalLLMAdapter(
-        mockBaseAdapter,
-        'openrouter/anthropic/claude-sonnet-4.5' as LLMModel,
-        collector,
-        'agent-reasoning'
-      )
-
-      await universalAdapter.generate(baseRequest)
-
-      const summary = collector.getSummary()
-
-      expect(summary.callCount).toBe(1)
-      expect(summary.calls[0].costUsd).toBe(0.00089)
-      expect(summary.totalCostUsd).toBe(0.00089)
-    })
-
-    it('collector calculates cost when response has no cost field', async () => {
-      // Mock adapter that returns response without cost (like OpenAI)
-      mockBaseAdapter = {
-        generate: vi.fn().mockResolvedValue({
-          output: { response: 'Hello!' },
-          usage: {
-            inputTokens: 15,
-            outputTokens: 25,
-            totalTokens: 40
-          }
-          // No cost field
-        } as LLMGenerateResponse)
-      }
-
-      const universalAdapter = new UniversalLLMAdapter(
-        mockBaseAdapter,
-        'gpt-5-mini' as LLMModel,
-        collector,
-        'agent-reasoning'
-      )
-
-      await universalAdapter.generate(baseRequest)
-
-      const summary = collector.getSummary()
-
-      expect(summary.callCount).toBe(1)
-      // Cost should be calculated (not zero, not undefined)
-      expect(summary.calls[0].costUsd).toBeGreaterThan(0)
-    })
-
-    it('tracks multiple calls with mixed cost sources', async () => {
-      // First call: OpenRouter with actual cost
-      const openRouterAdapter: LLMAdapter = {
-        generate: vi.fn().mockResolvedValue({
-          output: { response: 'Hello from Claude!' },
-          usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 },
-          cost: 0.0025
-        } as LLMGenerateResponse)
-      }
-
-      // Second call: OpenAI without cost
-      const openAIAdapter: LLMAdapter = {
-        generate: vi.fn().mockResolvedValue({
-          output: { response: 'Hello from GPT!' },
-          usage: { inputTokens: 80, outputTokens: 40, totalTokens: 120 }
-        } as LLMGenerateResponse)
-      }
-
-      const universalOpenRouter = new UniversalLLMAdapter(
-        openRouterAdapter,
-        'openrouter/anthropic/claude-sonnet-4.5' as LLMModel,
-        collector,
-        'agent-reasoning'
-      )
-
-      const universalOpenAI = new UniversalLLMAdapter(
-        openAIAdapter,
-        'gpt-5-mini' as LLMModel,
-        collector,
-        'workflow-step'
-      )
-
-      await universalOpenRouter.generate(baseRequest)
-      await universalOpenAI.generate(baseRequest)
-
-      const summary = collector.getSummary()
-
-      expect(summary.callCount).toBe(2)
-      expect(summary.calls[0].costUsd).toBe(0.0025) // Actual from OpenRouter
-      expect(summary.calls[1].costUsd).toBeGreaterThan(0) // Calculated for OpenAI
-      expect(summary.totalCostUsd).toBeGreaterThan(0.0025) // Sum
-    })
-
-    it('preserves call context alongside cost', async () => {
-      mockBaseAdapter = {
-        generate: vi.fn().mockResolvedValue({
-          output: { response: 'Hello!' },
-          usage: { inputTokens: 50, outputTokens: 25, totalTokens: 75 },
-          cost: 0.00045
-        } as LLMGenerateResponse)
-      }
-
-      const universalAdapter = new UniversalLLMAdapter(
-        mockBaseAdapter,
-        'openrouter/deepseek/deepseek-v3.2' as LLMModel,
-        collector,
-        'agent-reasoning',
-        { type: 'agent-reasoning', iteration: 3 }
-      )
-
-      await universalAdapter.generate(baseRequest)
-
-      const summary = collector.getSummary()
-      const call = summary.calls[0]
-
-      // Verify cost is captured
-      expect(call.costUsd).toBe(0.00045)
-
-      // Verify context is preserved
-      expect(call.callType).toBe('agent-reasoning')
-      expect(call.context).toEqual({ type: 'agent-reasoning', iteration: 3 })
-
-      // Verify tokens are tracked
-      expect(call.inputTokens).toBe(50)
-      expect(call.outputTokens).toBe(25)
-    })
-  })
-
-  describe('End-to-End Cost Accuracy', () => {
-    it('aggregates costs correctly across multiple calls', () => {
-      const collector = new AIUsageCollector()
-
-      // Simulate a typical execution with multiple LLM calls
-      const calls = [
-        { model: 'openrouter/anthropic/claude-sonnet-4.5', cost: 0.00125 },
-        { model: 'openrouter/anthropic/claude-sonnet-4.5', cost: 0.00089 },
-        { model: 'openrouter/deepseek/deepseek-v3.2', cost: 0.00034 }
-      ]
-
-      calls.forEach((call, i) => {
-        collector.record({
-          model: call.model as LLMModel,
-          inputTokens: 100 + i * 50,
-          outputTokens: 50 + i * 25,
-          latencyMs: 1000 + i * 200,
-          cost: call.cost
-        })
-      })
-
-      const summary = collector.getSummary()
-
-      expect(summary.callCount).toBe(3)
-      expect(summary.totalCostUsd).toBe(0.00125 + 0.00089 + 0.00034)
-      expect(summary.totalCostUsd).toBeCloseTo(0.00248, 5)
-    })
-
-    it('maintains precision for small costs', () => {
-      const collector = new AIUsageCollector()
-
-      // Very small cost (e.g., DeepSeek which is very cheap)
-      collector.record({
-        model: 'openrouter/deepseek/deepseek-v3.2',
-        inputTokens: 10,
-        outputTokens: 5,
-        latencyMs: 200,
-        cost: 0.000001 // $0.000001 = $1 per million tokens
-      })
-
-      const summary = collector.getSummary()
-
-      expect(summary.totalCostUsd).toBe(0.000001)
-      expect(summary.calls[0].costUsd).toBe(0.000001)
-    })
-  })
-})
+/**
+ * OpenRouter Cost Flow Integration Tests
+ * Verifies that actual cost from OpenRouter flows through the full observability pipeline:
+ * OpenRouterAdapter → UniversalLLMAdapter → AIUsageCollector → Database
+ */
+
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+import { AIUsageCollector } from '../ai-usage-collector'
+import { UniversalLLMAdapter } from '../../../execution/engine/llm/adapters/universal-adapter'
+import type { LLMAdapter, LLMGenerateRequest, LLMGenerateResponse } from '../../../execution/engine/llm/types'
+import type { LLMModel } from '../../../execution/engine/llm/model-info'
+
+describe('OpenRouter Cost Flow', () => {
+  describe('AIUsageCollector', () => {
+    it('uses actual cost when provided by adapter', () => {
+      const collector = new AIUsageCollector()
+
+      collector.record({
+        model: 'openrouter/anthropic/claude-sonnet-4.5',
+        inputTokens: 100,
+        outputTokens: 50,
+        latencyMs: 1500,
+        cost: 0.00125 // Actual cost from OpenRouter
+      })
+
+      const summary = collector.getSummary()
+
+      expect(summary.totalCostUsd).toBe(0.00125)
+      expect(summary.calls[0].costUsd).toBe(0.00125)
+    })
+
+    it('falls back to calculateCost when cost not provided', () => {
+      const collector = new AIUsageCollector()
+
+      collector.record({
+        model: 'gpt-5-mini',
+        inputTokens: 100,
+        outputTokens: 50,
+        latencyMs: 1000
+        // No cost field - should use calculateCost()
+      })
+
+      const summary = collector.getSummary()
+
+      // Should have calculated a cost (not zero, not undefined)
+      expect(summary.totalCostUsd).toBeGreaterThan(0)
+      expect(summary.calls[0].costUsd).toBeGreaterThan(0)
+    })
+
+    it('handles mixed providers - some with actual cost, some calculated', () => {
+      const collector = new AIUsageCollector()
+
+      // OpenRouter call with actual cost
+      collector.record({
+        model: 'openrouter/anthropic/claude-sonnet-4.5',
+        inputTokens: 100,
+        outputTokens: 50,
+        latencyMs: 1500,
+        cost: 0.00125
+      })
+
+      // OpenAI call without cost (will be calculated)
+      collector.record({
+        model: 'gpt-5-mini',
+        inputTokens: 200,
+        outputTokens: 100,
+        latencyMs: 800
+      })
+
+      const summary = collector.getSummary()
+
+      expect(summary.callCount).toBe(2)
+      expect(summary.calls[0].costUsd).toBe(0.00125) // Actual
+      expect(summary.calls[1].costUsd).toBeGreaterThan(0) // Calculated
+      expect(summary.totalCostUsd).toBeGreaterThan(0.00125) // Sum of both
+    })
+
+    it('uses actual cost of zero when explicitly provided', () => {
+      const collector = new AIUsageCollector()
+
+      // Edge case: OpenRouter returns cost of 0 (e.g., free tier, cached response)
+      collector.record({
+        model: 'openrouter/anthropic/claude-sonnet-4.5',
+        inputTokens: 100,
+        outputTokens: 50,
+        latencyMs: 500,
+        cost: 0 // Explicitly zero (not undefined)
+      })
+
+      const summary = collector.getSummary()
+
+      // Should use the explicit zero, not fall back to calculated cost
+      expect(summary.totalCostUsd).toBe(0)
+      expect(summary.calls[0].costUsd).toBe(0)
+    })
+  })
+
+  describe('UniversalLLMAdapter Cost Passthrough', () => {
+    let mockBaseAdapter: LLMAdapter
+    let collector: AIUsageCollector
+
+    const baseRequest: LLMGenerateRequest = {
+      messages: [{ role: 'user', content: 'Hello' }],
+      responseSchema: { type: 'object', properties: { response: { type: 'string' } } },
+      maxOutputTokens: 1000
+    }
+
+    beforeEach(() => {
+      vi.clearAllMocks()
+      collector = new AIUsageCollector()
+    })
+
+    it('passes actual cost from response to collector', async () => {
+      // Mock adapter that returns response with cost (like OpenRouter)
+      mockBaseAdapter = {
+        generate: vi.fn().mockResolvedValue({
+          output: { response: 'Hello!' },
+          usage: {
+            inputTokens: 15,
+            outputTokens: 25,
+            totalTokens: 40
+          },
+          cost: 0.00089 // Actual cost from OpenRouter
+        } as LLMGenerateResponse)
+      }
+
+      const universalAdapter = new UniversalLLMAdapter(
+        mockBaseAdapter,
+        'openrouter/anthropic/claude-sonnet-4.5' as LLMModel,
+        collector,
+        'agent-reasoning'
+      )
+
+      await universalAdapter.generate(baseRequest)
+
+      const summary = collector.getSummary()
+
+      expect(summary.callCount).toBe(1)
+      expect(summary.calls[0].costUsd).toBe(0.00089)
+      expect(summary.totalCostUsd).toBe(0.00089)
+    })
+
+    it('collector calculates cost when response has no cost field', async () => {
+      // Mock adapter that returns response without cost (like OpenAI)
+      mockBaseAdapter = {
+        generate: vi.fn().mockResolvedValue({
+          output: { response: 'Hello!' },
+          usage: {
+            inputTokens: 15,
+            outputTokens: 25,
+            totalTokens: 40
+          }
+          // No cost field
+        } as LLMGenerateResponse)
+      }
+
+      const universalAdapter = new UniversalLLMAdapter(
+        mockBaseAdapter,
+        'gpt-5-mini' as LLMModel,
+        collector,
+        'agent-reasoning'
+      )
+
+      await universalAdapter.generate(baseRequest)
+
+      const summary = collector.getSummary()
+
+      expect(summary.callCount).toBe(1)
+      // Cost should be calculated (not zero, not undefined)
+      expect(summary.calls[0].costUsd).toBeGreaterThan(0)
+    })
+
+    it('tracks multiple calls with mixed cost sources', async () => {
+      // First call: OpenRouter with actual cost
+      const openRouterAdapter: LLMAdapter = {
+        generate: vi.fn().mockResolvedValue({
+          output: { response: 'Hello from Claude!' },
+          usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 },
+          cost: 0.0025
+        } as LLMGenerateResponse)
+      }
+
+      // Second call: OpenAI without cost
+      const openAIAdapter: LLMAdapter = {
+        generate: vi.fn().mockResolvedValue({
+          output: { response: 'Hello from GPT!' },
+          usage: { inputTokens: 80, outputTokens: 40, totalTokens: 120 }
+        } as LLMGenerateResponse)
+      }
+
+      const universalOpenRouter = new UniversalLLMAdapter(
+        openRouterAdapter,
+        'openrouter/anthropic/claude-sonnet-4.5' as LLMModel,
+        collector,
+        'agent-reasoning'
+      )
+
+      const universalOpenAI = new UniversalLLMAdapter(
+        openAIAdapter,
+        'gpt-5-mini' as LLMModel,
+        collector,
+        'workflow-step'
+      )
+
+      await universalOpenRouter.generate(baseRequest)
+      await universalOpenAI.generate(baseRequest)
+
+      const summary = collector.getSummary()
+
+      expect(summary.callCount).toBe(2)
+      expect(summary.calls[0].costUsd).toBe(0.0025) // Actual from OpenRouter
+      expect(summary.calls[1].costUsd).toBeGreaterThan(0) // Calculated for OpenAI
+      expect(summary.totalCostUsd).toBeGreaterThan(0.0025) // Sum
+    })
+
+    it('preserves call context alongside cost', async () => {
+      mockBaseAdapter = {
+        generate: vi.fn().mockResolvedValue({
+          output: { response: 'Hello!' },
+          usage: { inputTokens: 50, outputTokens: 25, totalTokens: 75 },
+          cost: 0.00045
+        } as LLMGenerateResponse)
+      }
+
+      const universalAdapter = new UniversalLLMAdapter(
+        mockBaseAdapter,
+        'openrouter/deepseek/deepseek-v3.2' as LLMModel,
+        collector,
+        'agent-reasoning',
+        { type: 'agent-reasoning', iteration: 3 }
+      )
+
+      await universalAdapter.generate(baseRequest)
+
+      const summary = collector.getSummary()
+      const call = summary.calls[0]
+
+      // Verify cost is captured
+      expect(call.costUsd).toBe(0.00045)
+
+      // Verify context is preserved
+      expect(call.callType).toBe('agent-reasoning')
+      expect(call.context).toEqual({ type: 'agent-reasoning', iteration: 3 })
+
+      // Verify tokens are tracked
+      expect(call.inputTokens).toBe(50)
+      expect(call.outputTokens).toBe(25)
+    })
+  })
+
+  describe('End-to-End Cost Accuracy', () => {
+    it('aggregates costs correctly across multiple calls', () => {
+      const collector = new AIUsageCollector()
+
+      // Simulate a typical execution with multiple LLM calls
+      const calls = [
+        { model: 'openrouter/anthropic/claude-sonnet-4.5', cost: 0.00125 },
+        { model: 'openrouter/anthropic/claude-sonnet-4.5', cost: 0.00089 },
+        { model: 'openrouter/deepseek/deepseek-v3.2', cost: 0.00034 }
+      ]
+
+      calls.forEach((call, i) => {
+        collector.record({
+          model: call.model as LLMModel,
+          inputTokens: 100 + i * 50,
+          outputTokens: 50 + i * 25,
+          latencyMs: 1000 + i * 200,
+          cost: call.cost
+        })
+      })
+
+      const summary = collector.getSummary()
+
+      expect(summary.callCount).toBe(3)
+      expect(summary.totalCostUsd).toBe(0.00125 + 0.00089 + 0.00034)
+      expect(summary.totalCostUsd).toBeCloseTo(0.00248, 5)
+    })
+
+    it('maintains precision for small costs', () => {
+      const collector = new AIUsageCollector()
+
+      // Very small cost (e.g., DeepSeek which is very cheap)
+      collector.record({
+        model: 'openrouter/deepseek/deepseek-v3.2',
+        inputTokens: 10,
+        outputTokens: 5,
+        latencyMs: 200,
+        cost: 0.000001 // $0.000001 = $1 per million tokens
+      })
+
+      const summary = collector.getSummary()
+
+      expect(summary.totalCostUsd).toBe(0.000001)
+      expect(summary.calls[0].costUsd).toBe(0.000001)
+    })
+  })
+})