outcome-cli 1.0.0

package/src/runtime/README.md

@@ -0,0 +1,220 @@
+# Runtime Module
+
+The runtime module handles agent execution, model adapters, and cost tracking for the Earnd Bounty Engine.
+
+## Components
+
+### Model Adapter (`modelAdapter.ts`, `claudeAdapter.ts`, `openaiAdapter.ts`)
+
+Unified interface for AI model providers. Abstracts Claude and OpenAI behind a common interface with normalized responses and tool calling support.
+
+**Key Interfaces:**
+
+- `ModelAdapter` - Common interface for all model providers (includes `provider` and `modelId` readonly properties)
+- `ModelResponse` - Normalized response format (content, tokensUsed, model, toolCalls, requiresToolResponse)
+- `ModelOptions` - Request configuration (maxTokens, temperature, systemPrompt, tools)
+- `ModelAdapterConfig` - Configuration for creating adapters (provider, modelId, apiKey)
+- `ConversationMessage` - Message format for multi-turn tool use conversations
+- `ToolDefinition` - Tool schema definition (from skills module)
+- `ToolCall` - Tool invocation request from model
+
+**Key Functions:**
+
+- `createAdapter(config)` - Factory function to create adapter based on provider
+- `createClaudeAdapter(apiKey, modelId)` - Creates Claude-specific adapter
+- `createOpenAIAdapter(apiKey, modelId)` - Creates OpenAI-specific adapter
+
+**Usage:**
+
+```typescript
+import { createAdapter, createClaudeAdapter, createOpenAIAdapter } from './index.js';
+
+// Using factory function
+const adapter = await createAdapter({
+  provider: 'claude',
+  modelId: 'claude-3-sonnet-20240229',
+  apiKey: process.env.ANTHROPIC_API_KEY!
+});
+
+// Or directly create specific adapters
+const claudeAdapter = createClaudeAdapter(
+  process.env.ANTHROPIC_API_KEY!,
+  'claude-3-sonnet-20240229'
+);
+
+const openaiAdapter = createOpenAIAdapter(
+  process.env.OPENAI_API_KEY!,
+  'gpt-4-turbo-preview'
+);
+
+// Make completion request
+const response = await adapter.complete('Hello, how are you?', {
+  maxTokens: 1024,
+  temperature: 0.7,
+  systemPrompt: 'You are a helpful assistant.'
+});
+
+console.log(response.content);      // Generated text
+console.log(response.tokensUsed);   // Total tokens used
+console.log(response.model);        // Model identifier
+
+// Access adapter properties
+console.log(adapter.provider);      // 'claude' or 'openai'
+console.log(adapter.modelId);       // The specific model ID
+
+// Estimate tokens before sending
+const estimatedTokens = adapter.countTokens('Some text to count');
+```
+
+**Tool Calling:**
+
+Both adapters support function/tool calling with a unified interface:
+
+```typescript
+import type { ToolDefinition, ConversationMessage } from './modelAdapter.js';
+
+// Define tools available to the model
+const tools: ToolDefinition[] = [
+  {
+    name: 'get_company_info',
+    description: 'Get information about a company',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        companyName: { type: 'string', description: 'Name of the company' }
+      },
+      required: ['companyName']
+    }
+  }
+];
+
+// Request with tools
+const response = await adapter.complete('Look up TechCorp Industries', {
+  tools,
+  systemPrompt: 'You are a sales assistant.'
+});
+
+// Check if model wants to call tools
+if (response.toolCalls && response.toolCalls.length > 0) {
+  // Execute tool calls and gather results
+  const messages: ConversationMessage[] = [
+    { role: 'user', content: 'Look up TechCorp Industries' },
+    { role: 'assistant', content: response.content, toolCalls: response.toolCalls },
+  ];
+
+  // Add tool results
+  for (const toolCall of response.toolCalls) {
+    const result = await executeToolCall(toolCall); // Your tool execution logic
+    messages.push({
+      role: 'tool',
+      toolCallId: toolCall.id,
+      content: JSON.stringify(result)
+    });
+  }
+
+  // Continue conversation with tool results
+  const finalResponse = await adapter.continueWithToolResults(messages, { tools });
+  console.log(finalResponse.content);
+}
+```
+
+### Agent Runner (`agentRunner.ts`)
+
+Agent execution engine with attempt loop, cost tracking, and retry logic.
+
+**Key Interfaces:**
+
+- `AgentRun` - Result of running an agent (agentId, outcomeId, status, attempts, tokensSpent, artifacts)
+- `AgentRunConfig` - Configuration for running an agent (agent, outcome, lead, apiKey, onKillSignal)
+- `KillReason` - Reason for agent termination ('cost_exceeded', 'attempts_exceeded', 'timeout', 'competitor_won', 'success')
+
+**Key Functions:**
+
+- `runAgent(config)` - Executes an agent to achieve an outcome with full tracking
+- `runAgentMock(config)` - Mock runner for testing without API calls
+
+**Features:**
+
+- **Attempt Loop** - Tracks attempts against maxAttempts limit
+- **Cost Integration** - Records token usage to CostTracker in real-time
+- **Exponential Backoff** - Retries failed API calls with backoff (max 3 retries)
+- **Kill Conditions** - Monitors cost ceiling, attempt limit, runtime limit, and external signals
+- **Artifact Generation** - Creates AgentArtifact from model responses
+
+**Usage:**
+
+```typescript
+import { runAgent, runAgentMock } from './agentRunner.js';
+
+// Run agent with real API calls
+const result = await runAgent({
+  agent: salesAgentConfig,
+  outcome: qualifiedSalesInterest,
+  lead: leadData,
+  apiKey: process.env.ANTHROPIC_API_KEY,
+  onKillSignal: () => competitorWon, // Optional external kill signal
+});
+
+// Check result
+if (result.status === 'completed') {
+  console.log('Agent completed with', result.artifacts.length, 'artifacts');
+} else if (result.status === 'killed') {
+  console.log('Agent killed:', result.killReason);
+}
+
+// For testing without API calls
+const mockResult = await runAgentMock(config);
+```
+
+### Cost Tracker (`costTracker.ts`)
+
+Real-time token and cost tracking per agent. Enforces hard token caps to prevent runaway costs.
+
+**Key Functions:**
+
+- `createCostTracker(agentId, ceiling)` - Creates a new tracker for an agent
+- `recordUsage(tracker, tokens)` - Records token usage in real-time
+- `isOverBudget(tracker)` - Checks if agent exceeded its ceiling
+- `getRemainingBudget(tracker)` - Gets remaining token budget
+- `getBudgetUsagePercent(tracker)` - Gets percentage of budget used
+
+**Usage:**
+
+```typescript
+import { createCostTracker, recordUsage, isOverBudget } from './costTracker.js';
+
+// Create tracker with 10,000 token ceiling
+const tracker = createCostTracker('agent-1', 10000);
+
+// Record usage after each model call
+recordUsage(tracker, 500);
+
+// Check if agent should be terminated
+if (isOverBudget(tracker)) {
+  // Kill agent with reason 'cost_exceeded'
+}
+```
+
+## Requirements Reference
+
+- **4.2** - Track cost per agent independently
+- **4.3** - Terminate agent on attempt limit exceeded
+- **10.1** - Enforce hard token cap per agent
+- **10.2** - Enforce max runtime per agent
+- **10.5** - Track tokens spent per agent in real-time
+- **11.1** - Abstract underlying AI model provider
+- **11.2** - Route requests to Claude API
+- **11.3** - Route requests to OpenAI API
+- **11.4** - Normalize response format
+
+## Design Principles
+
+1. **Isolation** - Each agent has its own tracker, no cross-contamination
+2. **Real-time** - Usage is tracked immediately after each model call
+3. **Fail-closed** - When over budget, agents are terminated with no payout
+4. **Provider Agnostic** - Same interface regardless of Claude or OpenAI
+5. **Normalized Responses** - Consistent format for all model responses
+
+## Related Modules
+
+- **Skills** (`/src/skills`) - Defines agent tools/capabilities that can be invoked via model function calling. The `ToolDefinition`, `ToolCall`, and `ToolCallResult` types are imported from the skills module and re-exported by `modelAdapter.ts` for convenience.

package/src/runtime/agentRunner.test.ts

@@ -0,0 +1,341 @@
+/**
+ * Agent Runner Tests
+ *
+ * Tests for the agent execution engine including:
+ * - Attempt loop logic
+ * - Cost tracking integration
+ * - Kill condition handling
+ * - Mock runner functionality
+ *
+ * @module runtime/agentRunner.test
+ */
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import {
+  runAgent,
+  runAgentMock,
+  type AgentRunConfig,
+  type AgentRun,
+} from './agentRunner.js';
+import type { AgentConfig } from '../agents/agent.schema.js';
+import type { Outcome } from '../outcomes/outcome.schema.js';
+import type { Lead } from '../jobs/job.interface.js';
+
+/**
+ * Test fixtures
+ */
+const createTestAgent = (overrides?: Partial<AgentConfig>): AgentConfig => ({
+  id: 'test-agent-001',
+  name: 'Test Agent',
+  prompt: 'You are a test agent. Respond with a helpful message.',
+  strategyDescription: 'Test strategy',
+  toolAccess: [],
+  costCeiling: 5000,
+  modelProvider: 'claude',
+  modelId: 'claude-3-sonnet-20240229',
+  ...overrides,
+});
+
+const createTestOutcome = (overrides?: Partial<Outcome>): Outcome => ({
+  name: 'test_outcome',
+  description: 'Test outcome for unit tests',
+  payoutAmount: 100,
+  maxAttempts: 3,
+  timeLimitMs: 60000,
+  successCriteria: [
+    {
+      name: 'test_criterion',
+      validator: 'validateMessageLength',
+      params: { minWords: 5 },
+    },
+  ],
+  failureReasons: ['Test failure reason'],
+  ...overrides,
+});
+
+const createTestLead = (overrides?: Partial<Lead>): Lead => ({
+  email: 'test@example.com',
+  company: 'Test Company',
+  companySize: 100,
+  role: 'Manager',
+  previousInteractions: ['Downloaded whitepaper'],
+  ...overrides,
+});
+
+describe('agentRunner', () => {
+  describe('runAgentMock', () => {
+    it('should return a completed run with artifact', async () => {
+      const config: AgentRunConfig = {
+        agent: createTestAgent(),
+        outcome: createTestOutcome(),
+        lead: createTestLead(),
+      };
+
+      const result = await runAgentMock(config);
+
+      expect(result.agentId).toBe('test-agent-001');
+      expect(result.outcomeId).toBe('test_outcome');
+      expect(result.status).toBe('completed');
+      expect(result.attempts).toBe(1);
+      expect(result.tokensSpent).toBeGreaterThan(0);
+      expect(result.artifacts).toHaveLength(1);
+      expect(result.durationMs).toBeGreaterThan(0);
+    });
+
+    it('should include lead data in artifact content', async () => {
+      const lead = createTestLead({
+        email: 'custom@test.com',
+        company: 'Custom Corp',
+        companySize: 250,
+        role: 'Director',
+      });
+
+      const config: AgentRunConfig = {
+        agent: createTestAgent(),
+        outcome: createTestOutcome(),
+        lead,
+      };
+
+      const result = await runAgentMock(config);
+      const artifact = result.artifacts[0];
+
+      expect(artifact.content.targetEmail).toBe('custom@test.com');
+      expect(artifact.content.targetCompany).toBe('Custom Corp');
+      expect(artifact.content.targetCompanySize).toBe(250);
+      expect(artifact.content.targetRole).toBe('Director');
+    });
+
+    it('should set correct timestamp on artifact', async () => {
+      const config: AgentRunConfig = {
+        agent: createTestAgent(),
+        outcome: createTestOutcome(),
+        lead: createTestLead(),
+      };
+
+      const beforeRun = new Date().toISOString();
+      const result = await runAgentMock(config);
+      const afterRun = new Date().toISOString();
+
+      const artifactTime = result.artifacts[0].timestamp;
+      expect(artifactTime >= beforeRun).toBe(true);
+      expect(artifactTime <= afterRun).toBe(true);
+    });
+  });
+
+  describe('runAgent - configuration validation', () => {
+    it('should fail without API key', async () => {
+      const originalEnv = process.env.ANTHROPIC_API_KEY;
+      delete process.env.ANTHROPIC_API_KEY;
+
+      const config: AgentRunConfig = {
+        agent: createTestAgent(),
+        outcome: createTestOutcome(),
+        lead: createTestLead(),
+      };
+
+      const result = await runAgent(config);
+
+      expect(result.status).toBe('killed');
+      expect(result.error).toContain('Missing API key');
+
+      // Restore env
+      if (originalEnv) {
+        process.env.ANTHROPIC_API_KEY = originalEnv;
+      }
+    });
+
+    it('should use provided API key over environment variable', async () => {
+      const config: AgentRunConfig = {
+        agent: createTestAgent(),
+        outcome: createTestOutcome({ timeLimitMs: 100 }), // Short timeout for test
+        lead: createTestLead(),
+        apiKey: 'test-api-key',
+        onKillSignal: () => true, // Immediately kill to avoid API call
+      };
+
+      const result = await runAgent(config);
+
+      // Should have been killed by signal, not due to missing API key
+      // error is undefined when killed by signal (no API key error)
+      expect(result.error === undefined || !result.error.includes('Missing API key')).toBe(true);
+      expect(result.killReason).toBe('competitor_won');
+    });
+  });
+
+  describe('runAgent - kill conditions', () => {
+    it('should kill agent when cost ceiling is exceeded via onKillSignal', async () => {
+      let killSignalCalled = false;
+      const config: AgentRunConfig = {
+        agent: createTestAgent({ costCeiling: 100 }),
+        outcome: createTestOutcome(),
+        lead: createTestLead(),
+        apiKey: 'test-key',
+        onKillSignal: () => {
+          killSignalCalled = true;
+          return true; // Simulate competitor won
+        },
+      };
+
+      const result = await runAgent(config);
+
+      expect(killSignalCalled).toBe(true);
+      expect(result.status).toBe('killed');
+      expect(result.killReason).toBe('competitor_won');
+    });
+  });
+
+  describe('AgentRun structure', () => {
+    it('should have all required fields', async () => {
+      const config: AgentRunConfig = {
+        agent: createTestAgent(),
+        outcome: createTestOutcome(),
+        lead: createTestLead(),
+      };
+
+      const result = await runAgentMock(config);
+
+      // Check all required AgentRun fields exist
+      expect(result).toHaveProperty('agentId');
+      expect(result).toHaveProperty('outcomeId');
+      expect(result).toHaveProperty('status');
+      expect(result).toHaveProperty('attempts');
+      expect(result).toHaveProperty('tokensSpent');
+      expect(result).toHaveProperty('artifacts');
+      expect(result).toHaveProperty('durationMs');
+
+      // Check types
+      expect(typeof result.agentId).toBe('string');
+      expect(typeof result.outcomeId).toBe('string');
+      expect(['running', 'completed', 'killed']).toContain(result.status);
+      expect(typeof result.attempts).toBe('number');
+      expect(typeof result.tokensSpent).toBe('number');
+      expect(Array.isArray(result.artifacts)).toBe(true);
+      expect(typeof result.durationMs).toBe('number');
+    });
+
+    it('should track attempt count correctly', async () => {
+      const config: AgentRunConfig = {
+        agent: createTestAgent(),
+        outcome: createTestOutcome({ maxAttempts: 5 }),
+        lead: createTestLead(),
+      };
+
+      const result = await runAgentMock(config);
+
+      expect(result.attempts).toBeGreaterThanOrEqual(1);
+      expect(result.attempts).toBeLessThanOrEqual(5);
+    });
+  });
+
+  describe('Artifact structure', () => {
+    it('should create valid artifact with all required fields', async () => {
+      const config: AgentRunConfig = {
+        agent: createTestAgent({ id: 'artifact-test-agent' }),
+        outcome: createTestOutcome({ name: 'artifact_test_outcome' }),
+        lead: createTestLead(),
+      };
+
+      const result = await runAgentMock(config);
+      const artifact = result.artifacts[0];
+
+      expect(artifact.agentId).toBe('artifact-test-agent');
+      expect(artifact.outcomeId).toBe('artifact_test_outcome');
+      expect(artifact.attemptNumber).toBe(1);
+      expect(artifact.content).toBeDefined();
+      expect(artifact.timestamp).toBeDefined();
+
+      // Check content structure
+      expect(artifact.content).toHaveProperty('message');
+      expect(artifact.content).toHaveProperty('targetEmail');
+      expect(artifact.content).toHaveProperty('targetCompany');
+      expect(artifact.content).toHaveProperty('targetCompanySize');
+      expect(artifact.content).toHaveProperty('targetRole');
+    });
+  });
+});
+
+describe('AgentRunner - Property Tests', () => {
+  /**
+   * **Feature: earnd-bounty-engine, Property: Agent Run Structure Compliance**
+   * For any agent run, the result SHALL conform to the AgentRun interface
+   * with all required fields populated.
+   * **Validates: Requirements 4.2, 4.3**
+   */
+  it('Property: AgentRun always has valid structure', async () => {
+    const configs = [
+      {
+        agent: createTestAgent(),
+        outcome: createTestOutcome(),
+        lead: createTestLead(),
+      },
+      {
+        agent: createTestAgent({ id: 'agent-2', costCeiling: 1000 }),
+        outcome: createTestOutcome({ maxAttempts: 1 }),
+        lead: createTestLead({ companySize: 10 }),
+      },
+      {
+        agent: createTestAgent({ id: 'agent-3', modelProvider: 'openai' as const }),
+        outcome: createTestOutcome({ timeLimitMs: 1000 }),
+        lead: createTestLead({ role: 'Intern' }),
+      },
+    ];
+
+    for (const config of configs) {
+      const result = await runAgentMock(config);
+
+      // Verify structure invariants
+      expect(result.agentId).toBe(config.agent.id);
+      expect(result.outcomeId).toBe(config.outcome.name);
+      expect(['running', 'completed', 'killed']).toContain(result.status);
+      expect(result.attempts).toBeGreaterThanOrEqual(0);
+      expect(result.tokensSpent).toBeGreaterThanOrEqual(0);
+      expect(Array.isArray(result.artifacts)).toBe(true);
+      expect(result.durationMs).toBeGreaterThanOrEqual(0);
+    }
+  });
+
+  /**
+   * **Feature: earnd-bounty-engine, Property: Completed runs have artifacts**
+   * For any completed agent run, there SHALL be at least one artifact.
+   */
+  it('Property: Completed runs always have at least one artifact', async () => {
+    const config: AgentRunConfig = {
+      agent: createTestAgent(),
+      outcome: createTestOutcome(),
+      lead: createTestLead(),
+    };
+
+    const result = await runAgentMock(config);
+
+    if (result.status === 'completed') {
+      expect(result.artifacts.length).toBeGreaterThanOrEqual(1);
+    }
+  });
+
+  /**
+   * **Feature: earnd-bounty-engine, Property: Killed runs have kill reason**
+   * For any killed agent run, there SHALL be a kill reason specified.
+   */
+  it('Property: Killed runs always have kill reason', async () => {
+    const config: AgentRunConfig = {
+      agent: createTestAgent(),
+      outcome: createTestOutcome(),
+      lead: createTestLead(),
+      apiKey: 'test-key',
+      onKillSignal: () => true, // Immediately signal kill
+    };
+
+    const result = await runAgent(config);
+
+    if (result.status === 'killed') {
+      expect(result.killReason).toBeDefined();
+      expect([
+        'cost_exceeded',
+        'attempts_exceeded',
+        'timeout',
+        'competitor_won',
+        'success',
+      ]).toContain(result.killReason);
+    }
+  });
+});