outcome-cli 1.0.0

package/src/eval/weighted-scorer.ts

@@ -0,0 +1,285 @@
+/**
+ * Weighted Scoring System
+ *
+ * Implements granular scoring with weighted criteria instead of binary evaluation.
+ * Agents are rewarded for partial success based on criterion importance.
+ *
+ * @module eval/weighted-scorer
+ * @see Requirements 9.1, 9.2, 9.3, 9.4
+ */
+
+/**
+ * Result of a weighted validation operation.
+ *
+ * Unlike binary validation, this includes a numeric score (0.0 to 1.0)
+ * representing partial success.
+ *
+ * @see Requirements 9.1
+ */
+export interface WeightedValidationResult {
+  /** Whether the criterion passed (score >= threshold) */
+  success: boolean;
+  /** Numeric score from 0.0 to 1.0 */
+  score: number;
+  /** Human-readable reason for the result */
+  reason: string;
+  /** Optional additional details */
+  details?: Record<string, unknown>;
+}
+
+// DAWS presets for code generation (Outcome pivot)
+// GCR: tests pass, QAS: AI review, CEF: cost efficiency, DTT: turnaround, RES: resilience
+export const DAWS_CODE_WEIGHTS = {
+  GCR: 0.35,
+  QAS: 0.25,
+  CEF: 0.15,
+  DTT: 0.15,
+  RES: 0.10,
+} as const;
+
+export type DawsMetric = keyof typeof DAWS_CODE_WEIGHTS;
+
+export interface DawsScores {
+  GCR: number; // tests / correctness
+  QAS: number; // AI review score (0..1)
+  CEF: number; // cost efficiency normalized 0..1
+  DTT: number; // speed normalized 0..1
+  RES: number; // resilience / retries normalized 0..1
+}
+
+/**
+ * Calculates DAWS weighted score for code-gen tasks using the code defaults.
+ */
+export function calculateDawsCodeScore(scores: DawsScores): ScoringResult {
+  const criteria: WeightedCriterion[] = [
+    { name: 'GCR', weight: DAWS_CODE_WEIGHTS.GCR, validator: () => ({ success: scores.GCR >= 1, score: scores.GCR, reason: 'Goal completion (tests)' }) },
+    { name: 'QAS', weight: DAWS_CODE_WEIGHTS.QAS, validator: () => ({ success: scores.QAS >= 0.7, score: scores.QAS, reason: 'Quality alignment' }) },
+    { name: 'CEF', weight: DAWS_CODE_WEIGHTS.CEF, validator: () => ({ success: scores.CEF >= 0.5, score: scores.CEF, reason: 'Cost efficiency' }) },
+    { name: 'DTT', weight: DAWS_CODE_WEIGHTS.DTT, validator: () => ({ success: scores.DTT >= 0.5, score: scores.DTT, reason: 'Decision turnaround time' }) },
+    { name: 'RES', weight: DAWS_CODE_WEIGHTS.RES, validator: () => ({ success: scores.RES >= 0.5, score: scores.RES, reason: 'Resilience' }) },
+  ];
+
+  return calculateWeightedScore(scores, criteria);
+}
+
+/**
+ * A weighted criterion for evaluation.
+ *
+ * Each criterion has a weight representing its importance in the final score.
+ * The sum of all weights should equal 1.0.
+ *
+ * @see Requirements 9.1
+ */
+export interface WeightedCriterion {
+  /** Name of the criterion */
+  name: string;
+  /** Weight of this criterion (0.0 to 1.0, sum of all weights should be 1.0) */
+  weight: number;
+  /** Validator function that returns a WeightedValidationResult */
+  validator: (artifact: unknown) => WeightedValidationResult;
+}
+
+/**
+ * Result of a single criterion evaluation.
+ */
+export interface CriterionEvaluationResult {
+  /** Name of the criterion */
+  name: string;
+  /** Weight of this criterion */
+  weight: number;
+  /** Score achieved (0.0 to 1.0) */
+  score: number;
+  /** Human-readable reason */
+  reason: string;
+}
+
+/**
+ * Final scoring result after evaluating all criteria.
+ *
+ * @see Requirements 9.2
+ */
+export interface ScoringResult {
+  /** Final weighted score (0.0 to 1.0) */
+  finalScore: number;
+  /** Whether the artifact passed (finalScore >= passThreshold) */
+  passed: boolean;
+  /** Results for each individual criterion */
+  criteriaResults: CriterionEvaluationResult[];
+}
+
+/**
+ * Leaderboard entry with weighted scoring fields.
+ *
+ * @see Requirements 9.3, 9.4
+ */
+export interface WeightedLeaderboardEntry {
+  /** Rank position (1-indexed) */
+  rank: number;
+  /** Agent identifier */
+  agentId: string;
+  /** Agent display name */
+  agentName: string;
+  /** Owner user ID */
+  userId: string;
+  /** Cumulative weighted score across all battles */
+  cumulativeScore: number;
+  /** Total tokens used across all battles */
+  totalTokensUsed: number;
+  /** Efficiency: score per token (used as tiebreaker) */
+  efficiency: number;
+  /** Number of battles participated */
+  battlesCount: number;
+  /** Total earnings in USD */
+  totalEarnings: number;
+}
+
+/**
+ * Validates that weights sum to approximately 1.0.
+ *
+ * @param criteria - Array of weighted criteria
+ * @returns true if weights sum to 1.0 (within floating point tolerance)
+ */
+export function validateWeights(criteria: WeightedCriterion[]): boolean {
+  const sum = criteria.reduce((acc, c) => acc + c.weight, 0);
+  return Math.abs(sum - 1.0) < 0.0001;
+}
+
+/**
+ * Calculates the weighted score for an artifact against a set of criteria.
+ *
+ * The final score is computed as the weighted average of all criterion scores:
+ * finalScore = Σ(weight_i × score_i)
+ *
+ * @param artifact - The artifact to evaluate
+ * @param criteria - Array of weighted criteria to evaluate against
+ * @param passThreshold - Minimum score to pass (default 0.7)
+ * @returns ScoringResult with final score and individual criterion results
+ *
+ * @example
+ * const result = calculateWeightedScore(artifact, [
+ *   { name: 'accuracy', weight: 0.4, validator: validateAccuracy },
+ *   { name: 'completeness', weight: 0.3, validator: validateCompleteness },
+ *   { name: 'format', weight: 0.3, validator: validateFormat },
+ * ]);
+ *
+ * @see Requirements 9.2
+ */
+export function calculateWeightedScore(
+  artifact: unknown,
+  criteria: WeightedCriterion[],
+  passThreshold: number = 0.7
+): ScoringResult {
+  // Evaluate each criterion
+  const criteriaResults: CriterionEvaluationResult[] = criteria.map((criterion) => {
+    try {
+      const result = criterion.validator(artifact);
+      return {
+        name: criterion.name,
+        weight: criterion.weight,
+        score: Math.max(0, Math.min(1, result.score)), // Clamp to [0, 1]
+        reason: result.reason,
+      };
+    } catch (error) {
+      // Fail closed on validator errors
+      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+      return {
+        name: criterion.name,
+        weight: criterion.weight,
+        score: 0,
+        reason: `Validator error: ${errorMessage}`,
+      };
+    }
+  });
+
+  // Calculate weighted average
+  const finalScore = criteriaResults.reduce(
+    (sum, result) => sum + result.weight * result.score,
+    0
+  );
+
+  return {
+    finalScore,
+    passed: finalScore >= passThreshold,
+    criteriaResults,
+  };
+}
+
+/**
+ * Calculates efficiency (score per token) for tiebreaker purposes.
+ *
+ * @param score - The weighted score achieved
+ * @param tokensUsed - Number of tokens used
+ * @returns Efficiency ratio (higher is better)
+ *
+ * @see Requirements 9.4
+ */
+export function calculateEfficiency(score: number, tokensUsed: number): number {
+  if (tokensUsed <= 0) {
+    return score > 0 ? Infinity : 0;
+  }
+  return score / tokensUsed;
+}
+
+/**
+ * Ranks leaderboard entries by cumulative weighted score.
+ *
+ * Primary sort: cumulative score (descending)
+ * Tiebreaker: efficiency (score per token, descending)
+ *
+ * @param entries - Array of leaderboard entries to rank
+ * @returns Sorted array with updated rank positions
+ *
+ * @see Requirements 9.3, 9.4
+ */
+export function rankLeaderboardEntries(
+  entries: WeightedLeaderboardEntry[]
+): WeightedLeaderboardEntry[] {
+  // Sort by cumulative score (descending), then by efficiency (descending) for ties
+  const sorted = [...entries].sort((a, b) => {
+    // Primary: cumulative score
+    const scoreDiff = b.cumulativeScore - a.cumulativeScore;
+    if (Math.abs(scoreDiff) > 0.0001) {
+      return scoreDiff;
+    }
+    // Tiebreaker: efficiency (score per token)
+    return b.efficiency - a.efficiency;
+  });
+
+  // Update ranks
+  return sorted.map((entry, index) => ({
+    ...entry,
+    rank: index + 1,
+  }));
+}
+
+/**
+ * Creates a leaderboard entry from battle results.
+ *
+ * @param agentId - Agent identifier
+ * @param agentName - Agent display name
+ * @param userId - Owner user ID
+ * @param battleResults - Array of battle results with scores and tokens
+ * @returns WeightedLeaderboardEntry with calculated fields
+ */
+export function createLeaderboardEntry(
+  agentId: string,
+  agentName: string,
+  userId: string,
+  battleResults: Array<{ score: number; tokensUsed: number; earnings: number }>
+): WeightedLeaderboardEntry {
+  const cumulativeScore = battleResults.reduce((sum, r) => sum + r.score, 0);
+  const totalTokensUsed = battleResults.reduce((sum, r) => sum + r.tokensUsed, 0);
+  const totalEarnings = battleResults.reduce((sum, r) => sum + r.earnings, 0);
+
+  return {
+    rank: 0, // Will be set by rankLeaderboardEntries
+    agentId,
+    agentName,
+    userId,
+    cumulativeScore,
+    totalTokensUsed,
+    efficiency: calculateEfficiency(cumulativeScore, totalTokensUsed),
+    battlesCount: battleResults.length,
+    totalEarnings,
+  };
+}

package/src/index.ts

@@ -0,0 +1,17 @@
+/**
+ * Earnd Bounty Engine
+ *
+ * Outcome-based AI agent competition system where business outcomes are defined as code,
+ * multiple agents compete to achieve outcomes, success is deterministically evaluated,
+ * and payment is only possible when success criteria are met.
+ */
+
+export const VERSION = '1.0.0';
+
+// Re-export modules with explicit naming to avoid conflicts
+export * as outcomes from './outcomes/index.js';
+export * as agents from './agents/index.js';
+export * as eval from './eval/index.js';
+export * as jobs from './jobs/index.js';
+export * as runtime from './runtime/index.js';
+export * as utils from './utils/index.js';

package/src/league/README.md

@@ -0,0 +1,188 @@
+# League Module
+
+The league module implements the parallel agent competition system where N agents compete to achieve an outcome. The first agent to succeed wins the bounty.
+
+## Components
+
+### League Runner (`runLeague.ts`)
+
+Main orchestration for parallel agent execution.
+
+**Key Interfaces:**
+
+- `LeagueConfig` - Configuration for a league run (outcomeId, agentCount, globalSpendCeiling, etc.)
+- `LeagueResult` - Result of a league run (winnerId, agents, totalCost, duration)
+- `AgentResult` - Result for individual agent (status, killReason, tokensSpent, evaluationResult)
+
+**Key Functions:**
+
+- `runLeague(config)` - Runs N agents in parallel competing for an outcome
+- `runLeagueMock(config)` - Runs league in mock mode without real API calls
+
+**Usage:**
+
+```typescript
+import { runLeague, runLeagueMock } from './runLeague.js';
+
+const result = await runLeague({
+  outcomeId: 'qualified_sales_interest',
+  agentCount: 3,
+  globalSpendCeiling: 50000,
+  agentConfigs: [agent1, agent2, agent3],
+  outcome: qualifiedSalesInterest,
+  lead: leadData,
+  apiKey: process.env.ANTHROPIC_API_KEY,
+});
+
+if (result.winnerId) {
+  console.log(`Winner: ${result.winnerId}`);
+  console.log(`Total cost: ${result.totalCost} tokens`);
+}
+```
+
+### Kill Agent (`killAgent.ts`)
+
+Agent termination logic based on limits.
+
+**Key Functions:**
+
+- `shouldKillAgent(agent, limits)` - Checks if agent should be terminated
+- `killAgent(agentId, reason)` - Terminates an agent
+- `checkAllAgents(agents, limits)` - Checks all agents for termination
+
+**Kill Reasons:**
+
+- `cost_exceeded` - Agent exceeded token ceiling
+- `attempts_exceeded` - Agent exceeded max attempts
+- `timeout` - Agent exceeded runtime limit
+- `competitor_won` - Another agent achieved success first
+
+### Score Agent (`scoreAgent.ts`)
+
+Agent scoring and winner determination.
+
+**Key Functions:**
+
+- `scoreAgent(result, metrics)` - Creates a score for an agent
+- `determineWinner(scores)` - Finds the winning agent
+- `rankAgents(scores)` - Ranks all agents by performance
+- `calculateLeagueStats(scores)` - Calculates aggregate statistics
+
+## League Execution Flow
+
+```text
+1. Start N agents in parallel
+2. Each agent attempts to achieve the outcome
+3. First agent to get SUCCESS evaluation wins
+4. All other agents are terminated with 'competitor_won'
+5. If no agent succeeds, league ends with no winner
+6. Global spend ceiling terminates all agents if exceeded
+```
+
+## Requirements Reference
+
+- **4.1** - Spin up N agents in parallel
+- **4.3** - Terminate agent on attempt limit exceeded
+- **4.4** - Terminate agent on cost ceiling exceeded
+- **4.5** - Promote winning agent and halt others
+- **10.2** - Enforce max runtime per agent
+- **10.3** - Enforce global spend ceiling
+
+### Multi-Step Orchestrator (`multi-step-orchestrator.ts`)
+
+Manages multi-step bounty execution with sequential dependent tasks.
+
+**Key Interfaces:**
+
+- `TaskNode` - A single task in a multi-step bounty (id, name, description, dependencies, validator)
+- `MultiStepBounty` - Bounty definition with task graph (tasks, finalTaskId, payoutAmount)
+- `TaskContext` - Context passed to agent for task execution (task, dependencyOutputs, bounty info)
+- `TaskExecutionResult` - Result of a single task (success, score, output, tokensUsed)
+- `MultiStepResult` - Overall bounty execution result (taskResults, skippedTaskIds, overallScore)
+- `MultiStepAgent` - Agent interface for multi-step execution
+
+**Error Classes:**
+
+- `CyclicDependencyError` - Thrown when task graph contains cycles
+- `InvalidDependencyError` - Thrown when task references non-existent dependency
+- `InvalidFinalTaskError` - Thrown when final task ID doesn't exist
+- `TaskExecutionError` - Thrown when a task fails during execution
+
+**Usage:**
+
+```typescript
+import { 
+  MultiStepBounty, 
+  TaskNode,
+  CyclicDependencyError 
+} from './multi-step-orchestrator.js';
+
+const bounty: MultiStepBounty = {
+  id: 'research-bounty',
+  name: 'Market Research',
+  description: 'Complete market research workflow',
+  tasks: [
+    { id: 'gather', name: 'Gather Data', dependencies: [], ... },
+    { id: 'analyze', name: 'Analyze Data', dependencies: ['gather'], ... },
+    { id: 'report', name: 'Generate Report', dependencies: ['analyze'], ... },
+  ],
+  finalTaskId: 'report',
+  payoutAmount: 500,
+};
+```
+
+### Team Coordinator (`team-coordinator.ts`)
+
+Manages team battles where multiple agents collaborate on a bounty.
+
+**Key Interfaces:**
+
+- `TeamConfig` - Team configuration (teamId, memberIds, sharedStateEnabled)
+- `TeamState` - Shared state with optimistic locking (data, version, lastModifiedBy)
+- `StateUpdateResult` - Result of state update operation
+- `MemberContribution` - Contribution metrics per team member
+- `TeamPayoutDistribution` - Payout distribution across team members
+- `TeamStateChangeEvent` - Event emitted on state changes
+
+**Error Classes:**
+
+- `StateConflictError` - Thrown when optimistic locking detects a conflict
+- `TeamNotFoundError` - Thrown when team doesn't exist
+- `NotTeamMemberError` - Thrown when agent isn't a team member
+
+**Usage:**
+
+```typescript
+import { 
+  TeamConfig, 
+  TeamState, 
+  StateConflictError 
+} from './team-coordinator.js';
+
+const team: TeamConfig = {
+  teamId: 'team-alpha',
+  memberIds: ['agent-1', 'agent-2', 'agent-3'],
+  sharedStateEnabled: true,
+};
+
+// Shared state uses optimistic locking
+const state: TeamState = {
+  data: { researchNotes: [], completedTasks: [] },
+  version: 1,
+  lastModifiedBy: 'agent-1',
+  lastModifiedAt: new Date(),
+};
+```
+
+## Design Principles
+
+1. **Parallel Execution** - All agents run simultaneously
+2. **First Win** - First successful agent wins, no ties
+3. **Fail Closed** - Limit violations terminate without payout
+4. **Isolation** - Each agent tracks costs independently (solo battles)
+5. **Observable** - All terminations and results are logged
+6. **Team Coordination** - Shared state with optimistic locking for team battles
+7. **Fair Distribution** - Contribution-based payout distribution for teams
+8. **DAG Validation** - Multi-step bounties validate task graphs for cycles
+9. **Output Propagation** - Task outputs flow to dependent tasks automatically
+10. **Partial Completion** - Multi-step bounties track which tasks completed on failure