outcome-cli 1.0.0

package/src/eval/immigration-validators.ts

@@ -0,0 +1,122 @@
+export interface ValidationResult {
+  valid: boolean;
+  errors: string[];
+}
+
+export type ArtifactContent = Record<string, unknown>;
+
+export function validateI983RequiredFields(
+  content: ArtifactContent,
+  params: { requiredFields: string[] }
+): ValidationResult {
+  const errors: string[] = [];
+  const extracted = (content.extractedFormData || content) as Record<string, unknown>;
+  
+  for (const field of params.requiredFields) {
+    const value = extracted[field];
+    if (!value || (typeof value === 'string' && value.trim() === '')) {
+      const fieldName = field.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
+      errors.push(`Missing required field: ${fieldName}`);
+    }
+  }
+  
+  return { valid: errors.length === 0, errors };
+}
+
+export function validateOPTDateRange(
+  content: ArtifactContent,
+  params: { field: string; minDaysFromNow: number; maxDaysFromNow: number }
+): ValidationResult {
+  const extracted = (content.extractedFormData || content) as Record<string, unknown>;
+  const dateStr = extracted[params.field];
+  
+  if (!dateStr) {
+    return { 
+      valid: false, 
+      errors: [`Missing date field: ${params.field}`] 
+    };
+  }
+  
+  const date = new Date(String(dateStr));
+  const now = new Date();
+  now.setHours(0, 0, 0, 0);
+  date.setHours(0, 0, 0, 0);
+  
+  const daysDiff = Math.floor((date.getTime() - now.getTime()) / (1000 * 60 * 60 * 24));
+  
+  if (isNaN(date.getTime())) {
+    return { 
+      valid: false, 
+      errors: [`Invalid date format for ${params.field}. Please use MM/DD/YYYY format.`] 
+    };
+  }
+  
+  if (daysDiff < params.minDaysFromNow) {
+    return { 
+      valid: false, 
+      errors: [`${params.field} must be at least ${params.minDaysFromNow} days from today. Current date is ${daysDiff} days from today.`] 
+    };
+  }
+  
+  if (daysDiff > params.maxDaysFromNow) {
+    return { 
+      valid: false, 
+      errors: [`${params.field} must be within ${params.maxDaysFromNow} days from today. Current date is ${daysDiff} days from today.`] 
+    };
+  }
+  
+  return { valid: true, errors: [] };
+}
+
+export function validateEVerifyFormat(
+  content: ArtifactContent,
+  params: { field: string }
+): ValidationResult {
+  const extracted = (content.extractedFormData || content) as Record<string, unknown>;
+  const value = extracted[params.field];
+  
+  if (!value) {
+    return { 
+      valid: false, 
+      errors: ['Missing E-Verify number. Your employer must be registered with E-Verify.'] 
+    };
+  }
+  
+  const eVerifyRegex = /^\d{8}$/;
+  const cleanedValue = String(value).replace(/\D/g, '');
+  
+  if (!eVerifyRegex.test(cleanedValue)) {
+    return { 
+      valid: false, 
+      errors: [`E-Verify number must be exactly 8 digits (example: 12345678). Current value: ${value}`] 
+    };
+  }
+  
+  return { valid: true, errors: [] };
+}
+
+export function validateTrainingDescriptionLength(
+  content: ArtifactContent,
+  params: { field: string; minWords: number }
+): ValidationResult {
+  const extracted = (content.extractedFormData || content) as Record<string, unknown>;
+  const description = extracted[params.field];
+  
+  if (!description) {
+    return { 
+      valid: false, 
+      errors: ['Training description is required. Please provide a detailed training plan.'] 
+    };
+  }
+  
+  const wordCount = String(description).trim().split(/\s+/).filter(word => word.length > 0).length;
+  
+  if (wordCount < params.minWords) {
+    return { 
+      valid: false, 
+      errors: [`Training description must be at least ${params.minWords} words. Currently ${wordCount} words. Add more details about learning objectives, tasks, and outcomes.`] 
+    };
+  }
+  
+  return { valid: true, errors: [] };
+}

package/src/eval/index.ts

@@ -0,0 +1,90 @@
+/**
+ * Evaluation System
+ *
+ * Binary evaluation of agent artifacts against outcome success criteria.
+ * Now includes SWE-bench style code evaluation for engineering tasks.
+ * Also includes weighted scoring for granular evaluation.
+ *
+ * @module eval
+ */
+
+// Sales/Email validators (original)
+export {
+  type ValidationResult,
+  validateBuyingIntent,
+  validateCompanySize,
+  validateRole,
+  validateMessageLength,
+  validateEmail,
+  // Code Review validators (Tournament Seed Bounties)
+  validateSecurityIssue,
+  validatePerformanceIssue,
+  validateNoiseFreeness,
+  validateComplexityReduction,
+  validateExpertReview,
+  // Lead Gen validators (Tournament Seed Bounties)
+  validateLinkedIn,
+  validateLeadGenPrecision,
+} from './validators.js';
+
+// Core evaluation
+export {
+  type CriterionResult,
+  type ArtifactContent,
+  type AgentArtifact,
+  type EvaluationResult,
+  evaluateOutcome,
+} from './evaluateOutcome.js';
+
+// SWE-bench style code validators
+export {
+  type CodeArtifactContent,
+  type TestCase,
+  validateCodeSyntax,
+  validateCodeStructure,
+  validateTestCases,
+  validatePatchRelevance,
+  validateCodeQuality,
+  evaluateSWEBenchStyle,
+} from './code-validators.js';
+
+// Weighted scoring system
+export {
+  type WeightedValidationResult,
+  type WeightedCriterion,
+  type CriterionEvaluationResult,
+  type ScoringResult,
+  type WeightedLeaderboardEntry,
+  validateWeights,
+  calculateWeightedScore,
+  calculateEfficiency,
+  rankLeaderboardEntries,
+  createLeaderboardEntry,
+} from './weighted-scorer.js';
+
+// AI-powered evaluation (AI Judge)
+export {
+  type JudgeConfig,
+  AIJudgeError,
+  hashArtifact,
+  evaluateWithAIJudge,
+  validateJudgeConfig,
+} from './ai-judge.js';
+
+// Judge result caching
+export {
+  type JudgeModel,
+  type JudgeResult,
+  type JudgeCache,
+  type CacheEntry,
+  type CacheStats,
+  type RedisClient,
+  InMemoryJudgeCache,
+  KVJudgeCache,
+  RedisJudgeCache,
+  getJudgeCache,
+  setJudgeCache,
+  resetJudgeCache,
+  createKVCache,
+  createRedisCache,
+} from './judge-cache.js';

package/src/eval/judge-cache.ts

@@ -0,0 +1,402 @@
+/**
+ * AI Judge Result Caching
+ *
+ * Provides caching for AI judge evaluation results to ensure idempotence.
+ * Identical artifacts evaluated with the same rubric return cached scores.
+ *
+ * Supports multiple cache backends:
+ * - In-memory (default, for development/testing)
+ * - KV (Cloudflare Workers KV)
+ * - Redis (for production deployments)
+ *
+ * @module eval/judge-cache
+ * @see Requirements 10.4
+ */
+
+/**
+ * Supported AI judge models.
+ */
+export type JudgeModel = 'gpt-4o' | 'claude-opus';
+
+/**
+ * Result from an AI judge evaluation.
+ * Defined here to avoid circular dependency with ai-judge.ts.
+ */
+export interface JudgeResult {
+  /** Numeric score from 0 to maxScore */
+  score: number;
+  /** Normalized score from 0.0 to 1.0 */
+  normalizedScore: number;
+  /** Detailed reasoning for the score */
+  reasoning: string;
+  /** Key highlights or notable aspects of the artifact */
+  highlights: string[];
+  /** The model that performed the evaluation */
+  model: JudgeModel;
+  /** Whether this result was retrieved from cache */
+  cached: boolean;
+  /** Timestamp of evaluation */
+  evaluatedAt: string;
+}
+
+/**
+ * Cache entry with metadata.
+ */
+export interface CacheEntry {
+  /** The cached judge result */
+  result: JudgeResult;
+  /** Timestamp when the entry was created */
+  createdAt: string;
+  /** Number of times this entry has been accessed */
+  accessCount: number;
+}
+
+/**
+ * Interface for judge result cache implementations.
+ *
+ * @see Requirements 10.4
+ */
+export interface JudgeCache {
+  /**
+   * Retrieves a cached judge result by key.
+   *
+   * @param key - Cache key (hash of artifact + rubric)
+   * @returns Cached JudgeResult or null if not found
+   */
+  get(key: string): Promise<JudgeResult | null>;
+
+  /**
+   * Stores a judge result in the cache.
+   *
+   * @param key - Cache key (hash of artifact + rubric)
+   * @param result - Judge result to cache
+   * @param ttlSeconds - Optional TTL in seconds (default: 7 days)
+   */
+  set(key: string, result: JudgeResult, ttlSeconds?: number): Promise<void>;
+
+  /**
+   * Removes a cached result.
+   *
+   * @param key - Cache key to remove
+   * @returns true if entry was removed, false if not found
+   */
+  delete(key: string): Promise<boolean>;
+
+  /**
+   * Clears all cached results.
+   */
+  clear(): Promise<void>;
+
+  /**
+   * Gets cache statistics.
+   */
+  stats(): Promise<CacheStats>;
+}
+
+/**
+ * Cache statistics.
+ */
+export interface CacheStats {
+  /** Number of entries in the cache */
+  size: number;
+  /** Number of cache hits */
+  hits: number;
+  /** Number of cache misses */
+  misses: number;
+  /** Hit rate (hits / (hits + misses)) */
+  hitRate: number;
+}
+
+/**
+ * In-memory cache implementation for development and testing.
+ *
+ * @see Requirements 10.4
+ */
+export class InMemoryJudgeCache implements JudgeCache {
+  private cache: Map<string, CacheEntry> = new Map();
+  private hits = 0;
+  private misses = 0;
+  private defaultTtlSeconds: number;
+
+  constructor(defaultTtlSeconds: number = 7 * 24 * 60 * 60) {
+    this.defaultTtlSeconds = defaultTtlSeconds;
+  }
+
+  async get(key: string): Promise<JudgeResult | null> {
+    const entry = this.cache.get(key);
+
+    if (!entry) {
+      this.misses++;
+      return null;
+    }
+
+    // Check if entry has expired
+    const createdAt = new Date(entry.createdAt).getTime();
+    const now = Date.now();
+    const ageSeconds = (now - createdAt) / 1000;
+
+    if (ageSeconds > this.defaultTtlSeconds) {
+      this.cache.delete(key);
+      this.misses++;
+      return null;
+    }
+
+    // Update access count
+    entry.accessCount++;
+    this.hits++;
+
+    return entry.result;
+  }
+
+  async set(key: string, result: JudgeResult, _ttlSeconds?: number): Promise<void> {
+    const entry: CacheEntry = {
+      result,
+      createdAt: new Date().toISOString(),
+      accessCount: 0,
+    };
+    this.cache.set(key, entry);
+  }
+
+  async delete(key: string): Promise<boolean> {
+    return this.cache.delete(key);
+  }
+
+  async clear(): Promise<void> {
+    this.cache.clear();
+    this.hits = 0;
+    this.misses = 0;
+  }
+
+  async stats(): Promise<CacheStats> {
+    const total = this.hits + this.misses;
+    return {
+      size: this.cache.size,
+      hits: this.hits,
+      misses: this.misses,
+      hitRate: total > 0 ? this.hits / total : 0,
+    };
+  }
+}
+
+/**
+ * Cloudflare Workers KV cache implementation.
+ *
+ * @see Requirements 10.4
+ */
+export class KVJudgeCache implements JudgeCache {
+  private kv: KVNamespace;
+  private prefix: string;
+  private hits = 0;
+  private misses = 0;
+  private defaultTtlSeconds: number;
+
+  constructor(
+    kv: KVNamespace,
+    prefix: string = 'judge:',
+    defaultTtlSeconds: number = 7 * 24 * 60 * 60
+  ) {
+    this.kv = kv;
+    this.prefix = prefix;
+    this.defaultTtlSeconds = defaultTtlSeconds;
+  }
+
+  private getKey(key: string): string {
+    return `${this.prefix}${key}`;
+  }
+
+  async get(key: string): Promise<JudgeResult | null> {
+    const fullKey = this.getKey(key);
+    const data = await this.kv.get(fullKey, 'json');
+
+    if (!data) {
+      this.misses++;
+      return null;
+    }
+
+    this.hits++;
+    return data as JudgeResult;
+  }
+
+  async set(key: string, result: JudgeResult, ttlSeconds?: number): Promise<void> {
+    const fullKey = this.getKey(key);
+    const ttl = ttlSeconds ?? this.defaultTtlSeconds;
+
+    await this.kv.put(fullKey, JSON.stringify(result), {
+      expirationTtl: ttl,
+    });
+  }
+
+  async delete(key: string): Promise<boolean> {
+    const fullKey = this.getKey(key);
+    await this.kv.delete(fullKey);
+    return true; // KV doesn't return whether key existed
+  }
+
+  async clear(): Promise<void> {
+    // KV doesn't support bulk delete, so we list and delete
+    const list = await this.kv.list({ prefix: this.prefix });
+    await Promise.all(list.keys.map((k) => this.kv.delete(k.name)));
+    this.hits = 0;
+    this.misses = 0;
+  }
+
+  async stats(): Promise<CacheStats> {
+    const list = await this.kv.list({ prefix: this.prefix });
+    const total = this.hits + this.misses;
+    return {
+      size: list.keys.length,
+      hits: this.hits,
+      misses: this.misses,
+      hitRate: total > 0 ? this.hits / total : 0,
+    };
+  }
+}
+
+/**
+ * Redis cache implementation for production deployments.
+ *
+ * @see Requirements 10.4
+ */
+export class RedisJudgeCache implements JudgeCache {
+  private redis: RedisClient;
+  private prefix: string;
+  private hits = 0;
+  private misses = 0;
+  private defaultTtlSeconds: number;
+
+  constructor(
+    redis: RedisClient,
+    prefix: string = 'judge:',
+    defaultTtlSeconds: number = 7 * 24 * 60 * 60
+  ) {
+    this.redis = redis;
+    this.prefix = prefix;
+    this.defaultTtlSeconds = defaultTtlSeconds;
+  }
+
+  private getKey(key: string): string {
+    return `${this.prefix}${key}`;
+  }
+
+  async get(key: string): Promise<JudgeResult | null> {
+    const fullKey = this.getKey(key);
+    const data = await this.redis.get(fullKey);
+
+    if (!data) {
+      this.misses++;
+      return null;
+    }
+
+    this.hits++;
+    return JSON.parse(data) as JudgeResult;
+  }
+
+  async set(key: string, result: JudgeResult, ttlSeconds?: number): Promise<void> {
+    const fullKey = this.getKey(key);
+    const ttl = ttlSeconds ?? this.defaultTtlSeconds;
+
+    await this.redis.setex(fullKey, ttl, JSON.stringify(result));
+  }
+
+  async delete(key: string): Promise<boolean> {
+    const fullKey = this.getKey(key);
+    const deleted = await this.redis.del(fullKey);
+    return deleted > 0;
+  }
+
+  async clear(): Promise<void> {
+    const keys = await this.redis.keys(`${this.prefix}*`);
+    if (keys.length > 0) {
+      await this.redis.del(...keys);
+    }
+    this.hits = 0;
+    this.misses = 0;
+  }
+
+  async stats(): Promise<CacheStats> {
+    const keys = await this.redis.keys(`${this.prefix}*`);
+    const total = this.hits + this.misses;
+    return {
+      size: keys.length,
+      hits: this.hits,
+      misses: this.misses,
+      hitRate: total > 0 ? this.hits / total : 0,
+    };
+  }
+}
+
+/**
+ * Minimal Redis client interface for type safety.
+ * Compatible with ioredis and node-redis.
+ */
+export interface RedisClient {
+  get(key: string): Promise<string | null>;
+  setex(key: string, seconds: number, value: string): Promise<string>;
+  del(...keys: string[]): Promise<number>;
+  keys(pattern: string): Promise<string[]>;
+}
+
+/**
+ * Cloudflare Workers KV namespace interface.
+ */
+interface KVNamespace {
+  get(key: string, type: 'json'): Promise<unknown>;
+  put(key: string, value: string, options?: { expirationTtl?: number }): Promise<void>;
+  delete(key: string): Promise<void>;
+  list(options?: { prefix?: string }): Promise<{ keys: Array<{ name: string }> }>;
+}
+
+// Global cache instance
+let globalCache: JudgeCache | null = null;
+
+/**
+ * Gets the global judge cache instance.
+ * Creates an in-memory cache if not already initialized.
+ *
+ * @returns The global JudgeCache instance
+ */
+export function getJudgeCache(): JudgeCache {
+  if (!globalCache) {
+    globalCache = new InMemoryJudgeCache();
+  }
+  return globalCache;
+}
+
+/**
+ * Sets the global judge cache instance.
+ * Use this to configure a KV or Redis cache in production.
+ *
+ * @param cache - The cache instance to use globally
+ */
+export function setJudgeCache(cache: JudgeCache): void {
+  globalCache = cache;
+}
+
+/**
+ * Resets the global cache (primarily for testing).
+ */
+export function resetJudgeCache(): void {
+  globalCache = null;
+}
+
+/**
+ * Creates a KV-backed judge cache.
+ *
+ * @param kv - Cloudflare Workers KV namespace
+ * @param prefix - Key prefix (default: 'judge:')
+ * @returns KVJudgeCache instance
+ */
+export function createKVCache(kv: KVNamespace, prefix?: string): JudgeCache {
+  return new KVJudgeCache(kv, prefix);
+}
+
+/**
+ * Creates a Redis-backed judge cache.
+ *
+ * @param redis - Redis client instance
+ * @param prefix - Key prefix (default: 'judge:')
+ * @returns RedisJudgeCache instance
+ */
+export function createRedisCache(redis: RedisClient, prefix?: string): JudgeCache {
+  return new RedisJudgeCache(redis, prefix);
+}