cognitive-core 0.0.2 → 0.1.2

package/src/runtime/flows/validation.ts

@@ -0,0 +1,381 @@
+import type { Task } from '../../types/index.js';
+import type { MemorySystem } from '../../memory/system.js';
+import type { AgentManager } from '../manager.js';
+import type { AgentResult } from '../types.js';
+
+/**
+ * Configuration for the validation flow
+ */
+export interface ValidationFlowConfig {
+  /** Agent type to use for execution */
+  agentType: string;
+  /** Number of runs per task for statistical significance */
+  runsPerTask?: number;
+  /** Whether to run comparison (with vs without knowledge) */
+  runComparison?: boolean;
+  /** Timeout per task in ms */
+  taskTimeout?: number;
+}
+
+/**
+ * Metrics for measuring improvement
+ */
+export interface ValidationMetrics {
+  /** Success rate (0-1) */
+  successRate: number;
+  /** Average execution time in ms */
+  avgTime: number;
+  /** Average tool calls per task */
+  avgToolCalls: number;
+  /** Standard deviation of time */
+  timeStdDev: number;
+  /** Total tasks run */
+  totalRuns: number;
+}
+
+/**
+ * Comparison result between with/without knowledge
+ */
+export interface ComparisonResult {
+  /** Metrics with knowledge injection */
+  withKnowledge: ValidationMetrics;
+  /** Metrics without knowledge (baseline) */
+  baseline: ValidationMetrics;
+  /** Improvement metrics */
+  improvement: {
+    successRateDelta: number;
+    timeDeltaPercent: number;
+    toolCallDeltaPercent: number;
+    /** Statistical significance (p-value approximation) */
+    isSignificant: boolean;
+  };
+}
+
+/**
+ * Result of validation flow
+ */
+export interface ValidationFlowResult {
+  /** Results for each task */
+  taskResults: Array<{
+    task: Task;
+    results: AgentResult[];
+    metrics: ValidationMetrics;
+  }>;
+  /** Aggregate metrics */
+  aggregateMetrics: ValidationMetrics;
+  /** Comparison if runComparison was true */
+  comparison?: ComparisonResult;
+}
+
+/**
+ * Validation Flow
+ * Orchestrates: Knowledge injection → Agent execution → Performance measurement
+ *
+ * This flow is for the "injection" direction:
+ * - Test how well learning is working
+ * - Compare performance with vs without knowledge
+ * - Measure improvement over time
+ */
+export class ValidationFlow {
+  private manager: AgentManager;
+  private config: ValidationFlowConfig;
+
+  constructor(
+    manager: AgentManager,
+    _memory: MemorySystem, // Reserved for future memory state tracking
+    config: ValidationFlowConfig
+  ) {
+    this.manager = manager;
+    this.config = {
+      runsPerTask: 1,
+      runComparison: true,
+      taskTimeout: 300000, // 5 minutes
+      ...config,
+    };
+  }
+
+  /**
+   * Validate on a single task
+   */
+  async validateTask(task: Task): Promise<{
+    task: Task;
+    results: AgentResult[];
+    metrics: ValidationMetrics;
+    comparison?: ComparisonResult;
+  }> {
+    const runsPerTask = this.config.runsPerTask ?? 1;
+    const results: AgentResult[] = [];
+    const baselineResults: AgentResult[] = [];
+
+    // Run with knowledge injection
+    for (let i = 0; i < runsPerTask; i++) {
+      const result = await this.manager.spawn({
+        agentType: this.config.agentType,
+        task,
+        timeout: this.config.taskTimeout,
+        captureToolCalls: true,
+      });
+      results.push(result);
+    }
+
+    // Run baseline if comparison enabled
+    if (this.config.runComparison) {
+      for (let i = 0; i < runsPerTask; i++) {
+        const result = await this.manager.spawnBaseline({
+          agentType: this.config.agentType,
+          task,
+          timeout: this.config.taskTimeout,
+          captureToolCalls: true,
+        });
+        baselineResults.push(result);
+      }
+    }
+
+    const metrics = this.calculateMetrics(results);
+    let comparison: ComparisonResult | undefined;
+
+    if (this.config.runComparison && baselineResults.length > 0) {
+      const baselineMetrics = this.calculateMetrics(baselineResults);
+      comparison = this.calculateComparison(metrics, baselineMetrics);
+    }
+
+    return { task, results, metrics, comparison };
+  }
+
+  /**
+   * Validate on multiple tasks
+   */
+  async validate(tasks: Task[]): Promise<ValidationFlowResult> {
+    const taskResults: ValidationFlowResult['taskResults'] = [];
+    const allResults: AgentResult[] = [];
+    const allBaselineResults: AgentResult[] = [];
+
+    for (const task of tasks) {
+      const result = await this.validateTask(task);
+      taskResults.push({
+        task: result.task,
+        results: result.results,
+        metrics: result.metrics,
+      });
+
+      allResults.push(...result.results);
+
+      // Collect baseline results for aggregate comparison
+      if (this.config.runComparison) {
+        // Re-run baseline for aggregate (or we could store from validateTask)
+        for (let i = 0; i < (this.config.runsPerTask ?? 1); i++) {
+          const baselineResult = await this.manager.spawnBaseline({
+            agentType: this.config.agentType,
+            task,
+            timeout: this.config.taskTimeout,
+            captureToolCalls: true,
+          });
+          allBaselineResults.push(baselineResult);
+        }
+      }
+    }
+
+    const aggregateMetrics = this.calculateMetrics(allResults);
+    let comparison: ComparisonResult | undefined;
+
+    if (this.config.runComparison && allBaselineResults.length > 0) {
+      const baselineMetrics = this.calculateMetrics(allBaselineResults);
+      comparison = this.calculateComparison(aggregateMetrics, baselineMetrics);
+    }
+
+    return {
+      taskResults,
+      aggregateMetrics,
+      comparison,
+    };
+  }
+
+  /**
+   * Quick validation - single run per task, with comparison
+   */
+  async quickValidate(tasks: Task[]): Promise<{
+    successRateWithKnowledge: number;
+    successRateBaseline: number;
+    improvement: number;
+    details: Array<{
+      task: Task;
+      withKnowledge: boolean;
+      baseline: boolean;
+    }>;
+  }> {
+    const details: Array<{
+      task: Task;
+      withKnowledge: boolean;
+      baseline: boolean;
+    }> = [];
+
+    for (const task of tasks) {
+      const [withKnowledge, baseline] = await Promise.all([
+        this.manager.spawn({
+          agentType: this.config.agentType,
+          task,
+          timeout: this.config.taskTimeout,
+        }),
+        this.manager.spawnBaseline({
+          agentType: this.config.agentType,
+          task,
+          timeout: this.config.taskTimeout,
+        }),
+      ]);
+
+      details.push({
+        task,
+        withKnowledge: withKnowledge.success,
+        baseline: baseline.success,
+      });
+    }
+
+    const successRateWithKnowledge =
+      details.filter((d) => d.withKnowledge).length / details.length;
+    const successRateBaseline =
+      details.filter((d) => d.baseline).length / details.length;
+
+    return {
+      successRateWithKnowledge,
+      successRateBaseline,
+      improvement: successRateWithKnowledge - successRateBaseline,
+      details,
+    };
+  }
+
+  /**
+   * Calculate metrics from results
+   */
+  private calculateMetrics(results: AgentResult[]): ValidationMetrics {
+    if (results.length === 0) {
+      return {
+        successRate: 0,
+        avgTime: 0,
+        avgToolCalls: 0,
+        timeStdDev: 0,
+        totalRuns: 0,
+      };
+    }
+
+    const successCount = results.filter((r) => r.success).length;
+    const times = results.map((r) => r.metrics.totalTime);
+    const toolCalls = results.map((r) => r.metrics.toolCallCount);
+
+    const avgTime = times.reduce((a, b) => a + b, 0) / times.length;
+    const avgToolCalls =
+      toolCalls.reduce((a, b) => a + b, 0) / toolCalls.length;
+
+    // Calculate standard deviation
+    const timeVariance =
+      times.reduce((sum, t) => sum + Math.pow(t - avgTime, 2), 0) /
+      times.length;
+    const timeStdDev = Math.sqrt(timeVariance);
+
+    return {
+      successRate: successCount / results.length,
+      avgTime,
+      avgToolCalls,
+      timeStdDev,
+      totalRuns: results.length,
+    };
+  }
+
+  /**
+   * Calculate comparison between two sets of metrics
+   */
+  private calculateComparison(
+    withKnowledge: ValidationMetrics,
+    baseline: ValidationMetrics
+  ): ComparisonResult {
+    const successRateDelta =
+      withKnowledge.successRate - baseline.successRate;
+
+    const timeDeltaPercent =
+      baseline.avgTime > 0
+        ? ((baseline.avgTime - withKnowledge.avgTime) / baseline.avgTime) * 100
+        : 0;
+
+    const toolCallDeltaPercent =
+      baseline.avgToolCalls > 0
+        ? ((baseline.avgToolCalls - withKnowledge.avgToolCalls) /
+            baseline.avgToolCalls) *
+          100
+        : 0;
+
+    // Simple significance test (would need proper statistical test in production)
+    // Using a rough heuristic: significant if improvement > 2 std devs
+    const isSignificant =
+      Math.abs(withKnowledge.avgTime - baseline.avgTime) >
+      2 * Math.max(withKnowledge.timeStdDev, baseline.timeStdDev);
+
+    return {
+      withKnowledge,
+      baseline,
+      improvement: {
+        successRateDelta,
+        timeDeltaPercent,
+        toolCallDeltaPercent,
+        isSignificant,
+      },
+    };
+  }
+
+  /**
+   * Generate a validation report
+   */
+  generateReport(result: ValidationFlowResult): string {
+    const lines: string[] = [
+      '# Atlas Validation Report',
+      '',
+      '## Aggregate Metrics',
+      `- Success Rate: ${(result.aggregateMetrics.successRate * 100).toFixed(1)}%`,
+      `- Avg Time: ${result.aggregateMetrics.avgTime.toFixed(0)}ms`,
+      `- Avg Tool Calls: ${result.aggregateMetrics.avgToolCalls.toFixed(1)}`,
+      `- Total Runs: ${result.aggregateMetrics.totalRuns}`,
+      '',
+    ];
+
+    if (result.comparison) {
+      const c = result.comparison;
+      lines.push(
+        '## Comparison (With Knowledge vs Baseline)',
+        '',
+        '| Metric | With Knowledge | Baseline | Improvement |',
+        '|--------|---------------|----------|-------------|',
+        `| Success Rate | ${(c.withKnowledge.successRate * 100).toFixed(1)}% | ${(c.baseline.successRate * 100).toFixed(1)}% | ${c.improvement.successRateDelta > 0 ? '+' : ''}${(c.improvement.successRateDelta * 100).toFixed(1)}% |`,
+        `| Avg Time | ${c.withKnowledge.avgTime.toFixed(0)}ms | ${c.baseline.avgTime.toFixed(0)}ms | ${c.improvement.timeDeltaPercent > 0 ? '+' : ''}${c.improvement.timeDeltaPercent.toFixed(1)}% faster |`,
+        `| Avg Tool Calls | ${c.withKnowledge.avgToolCalls.toFixed(1)} | ${c.baseline.avgToolCalls.toFixed(1)} | ${c.improvement.toolCallDeltaPercent > 0 ? '+' : ''}${c.improvement.toolCallDeltaPercent.toFixed(1)}% fewer |`,
+        '',
+        `Statistical Significance: ${c.improvement.isSignificant ? '✓ Yes' : '✗ No'}`,
+        ''
+      );
+    }
+
+    lines.push(
+      '## Per-Task Results',
+      ''
+    );
+
+    for (const taskResult of result.taskResults) {
+      lines.push(
+        `### ${taskResult.task.description.slice(0, 50)}...`,
+        `- Success Rate: ${(taskResult.metrics.successRate * 100).toFixed(1)}%`,
+        `- Avg Time: ${taskResult.metrics.avgTime.toFixed(0)}ms`,
+        ''
+      );
+    }
+
+    return lines.join('\n');
+  }
+}
+
+/**
+ * Create a validation flow
+ */
+export function createValidationFlow(
+  manager: AgentManager,
+  memory: MemorySystem,
+  config: ValidationFlowConfig
+): ValidationFlow {
+  return new ValidationFlow(manager, memory, config);
+}

package/src/runtime/index.ts

@@ -0,0 +1,75 @@
+// Types
+export type {
+  AgentMessage,
+  ToolCall,
+  AgentState,
+  AgentSession,
+  AgentSpawnConfig,
+  AgentResult,
+  AgentBackend,
+  KnowledgeInjector,
+  TrajectoryExtractor,
+  AgentObserverCallbacks,
+  AgentDelegate,
+  AgentDelegateOptions,
+  AgentDelegateResult,
+} from './types.js';
+
+// Manager
+export {
+  AgentManager,
+  createAgentManager,
+  DefaultKnowledgeInjector,
+  DefaultTrajectoryExtractor,
+} from './manager.js';
+
+// Backends
+export {
+  SubprocessBackend,
+  createSubprocessBackend,
+  claudeCodeConfig,
+  type SubprocessAgentConfig,
+  MockBackend,
+  createMockBackend,
+  type MockAgentBehavior,
+  LocalACPBackend,
+  ACPBackend,
+  createACPBackend,
+  claudeCodeACPConfig,
+  claudeCodeDirectConfig,
+  type ACPAgentConfig,
+  ACPProtocolHandler,
+  type ACPSessionBase,
+  MacroAgentBackendAdapter,
+  adaptMacroBackend,
+  createMacroAgentSource,
+  type MacroAgentBackendLike,
+  type MacroSessionLike,
+} from './backends/index.js';
+
+// Compute Provider
+export {
+  DefaultComputeProvider,
+  createComputeProvider,
+  type ComputeProvider,
+  type ComputeRequirements,
+  type ComputeTarget,
+  type ComputeSource,
+  type CostTier,
+} from './compute-provider.js';
+
+// Flows
+export {
+  LearningFlow,
+  createLearningFlow,
+  type LearningFlowConfig,
+  type LearningFlowResult,
+} from './flows/learning.js';
+
+export {
+  ValidationFlow,
+  createValidationFlow,
+  type ValidationFlowConfig,
+  type ValidationFlowResult,
+  type ValidationMetrics,
+} from './flows/validation.js';