cognitive-core 0.2.5 → 0.3.1

package/src/runtime/flows/validation.ts

@@ -1,381 +0,0 @@
-import type { Task } from '../../types/index.js';
-import type { MemorySystem } from '../../memory/system.js';
-import type { AgentManager } from '../manager.js';
-import type { AgentResult } from '../types.js';
-
-/**
- * Configuration for the validation flow
- */
-export interface ValidationFlowConfig {
-  /** Agent type to use for execution */
-  agentType: string;
-  /** Number of runs per task for statistical significance */
-  runsPerTask?: number;
-  /** Whether to run comparison (with vs without knowledge) */
-  runComparison?: boolean;
-  /** Timeout per task in ms */
-  taskTimeout?: number;
-}
-
-/**
- * Metrics for measuring improvement
- */
-export interface ValidationMetrics {
-  /** Success rate (0-1) */
-  successRate: number;
-  /** Average execution time in ms */
-  avgTime: number;
-  /** Average tool calls per task */
-  avgToolCalls: number;
-  /** Standard deviation of time */
-  timeStdDev: number;
-  /** Total tasks run */
-  totalRuns: number;
-}
-
-/**
- * Comparison result between with/without knowledge
- */
-export interface ComparisonResult {
-  /** Metrics with knowledge injection */
-  withKnowledge: ValidationMetrics;
-  /** Metrics without knowledge (baseline) */
-  baseline: ValidationMetrics;
-  /** Improvement metrics */
-  improvement: {
-    successRateDelta: number;
-    timeDeltaPercent: number;
-    toolCallDeltaPercent: number;
-    /** Statistical significance (p-value approximation) */
-    isSignificant: boolean;
-  };
-}
-
-/**
- * Result of validation flow
- */
-export interface ValidationFlowResult {
-  /** Results for each task */
-  taskResults: Array<{
-    task: Task;
-    results: AgentResult[];
-    metrics: ValidationMetrics;
-  }>;
-  /** Aggregate metrics */
-  aggregateMetrics: ValidationMetrics;
-  /** Comparison if runComparison was true */
-  comparison?: ComparisonResult;
-}
-
-/**
- * Validation Flow
- * Orchestrates: Knowledge injection → Agent execution → Performance measurement
- *
- * This flow is for the "injection" direction:
- * - Test how well learning is working
- * - Compare performance with vs without knowledge
- * - Measure improvement over time
- */
-export class ValidationFlow {
-  private manager: AgentManager;
-  private config: ValidationFlowConfig;
-
-  constructor(
-    manager: AgentManager,
-    _memory: MemorySystem, // Reserved for future memory state tracking
-    config: ValidationFlowConfig
-  ) {
-    this.manager = manager;
-    this.config = {
-      runsPerTask: 1,
-      runComparison: true,
-      taskTimeout: 300000, // 5 minutes
-      ...config,
-    };
-  }
-
-  /**
-   * Validate on a single task
-   */
-  async validateTask(task: Task): Promise<{
-    task: Task;
-    results: AgentResult[];
-    metrics: ValidationMetrics;
-    comparison?: ComparisonResult;
-  }> {
-    const runsPerTask = this.config.runsPerTask ?? 1;
-    const results: AgentResult[] = [];
-    const baselineResults: AgentResult[] = [];
-
-    // Run with knowledge injection
-    for (let i = 0; i < runsPerTask; i++) {
-      const result = await this.manager.spawn({
-        agentType: this.config.agentType,
-        task,
-        timeout: this.config.taskTimeout,
-        captureToolCalls: true,
-      });
-      results.push(result);
-    }
-
-    // Run baseline if comparison enabled
-    if (this.config.runComparison) {
-      for (let i = 0; i < runsPerTask; i++) {
-        const result = await this.manager.spawnBaseline({
-          agentType: this.config.agentType,
-          task,
-          timeout: this.config.taskTimeout,
-          captureToolCalls: true,
-        });
-        baselineResults.push(result);
-      }
-    }
-
-    const metrics = this.calculateMetrics(results);
-    let comparison: ComparisonResult | undefined;
-
-    if (this.config.runComparison && baselineResults.length > 0) {
-      const baselineMetrics = this.calculateMetrics(baselineResults);
-      comparison = this.calculateComparison(metrics, baselineMetrics);
-    }
-
-    return { task, results, metrics, comparison };
-  }
-
-  /**
-   * Validate on multiple tasks
-   */
-  async validate(tasks: Task[]): Promise<ValidationFlowResult> {
-    const taskResults: ValidationFlowResult['taskResults'] = [];
-    const allResults: AgentResult[] = [];
-    const allBaselineResults: AgentResult[] = [];
-
-    for (const task of tasks) {
-      const result = await this.validateTask(task);
-      taskResults.push({
-        task: result.task,
-        results: result.results,
-        metrics: result.metrics,
-      });
-
-      allResults.push(...result.results);
-
-      // Collect baseline results for aggregate comparison
-      if (this.config.runComparison) {
-        // Re-run baseline for aggregate (or we could store from validateTask)
-        for (let i = 0; i < (this.config.runsPerTask ?? 1); i++) {
-          const baselineResult = await this.manager.spawnBaseline({
-            agentType: this.config.agentType,
-            task,
-            timeout: this.config.taskTimeout,
-            captureToolCalls: true,
-          });
-          allBaselineResults.push(baselineResult);
-        }
-      }
-    }
-
-    const aggregateMetrics = this.calculateMetrics(allResults);
-    let comparison: ComparisonResult | undefined;
-
-    if (this.config.runComparison && allBaselineResults.length > 0) {
-      const baselineMetrics = this.calculateMetrics(allBaselineResults);
-      comparison = this.calculateComparison(aggregateMetrics, baselineMetrics);
-    }
-
-    return {
-      taskResults,
-      aggregateMetrics,
-      comparison,
-    };
-  }
-
-  /**
-   * Quick validation - single run per task, with comparison
-   */
-  async quickValidate(tasks: Task[]): Promise<{
-    successRateWithKnowledge: number;
-    successRateBaseline: number;
-    improvement: number;
-    details: Array<{
-      task: Task;
-      withKnowledge: boolean;
-      baseline: boolean;
-    }>;
-  }> {
-    const details: Array<{
-      task: Task;
-      withKnowledge: boolean;
-      baseline: boolean;
-    }> = [];
-
-    for (const task of tasks) {
-      const [withKnowledge, baseline] = await Promise.all([
-        this.manager.spawn({
-          agentType: this.config.agentType,
-          task,
-          timeout: this.config.taskTimeout,
-        }),
-        this.manager.spawnBaseline({
-          agentType: this.config.agentType,
-          task,
-          timeout: this.config.taskTimeout,
-        }),
-      ]);
-
-      details.push({
-        task,
-        withKnowledge: withKnowledge.success,
-        baseline: baseline.success,
-      });
-    }
-
-    const successRateWithKnowledge =
-      details.filter((d) => d.withKnowledge).length / details.length;
-    const successRateBaseline =
-      details.filter((d) => d.baseline).length / details.length;
-
-    return {
-      successRateWithKnowledge,
-      successRateBaseline,
-      improvement: successRateWithKnowledge - successRateBaseline,
-      details,
-    };
-  }
-
-  /**
-   * Calculate metrics from results
-   */
-  private calculateMetrics(results: AgentResult[]): ValidationMetrics {
-    if (results.length === 0) {
-      return {
-        successRate: 0,
-        avgTime: 0,
-        avgToolCalls: 0,
-        timeStdDev: 0,
-        totalRuns: 0,
-      };
-    }
-
-    const successCount = results.filter((r) => r.success).length;
-    const times = results.map((r) => r.metrics.totalTime);
-    const toolCalls = results.map((r) => r.metrics.toolCallCount);
-
-    const avgTime = times.reduce((a, b) => a + b, 0) / times.length;
-    const avgToolCalls =
-      toolCalls.reduce((a, b) => a + b, 0) / toolCalls.length;
-
-    // Calculate standard deviation
-    const timeVariance =
-      times.reduce((sum, t) => sum + Math.pow(t - avgTime, 2), 0) /
-      times.length;
-    const timeStdDev = Math.sqrt(timeVariance);
-
-    return {
-      successRate: successCount / results.length,
-      avgTime,
-      avgToolCalls,
-      timeStdDev,
-      totalRuns: results.length,
-    };
-  }
-
-  /**
-   * Calculate comparison between two sets of metrics
-   */
-  private calculateComparison(
-    withKnowledge: ValidationMetrics,
-    baseline: ValidationMetrics
-  ): ComparisonResult {
-    const successRateDelta =
-      withKnowledge.successRate - baseline.successRate;
-
-    const timeDeltaPercent =
-      baseline.avgTime > 0
-        ? ((baseline.avgTime - withKnowledge.avgTime) / baseline.avgTime) * 100
-        : 0;
-
-    const toolCallDeltaPercent =
-      baseline.avgToolCalls > 0
-        ? ((baseline.avgToolCalls - withKnowledge.avgToolCalls) /
-            baseline.avgToolCalls) *
-          100
-        : 0;
-
-    // Simple significance test (would need proper statistical test in production)
-    // Using a rough heuristic: significant if improvement > 2 std devs
-    const isSignificant =
-      Math.abs(withKnowledge.avgTime - baseline.avgTime) >
-      2 * Math.max(withKnowledge.timeStdDev, baseline.timeStdDev);
-
-    return {
-      withKnowledge,
-      baseline,
-      improvement: {
-        successRateDelta,
-        timeDeltaPercent,
-        toolCallDeltaPercent,
-        isSignificant,
-      },
-    };
-  }
-
-  /**
-   * Generate a validation report
-   */
-  generateReport(result: ValidationFlowResult): string {
-    const lines: string[] = [
-      '# Atlas Validation Report',
-      '',
-      '## Aggregate Metrics',
-      `- Success Rate: ${(result.aggregateMetrics.successRate * 100).toFixed(1)}%`,
-      `- Avg Time: ${result.aggregateMetrics.avgTime.toFixed(0)}ms`,
-      `- Avg Tool Calls: ${result.aggregateMetrics.avgToolCalls.toFixed(1)}`,
-      `- Total Runs: ${result.aggregateMetrics.totalRuns}`,
-      '',
-    ];
-
-    if (result.comparison) {
-      const c = result.comparison;
-      lines.push(
-        '## Comparison (With Knowledge vs Baseline)',
-        '',
-        '| Metric | With Knowledge | Baseline | Improvement |',
-        '|--------|---------------|----------|-------------|',
-        `| Success Rate | ${(c.withKnowledge.successRate * 100).toFixed(1)}% | ${(c.baseline.successRate * 100).toFixed(1)}% | ${c.improvement.successRateDelta > 0 ? '+' : ''}${(c.improvement.successRateDelta * 100).toFixed(1)}% |`,
-        `| Avg Time | ${c.withKnowledge.avgTime.toFixed(0)}ms | ${c.baseline.avgTime.toFixed(0)}ms | ${c.improvement.timeDeltaPercent > 0 ? '+' : ''}${c.improvement.timeDeltaPercent.toFixed(1)}% faster |`,
-        `| Avg Tool Calls | ${c.withKnowledge.avgToolCalls.toFixed(1)} | ${c.baseline.avgToolCalls.toFixed(1)} | ${c.improvement.toolCallDeltaPercent > 0 ? '+' : ''}${c.improvement.toolCallDeltaPercent.toFixed(1)}% fewer |`,
-        '',
-        `Statistical Significance: ${c.improvement.isSignificant ? '✓ Yes' : '✗ No'}`,
-        ''
-      );
-    }
-
-    lines.push(
-      '## Per-Task Results',
-      ''
-    );
-
-    for (const taskResult of result.taskResults) {
-      lines.push(
-        `### ${taskResult.task.description.slice(0, 50)}...`,
-        `- Success Rate: ${(taskResult.metrics.successRate * 100).toFixed(1)}%`,
-        `- Avg Time: ${taskResult.metrics.avgTime.toFixed(0)}ms`,
-        ''
-      );
-    }
-
-    return lines.join('\n');
-  }
-}
-
-/**
- * Create a validation flow
- */
-export function createValidationFlow(
-  manager: AgentManager,
-  memory: MemorySystem,
-  config: ValidationFlowConfig
-): ValidationFlow {
-  return new ValidationFlow(manager, memory, config);
-}

package/src/runtime/index.ts

@@ -1,75 +0,0 @@
-// Types
-export type {
-  AgentMessage,
-  ToolCall,
-  AgentState,
-  AgentSession,
-  AgentSpawnConfig,
-  AgentResult,
-  AgentBackend,
-  KnowledgeInjector,
-  TrajectoryExtractor,
-  AgentObserverCallbacks,
-  AgentDelegate,
-  AgentDelegateOptions,
-  AgentDelegateResult,
-} from './types.js';
-
-// Manager
-export {
-  AgentManager,
-  createAgentManager,
-  DefaultKnowledgeInjector,
-  DefaultTrajectoryExtractor,
-} from './manager.js';
-
-// Backends
-export {
-  SubprocessBackend,
-  createSubprocessBackend,
-  claudeCodeConfig,
-  type SubprocessAgentConfig,
-  MockBackend,
-  createMockBackend,
-  type MockAgentBehavior,
-  LocalACPBackend,
-  ACPBackend,
-  createACPBackend,
-  claudeCodeACPConfig,
-  claudeCodeDirectConfig,
-  type ACPAgentConfig,
-  ACPProtocolHandler,
-  type ACPSessionBase,
-  MacroAgentBackendAdapter,
-  adaptMacroBackend,
-  createMacroAgentSource,
-  type MacroAgentBackendLike,
-  type MacroSessionLike,
-} from './backends/index.js';
-
-// Compute Provider
-export {
-  DefaultComputeProvider,
-  createComputeProvider,
-  type ComputeProvider,
-  type ComputeRequirements,
-  type ComputeTarget,
-  type ComputeSource,
-  type CostTier,
-} from './compute-provider.js';
-
-// Flows
-export {
-  LearningFlow,
-  createLearningFlow,
-  type LearningFlowConfig,
-  type LearningFlowResult,
-} from './flows/learning.js';
-
-export {
-  ValidationFlow,
-  createValidationFlow,
-  type ValidationFlowConfig,
-  type ValidationFlowResult,
-  type ValidationMetrics,
-} from './flows/validation.js';