cognitive-core 0.2.4 → 0.3.1

package/src/learning/effectiveness.ts

@@ -1,428 +0,0 @@
-/**
- * Learning Effectiveness Tracker
- *
- * Measures whether the learning system is actually improving agent outcomes.
- * Uses post-task reflection annotations rather than A/B replay evaluation.
- *
- * After each trajectory completes, the tracker annotates it with:
- * - What knowledge was surfaced (playbooks, experiences)
- * - Whether the surfaced knowledge was applied (inferred from trajectory)
- * - Whether the outcome improved relative to similar unaided tasks
- *
- * These annotations accumulate into aggregate metrics that answer:
- * "Is learning making agents better over time?"
- *
- * Inspired by Dash's eval framework but adapted for domain-agnostic
- * trajectory learning rather than text-to-SQL.
- */
-
-import type { Trajectory } from '../types/index.js';
-import type { PlaybookMatch } from '../memory/playbook.js';
-import { JsonStore } from '../utils/storage.js';
-
-// === TYPES ===
-
-/**
- * Annotation attached to a trajectory after completion.
- * Records what knowledge was available and whether it helped.
- */
-export interface TaskAnnotation {
-  id: string;
-  trajectoryId: string;
-  timestamp: Date;
-
-  /** What was surfaced before/during execution */
-  knowledgeSurfaced: {
-    playbookIds: string[];
-    playbookNames: string[];
-    experienceIds: string[];
-    /** Total number of knowledge items retrieved */
-    totalItems: number;
-  };
-
-  /** What was actually applied (inferred from trajectory analysis) */
-  knowledgeApplied: {
-    /** Playbook IDs whose tactics appeared in the trajectory steps */
-    playbookIdsUsed: string[];
-    /** Whether any surfaced knowledge was used */
-    anyKnowledgeUsed: boolean;
-  };
-
-  /** Outcome metrics for this task */
-  outcome: {
-    success: boolean;
-    stepCount: number;
-    /** Number of error-recovery cycles in the trajectory */
-    errorRecoveries: number;
-    /** Whether the task domain had prior playbooks */
-    domainHadPlaybooks: boolean;
-  };
-
-  /** Optional human or agent reflection after task completion */
-  reflection?: {
-    /** Was the surfaced knowledge relevant? */
-    knowledgeRelevance: 'helpful' | 'irrelevant' | 'misleading' | 'not_assessed';
-    /** Free-form notes on what worked or didn't */
-    notes?: string;
-  };
-}
-
-/**
- * Aggregate effectiveness metrics computed from annotations
- */
-export interface EffectivenessMetrics {
-  /** Total tasks tracked */
-  totalTasks: number;
-
-  /** Tasks where knowledge was surfaced vs not */
-  guidedTasks: number;
-  unguidedTasks: number;
-
-  /** Success rates */
-  guidedSuccessRate: number;
-  unguidedSuccessRate: number;
-  /** Difference: positive means guidance helps */
-  successRateDelta: number;
-
-  /** Average step counts */
-  guidedAvgSteps: number;
-  unguidedAvgSteps: number;
-  /** Negative means guidance reduces steps (good) */
-  stepCountDelta: number;
-
-  /** Knowledge application rates */
-  knowledgeApplicationRate: number;
-  /** Of applied knowledge, how often did the task succeed? */
-  appliedKnowledgeSuccessRate: number;
-
-  /** Error recovery */
-  guidedAvgErrorRecoveries: number;
-  unguidedAvgErrorRecoveries: number;
-
-  /** Per-playbook effectiveness */
-  playbookEffectiveness: PlaybookEffectivenessEntry[];
-
-  /** Time window these metrics cover */
-  windowStart: Date;
-  windowEnd: Date;
-}
-
-/**
- * Effectiveness metrics for a single playbook
- */
-export interface PlaybookEffectivenessEntry {
-  playbookId: string;
-  playbookName: string;
-  /** Times this playbook was surfaced */
-  surfacedCount: number;
-  /** Times this playbook was actually applied */
-  appliedCount: number;
-  /** Success rate when applied */
-  appliedSuccessRate: number;
-  /** Average step count when applied */
-  appliedAvgSteps: number;
-  /** How often agents found this knowledge relevant */
-  relevanceRate: number;
-}
-
-// === TRACKER ===
-
-/**
- * Tracks learning effectiveness through post-task annotations
- */
-export class LearningEffectivenessTracker {
-  private store: JsonStore<TaskAnnotation>;
-  private initialized = false;
-
-  constructor(baseDir: string) {
-    this.store = new JsonStore<TaskAnnotation>(baseDir, 'effectiveness', {
-      autoSaveInterval: 30000,
-      pretty: true,
-    });
-  }
-
-  async init(): Promise<void> {
-    if (this.initialized) return;
-    await this.store.init();
-    this.initialized = true;
-  }
-
-  /**
-   * Annotate a completed trajectory with knowledge usage data.
-   * Call this after a task finishes and usage inference has run.
-   */
-  async annotate(
-    trajectory: Trajectory,
-    surfacedPlaybooks: PlaybookMatch[],
-    surfacedExperienceIds: string[],
-    appliedPlaybookIds: string[],
-  ): Promise<TaskAnnotation> {
-    await this.init();
-
-    const errorRecoveries = countErrorRecoveries(trajectory);
-
-    const annotation: TaskAnnotation = {
-      id: `ann-${trajectory.id}`,
-      trajectoryId: trajectory.id,
-      timestamp: new Date(),
-      knowledgeSurfaced: {
-        playbookIds: surfacedPlaybooks.map((m) => m.playbook.id),
-        playbookNames: surfacedPlaybooks.map((m) => m.playbook.name),
-        experienceIds: surfacedExperienceIds,
-        totalItems: surfacedPlaybooks.length + surfacedExperienceIds.length,
-      },
-      knowledgeApplied: {
-        playbookIdsUsed: appliedPlaybookIds,
-        anyKnowledgeUsed: appliedPlaybookIds.length > 0,
-      },
-      outcome: {
-        success: trajectory.outcome.success,
-        stepCount: trajectory.steps.length,
-        errorRecoveries,
-        domainHadPlaybooks: surfacedPlaybooks.length > 0,
-      },
-    };
-
-    this.store.set(annotation.id, annotation);
-    await this.store.save(annotation.id);
-
-    return annotation;
-  }
-
-  /**
-   * Add a reflection to an existing annotation.
-   * Called after human review or agent self-reflection.
-   */
-  async addReflection(
-    annotationId: string,
-    reflection: TaskAnnotation['reflection'],
-  ): Promise<void> {
-    await this.init();
-
-    const annotation = this.store.get(annotationId);
-    if (!annotation) return;
-
-    annotation.reflection = reflection;
-    this.store.set(annotationId, annotation);
-    await this.store.save(annotationId);
-  }
-
-  /**
-   * Compute aggregate effectiveness metrics over a time window.
-   * Defaults to all time if no window specified.
-   */
-  async computeMetrics(options?: {
-    since?: Date;
-    until?: Date;
-    domain?: string;
-  }): Promise<EffectivenessMetrics> {
-    await this.init();
-
-    let annotations = this.store.values();
-
-    // Filter by time window
-    if (options?.since) {
-      const since = options.since.getTime();
-      annotations = annotations.filter(
-        (a) => new Date(a.timestamp).getTime() >= since
-      );
-    }
-    if (options?.until) {
-      const until = options.until.getTime();
-      annotations = annotations.filter(
-        (a) => new Date(a.timestamp).getTime() <= until
-      );
-    }
-
-    const guided = annotations.filter((a) => a.knowledgeSurfaced.totalItems > 0);
-    const unguided = annotations.filter((a) => a.knowledgeSurfaced.totalItems === 0);
-
-    const guidedSuccesses = guided.filter((a) => a.outcome.success);
-    const unguidedSuccesses = unguided.filter((a) => a.outcome.success);
-
-    const guidedSuccessRate = guided.length > 0
-      ? guidedSuccesses.length / guided.length
-      : 0;
-    const unguidedSuccessRate = unguided.length > 0
-      ? unguidedSuccesses.length / unguided.length
-      : 0;
-
-    const guidedAvgSteps = guided.length > 0
-      ? guided.reduce((sum, a) => sum + a.outcome.stepCount, 0) / guided.length
-      : 0;
-    const unguidedAvgSteps = unguided.length > 0
-      ? unguided.reduce((sum, a) => sum + a.outcome.stepCount, 0) / unguided.length
-      : 0;
-
-    const applied = annotations.filter((a) => a.knowledgeApplied.anyKnowledgeUsed);
-    const appliedSuccesses = applied.filter((a) => a.outcome.success);
-
-    const knowledgeApplicationRate = guided.length > 0
-      ? applied.length / guided.length
-      : 0;
-    const appliedKnowledgeSuccessRate = applied.length > 0
-      ? appliedSuccesses.length / applied.length
-      : 0;
-
-    const guidedAvgErrors = guided.length > 0
-      ? guided.reduce((sum, a) => sum + a.outcome.errorRecoveries, 0) / guided.length
-      : 0;
-    const unguidedAvgErrors = unguided.length > 0
-      ? unguided.reduce((sum, a) => sum + a.outcome.errorRecoveries, 0) / unguided.length
-      : 0;
-
-    // Per-playbook effectiveness
-    const playbookEffectiveness = this.computePlaybookEffectiveness(annotations);
-
-    return {
-      totalTasks: annotations.length,
-      guidedTasks: guided.length,
-      unguidedTasks: unguided.length,
-      guidedSuccessRate,
-      unguidedSuccessRate,
-      successRateDelta: guidedSuccessRate - unguidedSuccessRate,
-      guidedAvgSteps,
-      unguidedAvgSteps,
-      stepCountDelta: guidedAvgSteps - unguidedAvgSteps,
-      knowledgeApplicationRate,
-      appliedKnowledgeSuccessRate,
-      guidedAvgErrorRecoveries: guidedAvgErrors,
-      unguidedAvgErrorRecoveries: unguidedAvgErrors,
-      playbookEffectiveness,
-      windowStart: options?.since ?? new Date(0),
-      windowEnd: options?.until ?? new Date(),
-    };
-  }
-
-  /**
-   * Get all annotations (for export/inspection)
-   */
-  async getAll(): Promise<TaskAnnotation[]> {
-    await this.init();
-    return this.store.values();
-  }
-
-  /**
-   * Get annotation by trajectory ID
-   */
-  async getByTrajectoryId(trajectoryId: string): Promise<TaskAnnotation | undefined> {
-    await this.init();
-    return this.store.values().find((a) => a.trajectoryId === trajectoryId);
-  }
-
-  /**
-   * Get annotation count
-   */
-  async count(): Promise<number> {
-    await this.init();
-    return this.store.size();
-  }
-
-  /**
-   * Close the store
-   */
-  async close(): Promise<void> {
-    await this.store.close();
-  }
-
-  // === PRIVATE ===
-
-  private computePlaybookEffectiveness(
-    annotations: TaskAnnotation[]
-  ): PlaybookEffectivenessEntry[] {
-    const playbookStats = new Map<
-      string,
-      {
-        name: string;
-        surfaced: number;
-        applied: number;
-        appliedSuccesses: number;
-        appliedStepSum: number;
-        relevantCount: number;
-        assessedCount: number;
-      }
-    >();
-
-    for (const ann of annotations) {
-      // Track surfaced
-      for (let i = 0; i < ann.knowledgeSurfaced.playbookIds.length; i++) {
-        const id = ann.knowledgeSurfaced.playbookIds[i];
-        const name = ann.knowledgeSurfaced.playbookNames[i] ?? id;
-
-        if (!playbookStats.has(id)) {
-          playbookStats.set(id, {
-            name,
-            surfaced: 0,
-            applied: 0,
-            appliedSuccesses: 0,
-            appliedStepSum: 0,
-            relevantCount: 0,
-            assessedCount: 0,
-          });
-        }
-
-        const stats = playbookStats.get(id)!;
-        stats.surfaced++;
-
-        const wasApplied = ann.knowledgeApplied.playbookIdsUsed.includes(id);
-        if (wasApplied) {
-          stats.applied++;
-          if (ann.outcome.success) stats.appliedSuccesses++;
-          stats.appliedStepSum += ann.outcome.stepCount;
-        }
-
-        if (ann.reflection) {
-          stats.assessedCount++;
-          if (ann.reflection.knowledgeRelevance === 'helpful') {
-            stats.relevantCount++;
-          }
-        }
-      }
-    }
-
-    return Array.from(playbookStats.entries())
-      .map(([id, stats]) => ({
-        playbookId: id,
-        playbookName: stats.name,
-        surfacedCount: stats.surfaced,
-        appliedCount: stats.applied,
-        appliedSuccessRate:
-          stats.applied > 0 ? stats.appliedSuccesses / stats.applied : 0,
-        appliedAvgSteps:
-          stats.applied > 0 ? stats.appliedStepSum / stats.applied : 0,
-        relevanceRate:
-          stats.assessedCount > 0 ? stats.relevantCount / stats.assessedCount : 0,
-      }))
-      .sort((a, b) => b.surfacedCount - a.surfacedCount);
-  }
-}
-
-// === HELPERS ===
-
-/**
- * Count error-recovery cycles in a trajectory.
- * An error recovery is when a step has an error observation
- * followed by a subsequent step that doesn't.
- */
-function countErrorRecoveries(trajectory: Trajectory): number {
-  let recoveries = 0;
-  for (let i = 0; i < trajectory.steps.length - 1; i++) {
-    const current = trajectory.steps[i];
-    const next = trajectory.steps[i + 1];
-    const hasError = current.observation?.toLowerCase().includes('error');
-    const nextOk = !next.observation?.toLowerCase().includes('error');
-    if (hasError && nextOk) {
-      recoveries++;
-    }
-  }
-  return recoveries;
-}
-
-/**
- * Create a learning effectiveness tracker
- */
-export function createEffectivenessTracker(
-  baseDir: string
-): LearningEffectivenessTracker {
-  return new LearningEffectivenessTracker(baseDir);
-}

package/src/learning/energy-evaluator.ts

@@ -1,282 +0,0 @@
-/**
- * EnergyEvaluator — Ruvector-inspired energy-threshold batch trigger.
- *
- * Instead of a simple count threshold, the evaluator measures "incoherence
- * energy" from trajectory signals. A trajectory that introduces contradictions,
- * encounters a novel domain with no playbooks, or triggers a high error rate
- * can escalate to batch processing even before the count threshold is met.
- *
- * Inspired by ruvector's Compute Ladder (prime-radiant/execution/ladder.rs)
- * which uses energy-driven escalation with named transition reasons.
- */
-
-import type { Trajectory } from '../types/index.js';
-import type { InstantLoopResult } from './instant-loop.js';
-
-/**
- * Reasons why a batch was triggered — named transitions for audit.
- */
-export type EscalationReason =
-  | 'count_threshold'
-  | 'contradiction_detected'
-  | 'novel_domain'
-  | 'high_error_rate'
-  | 'pattern_shift'
-  | 'manual_override';
-
-/**
- * Signals extracted from a trajectory that contribute to energy.
- */
-export interface EnergySignal {
-  /** Whether a coherence contradiction was detected during instant loop */
-  contradictionDetected: boolean;
-  /** Whether this trajectory's domain has no existing playbooks */
-  novelDomain: boolean;
-  /** Error rate in the recent window exceeds threshold */
-  highErrorRate: boolean;
-  /** Significant shift in patterns (e.g., new tool usage, domain change) */
-  patternShift: boolean;
-}
-
-/**
- * Result of an energy evaluation.
- */
-export interface EnergyEvaluation {
-  /** Computed energy score (0.0 = calm, 1.0 = high incoherence) */
-  energy: number;
-  /** Whether the energy exceeds the batch trigger threshold */
-  shouldTriggerBatch: boolean;
-  /** Named reason(s) for the trigger decision */
-  reasons: EscalationReason[];
-}
-
-/**
- * Lane transition record for audit purposes.
- */
-export interface LaneTransition {
-  timestamp: Date;
-  fromPhase: 'immediate' | 'batch' | 'maintenance';
-  toPhase: 'immediate' | 'batch' | 'maintenance';
-  reasons: EscalationReason[];
-  energy: number;
-  trajectoryId: string;
-}
-
-/**
- * Configuration for the EnergyEvaluator.
- */
-export interface EnergyEvaluatorConfig {
-  /** Trajectories needed before count-based batch trigger (default: 10) */
-  countThreshold: number;
-  /** Trigger batch when a contradiction is detected (default: true) */
-  triggerOnContradiction: boolean;
-  /** Trigger batch when a novel domain is encountered (default: true) */
-  triggerOnNovelDomain: boolean;
-  /** Error rate threshold for triggering batch (default: 0.6) */
-  errorRateThreshold: number;
-  /** Minimum time between batches in ms — debounce (default: 30000) */
-  minIntervalMs: number;
-  /** Number of recent trajectories to consider for error rate (default: 5) */
-  recentWindowSize: number;
-  /** Energy threshold for triggering batch (0.0–1.0, default: 0.7) */
-  energyThreshold: number;
-  /** Energy contribution weights */
-  weights: {
-    contradiction: number;  // default: 0.4
-    novelDomain: number;    // default: 0.3
-    highErrorRate: number;  // default: 0.2
-    patternShift: number;   // default: 0.1
-  };
-}
-
-const DEFAULT_CONFIG: EnergyEvaluatorConfig = {
-  countThreshold: 10,
-  triggerOnContradiction: true,
-  triggerOnNovelDomain: true,
-  errorRateThreshold: 0.6,
-  minIntervalMs: 30_000,
-  recentWindowSize: 5,
-  energyThreshold: 0.7,
-  weights: {
-    contradiction: 0.4,
-    novelDomain: 0.3,
-    highErrorRate: 0.2,
-    patternShift: 0.1,
-  },
-};
-
-/**
- * EnergyEvaluator — determines when batch processing should be triggered
- * based on measured incoherence energy from trajectory signals.
- */
-export class EnergyEvaluator {
-  private config: EnergyEvaluatorConfig;
-  private pendingCount = 0;
-  private lastBatchTime = 0;
-  private recentOutcomes: boolean[] = [];
-  private knownDomains = new Set<string>();
-  private transitionLog: LaneTransition[] = [];
-
-  constructor(config?: Partial<EnergyEvaluatorConfig>) {
-    this.config = {
-      ...DEFAULT_CONFIG,
-      ...config,
-      weights: { ...DEFAULT_CONFIG.weights, ...config?.weights },
-    };
-  }
-
-  /**
-   * Record a trajectory and evaluate whether batch should be triggered.
-   */
-  evaluate(
-    trajectory: Trajectory,
-    _instantResult: InstantLoopResult,
-    signals?: Partial<EnergySignal>,
-  ): EnergyEvaluation {
-    this.pendingCount++;
-
-    // Track recent outcomes for error rate
-    this.recentOutcomes.push(trajectory.outcome.success);
-    if (this.recentOutcomes.length > this.config.recentWindowSize) {
-      this.recentOutcomes.shift();
-    }
-
-    // Build the energy signal
-    const domain = trajectory.task.domain ?? 'general';
-    const isNovelDomain = !this.knownDomains.has(domain);
-    this.knownDomains.add(domain);
-
-    const errorRate = this.recentOutcomes.length > 0
-      ? this.recentOutcomes.filter(s => !s).length / this.recentOutcomes.length
-      : 0;
-
-    const signal: EnergySignal = {
-      contradictionDetected: signals?.contradictionDetected ?? false,
-      novelDomain: isNovelDomain && (signals?.novelDomain ?? true),
-      highErrorRate: errorRate >= this.config.errorRateThreshold,
-      patternShift: signals?.patternShift ?? false,
-    };
-
-    // Compute energy
-    const energy = this.computeEnergy(signal);
-
-    // Determine reasons
-    const reasons: EscalationReason[] = [];
-
-    if (this.pendingCount >= this.config.countThreshold) {
-      reasons.push('count_threshold');
-    }
-    if (signal.contradictionDetected && this.config.triggerOnContradiction) {
-      reasons.push('contradiction_detected');
-    }
-    if (signal.novelDomain && this.config.triggerOnNovelDomain) {
-      reasons.push('novel_domain');
-    }
-    if (signal.highErrorRate) {
-      reasons.push('high_error_rate');
-    }
-    if (signal.patternShift) {
-      reasons.push('pattern_shift');
-    }
-
-    // Check debounce
-    const now = Date.now();
-    const withinDebounce = (now - this.lastBatchTime) < this.config.minIntervalMs;
-
-    // Trigger if count threshold reached OR energy exceeds threshold
-    const shouldTrigger = !withinDebounce && (
-      this.pendingCount >= this.config.countThreshold ||
-      energy >= this.config.energyThreshold
-    );
-
-    if (shouldTrigger && reasons.length > 0) {
-      this.transitionLog.push({
-        timestamp: new Date(),
-        fromPhase: 'immediate',
-        toPhase: 'batch',
-        reasons,
-        energy,
-        trajectoryId: trajectory.id,
-      });
-    }
-
-    return {
-      energy,
-      shouldTriggerBatch: shouldTrigger,
-      reasons: shouldTrigger ? reasons : [],
-    };
-  }
-
-  /**
-   * Compute energy from signal using weighted sum.
-   */
-  private computeEnergy(signal: EnergySignal): number {
-    const w = this.config.weights;
-    let energy = 0;
-
-    if (signal.contradictionDetected) energy += w.contradiction;
-    if (signal.novelDomain) energy += w.novelDomain;
-    if (signal.highErrorRate) energy += w.highErrorRate;
-    if (signal.patternShift) energy += w.patternShift;
-
-    return Math.min(1.0, energy);
-  }
-
-  /**
-   * Reset after a batch run completes.
-   */
-  resetAfterBatch(): void {
-    this.pendingCount = 0;
-    this.lastBatchTime = Date.now();
-  }
-
-  /**
-   * Force-trigger evaluation (for manual override).
-   */
-  forceEvaluate(): EnergyEvaluation {
-    return {
-      energy: 1.0,
-      shouldTriggerBatch: true,
-      reasons: ['manual_override'],
-    };
-  }
-
-  /**
-   * Seed known domains (e.g., from existing playbooks on init).
-   */
-  seedKnownDomains(domains: string[]): void {
-    for (const d of domains) {
-      this.knownDomains.add(d);
-    }
-  }
-
-  /**
-   * Get the current pending count.
-   */
-  getPendingCount(): number {
-    return this.pendingCount;
-  }
-
-  /**
-   * Get the transition log for audit/debugging.
-   */
-  getTransitionLog(): readonly LaneTransition[] {
-    return this.transitionLog;
-  }
-
-  /**
-   * Get current configuration.
-   */
-  getConfig(): Readonly<EnergyEvaluatorConfig> {
-    return this.config;
-  }
-}
-
-/**
- * Create an EnergyEvaluator.
- */
-export function createEnergyEvaluator(
-  config?: Partial<EnergyEvaluatorConfig>,
-): EnergyEvaluator {
-  return new EnergyEvaluator(config);
-}