cognitive-core 0.0.2 → 0.1.0

package/src/learning/effectiveness.ts

@@ -0,0 +1,428 @@
+/**
+ * Learning Effectiveness Tracker
+ *
+ * Measures whether the learning system is actually improving agent outcomes.
+ * Uses post-task reflection annotations rather than A/B replay evaluation.
+ *
+ * After each trajectory completes, the tracker annotates it with:
+ * - What knowledge was surfaced (playbooks, experiences)
+ * - Whether the surfaced knowledge was applied (inferred from trajectory)
+ * - Whether the outcome improved relative to similar unaided tasks
+ *
+ * These annotations accumulate into aggregate metrics that answer:
+ * "Is learning making agents better over time?"
+ *
+ * Inspired by Dash's eval framework but adapted for domain-agnostic
+ * trajectory learning rather than text-to-SQL.
+ */
+
+import type { Trajectory } from '../types/index.js';
+import type { PlaybookMatch } from '../memory/playbook.js';
+import { JsonStore } from '../utils/storage.js';
+
+// === TYPES ===
+
+/**
+ * Annotation attached to a trajectory after completion.
+ * Records what knowledge was available and whether it helped.
+ */
+export interface TaskAnnotation {
+  id: string;
+  trajectoryId: string;
+  timestamp: Date;
+
+  /** What was surfaced before/during execution */
+  knowledgeSurfaced: {
+    playbookIds: string[];
+    playbookNames: string[];
+    experienceIds: string[];
+    /** Total number of knowledge items retrieved */
+    totalItems: number;
+  };
+
+  /** What was actually applied (inferred from trajectory analysis) */
+  knowledgeApplied: {
+    /** Playbook IDs whose tactics appeared in the trajectory steps */
+    playbookIdsUsed: string[];
+    /** Whether any surfaced knowledge was used */
+    anyKnowledgeUsed: boolean;
+  };
+
+  /** Outcome metrics for this task */
+  outcome: {
+    success: boolean;
+    stepCount: number;
+    /** Number of error-recovery cycles in the trajectory */
+    errorRecoveries: number;
+    /** Whether the task domain had prior playbooks */
+    domainHadPlaybooks: boolean;
+  };
+
+  /** Optional human or agent reflection after task completion */
+  reflection?: {
+    /** Was the surfaced knowledge relevant? */
+    knowledgeRelevance: 'helpful' | 'irrelevant' | 'misleading' | 'not_assessed';
+    /** Free-form notes on what worked or didn't */
+    notes?: string;
+  };
+}
+
+/**
+ * Aggregate effectiveness metrics computed from annotations
+ */
+export interface EffectivenessMetrics {
+  /** Total tasks tracked */
+  totalTasks: number;
+
+  /** Tasks where knowledge was surfaced vs not */
+  guidedTasks: number;
+  unguidedTasks: number;
+
+  /** Success rates */
+  guidedSuccessRate: number;
+  unguidedSuccessRate: number;
+  /** Difference: positive means guidance helps */
+  successRateDelta: number;
+
+  /** Average step counts */
+  guidedAvgSteps: number;
+  unguidedAvgSteps: number;
+  /** Negative means guidance reduces steps (good) */
+  stepCountDelta: number;
+
+  /** Knowledge application rates */
+  knowledgeApplicationRate: number;
+  /** Of applied knowledge, how often did the task succeed? */
+  appliedKnowledgeSuccessRate: number;
+
+  /** Error recovery */
+  guidedAvgErrorRecoveries: number;
+  unguidedAvgErrorRecoveries: number;
+
+  /** Per-playbook effectiveness */
+  playbookEffectiveness: PlaybookEffectivenessEntry[];
+
+  /** Time window these metrics cover */
+  windowStart: Date;
+  windowEnd: Date;
+}
+
+/**
+ * Effectiveness metrics for a single playbook
+ */
+export interface PlaybookEffectivenessEntry {
+  playbookId: string;
+  playbookName: string;
+  /** Times this playbook was surfaced */
+  surfacedCount: number;
+  /** Times this playbook was actually applied */
+  appliedCount: number;
+  /** Success rate when applied */
+  appliedSuccessRate: number;
+  /** Average step count when applied */
+  appliedAvgSteps: number;
+  /** How often agents found this knowledge relevant */
+  relevanceRate: number;
+}
+
+// === TRACKER ===
+
+/**
+ * Tracks learning effectiveness through post-task annotations
+ */
+export class LearningEffectivenessTracker {
+  private store: JsonStore<TaskAnnotation>;
+  private initialized = false;
+
+  constructor(baseDir: string) {
+    this.store = new JsonStore<TaskAnnotation>(baseDir, 'effectiveness', {
+      autoSaveInterval: 30000,
+      pretty: true,
+    });
+  }
+
+  async init(): Promise<void> {
+    if (this.initialized) return;
+    await this.store.init();
+    this.initialized = true;
+  }
+
+  /**
+   * Annotate a completed trajectory with knowledge usage data.
+   * Call this after a task finishes and usage inference has run.
+   */
+  async annotate(
+    trajectory: Trajectory,
+    surfacedPlaybooks: PlaybookMatch[],
+    surfacedExperienceIds: string[],
+    appliedPlaybookIds: string[],
+  ): Promise<TaskAnnotation> {
+    await this.init();
+
+    const errorRecoveries = countErrorRecoveries(trajectory);
+
+    const annotation: TaskAnnotation = {
+      id: `ann-${trajectory.id}`,
+      trajectoryId: trajectory.id,
+      timestamp: new Date(),
+      knowledgeSurfaced: {
+        playbookIds: surfacedPlaybooks.map((m) => m.playbook.id),
+        playbookNames: surfacedPlaybooks.map((m) => m.playbook.name),
+        experienceIds: surfacedExperienceIds,
+        totalItems: surfacedPlaybooks.length + surfacedExperienceIds.length,
+      },
+      knowledgeApplied: {
+        playbookIdsUsed: appliedPlaybookIds,
+        anyKnowledgeUsed: appliedPlaybookIds.length > 0,
+      },
+      outcome: {
+        success: trajectory.outcome.success,
+        stepCount: trajectory.steps.length,
+        errorRecoveries,
+        domainHadPlaybooks: surfacedPlaybooks.length > 0,
+      },
+    };
+
+    this.store.set(annotation.id, annotation);
+    await this.store.save(annotation.id);
+
+    return annotation;
+  }
+
+  /**
+   * Add a reflection to an existing annotation.
+   * Called after human review or agent self-reflection.
+   */
+  async addReflection(
+    annotationId: string,
+    reflection: TaskAnnotation['reflection'],
+  ): Promise<void> {
+    await this.init();
+
+    const annotation = this.store.get(annotationId);
+    if (!annotation) return;
+
+    annotation.reflection = reflection;
+    this.store.set(annotationId, annotation);
+    await this.store.save(annotationId);
+  }
+
+  /**
+   * Compute aggregate effectiveness metrics over a time window.
+   * Defaults to all time if no window specified.
+   */
+  async computeMetrics(options?: {
+    since?: Date;
+    until?: Date;
+    domain?: string;
+  }): Promise<EffectivenessMetrics> {
+    await this.init();
+
+    let annotations = this.store.values();
+
+    // Filter by time window
+    if (options?.since) {
+      const since = options.since.getTime();
+      annotations = annotations.filter(
+        (a) => new Date(a.timestamp).getTime() >= since
+      );
+    }
+    if (options?.until) {
+      const until = options.until.getTime();
+      annotations = annotations.filter(
+        (a) => new Date(a.timestamp).getTime() <= until
+      );
+    }
+
+    const guided = annotations.filter((a) => a.knowledgeSurfaced.totalItems > 0);
+    const unguided = annotations.filter((a) => a.knowledgeSurfaced.totalItems === 0);
+
+    const guidedSuccesses = guided.filter((a) => a.outcome.success);
+    const unguidedSuccesses = unguided.filter((a) => a.outcome.success);
+
+    const guidedSuccessRate = guided.length > 0
+      ? guidedSuccesses.length / guided.length
+      : 0;
+    const unguidedSuccessRate = unguided.length > 0
+      ? unguidedSuccesses.length / unguided.length
+      : 0;
+
+    const guidedAvgSteps = guided.length > 0
+      ? guided.reduce((sum, a) => sum + a.outcome.stepCount, 0) / guided.length
+      : 0;
+    const unguidedAvgSteps = unguided.length > 0
+      ? unguided.reduce((sum, a) => sum + a.outcome.stepCount, 0) / unguided.length
+      : 0;
+
+    const applied = annotations.filter((a) => a.knowledgeApplied.anyKnowledgeUsed);
+    const appliedSuccesses = applied.filter((a) => a.outcome.success);
+
+    const knowledgeApplicationRate = guided.length > 0
+      ? applied.length / guided.length
+      : 0;
+    const appliedKnowledgeSuccessRate = applied.length > 0
+      ? appliedSuccesses.length / applied.length
+      : 0;
+
+    const guidedAvgErrors = guided.length > 0
+      ? guided.reduce((sum, a) => sum + a.outcome.errorRecoveries, 0) / guided.length
+      : 0;
+    const unguidedAvgErrors = unguided.length > 0
+      ? unguided.reduce((sum, a) => sum + a.outcome.errorRecoveries, 0) / unguided.length
+      : 0;
+
+    // Per-playbook effectiveness
+    const playbookEffectiveness = this.computePlaybookEffectiveness(annotations);
+
+    return {
+      totalTasks: annotations.length,
+      guidedTasks: guided.length,
+      unguidedTasks: unguided.length,
+      guidedSuccessRate,
+      unguidedSuccessRate,
+      successRateDelta: guidedSuccessRate - unguidedSuccessRate,
+      guidedAvgSteps,
+      unguidedAvgSteps,
+      stepCountDelta: guidedAvgSteps - unguidedAvgSteps,
+      knowledgeApplicationRate,
+      appliedKnowledgeSuccessRate,
+      guidedAvgErrorRecoveries: guidedAvgErrors,
+      unguidedAvgErrorRecoveries: unguidedAvgErrors,
+      playbookEffectiveness,
+      windowStart: options?.since ?? new Date(0),
+      windowEnd: options?.until ?? new Date(),
+    };
+  }
+
+  /**
+   * Get all annotations (for export/inspection)
+   */
+  async getAll(): Promise<TaskAnnotation[]> {
+    await this.init();
+    return this.store.values();
+  }
+
+  /**
+   * Get annotation by trajectory ID
+   */
+  async getByTrajectoryId(trajectoryId: string): Promise<TaskAnnotation | undefined> {
+    await this.init();
+    return this.store.values().find((a) => a.trajectoryId === trajectoryId);
+  }
+
+  /**
+   * Get annotation count
+   */
+  async count(): Promise<number> {
+    await this.init();
+    return this.store.size();
+  }
+
+  /**
+   * Close the store
+   */
+  async close(): Promise<void> {
+    await this.store.close();
+  }
+
+  // === PRIVATE ===
+
+  private computePlaybookEffectiveness(
+    annotations: TaskAnnotation[]
+  ): PlaybookEffectivenessEntry[] {
+    const playbookStats = new Map<
+      string,
+      {
+        name: string;
+        surfaced: number;
+        applied: number;
+        appliedSuccesses: number;
+        appliedStepSum: number;
+        relevantCount: number;
+        assessedCount: number;
+      }
+    >();
+
+    for (const ann of annotations) {
+      // Track surfaced
+      for (let i = 0; i < ann.knowledgeSurfaced.playbookIds.length; i++) {
+        const id = ann.knowledgeSurfaced.playbookIds[i];
+        const name = ann.knowledgeSurfaced.playbookNames[i] ?? id;
+
+        if (!playbookStats.has(id)) {
+          playbookStats.set(id, {
+            name,
+            surfaced: 0,
+            applied: 0,
+            appliedSuccesses: 0,
+            appliedStepSum: 0,
+            relevantCount: 0,
+            assessedCount: 0,
+          });
+        }
+
+        const stats = playbookStats.get(id)!;
+        stats.surfaced++;
+
+        const wasApplied = ann.knowledgeApplied.playbookIdsUsed.includes(id);
+        if (wasApplied) {
+          stats.applied++;
+          if (ann.outcome.success) stats.appliedSuccesses++;
+          stats.appliedStepSum += ann.outcome.stepCount;
+        }
+
+        if (ann.reflection) {
+          stats.assessedCount++;
+          if (ann.reflection.knowledgeRelevance === 'helpful') {
+            stats.relevantCount++;
+          }
+        }
+      }
+    }
+
+    return Array.from(playbookStats.entries())
+      .map(([id, stats]) => ({
+        playbookId: id,
+        playbookName: stats.name,
+        surfacedCount: stats.surfaced,
+        appliedCount: stats.applied,
+        appliedSuccessRate:
+          stats.applied > 0 ? stats.appliedSuccesses / stats.applied : 0,
+        appliedAvgSteps:
+          stats.applied > 0 ? stats.appliedStepSum / stats.applied : 0,
+        relevanceRate:
+          stats.assessedCount > 0 ? stats.relevantCount / stats.assessedCount : 0,
+      }))
+      .sort((a, b) => b.surfacedCount - a.surfacedCount);
+  }
+}
+
+// === HELPERS ===
+
+/**
+ * Count error-recovery cycles in a trajectory.
+ * An error recovery is when a step has an error observation
+ * followed by a subsequent step that doesn't.
+ */
+function countErrorRecoveries(trajectory: Trajectory): number {
+  let recoveries = 0;
+  for (let i = 0; i < trajectory.steps.length - 1; i++) {
+    const current = trajectory.steps[i];
+    const next = trajectory.steps[i + 1];
+    const hasError = current.observation?.toLowerCase().includes('error');
+    const nextOk = !next.observation?.toLowerCase().includes('error');
+    if (hasError && nextOk) {
+      recoveries++;
+    }
+  }
+  return recoveries;
+}
+
+/**
+ * Create a learning effectiveness tracker
+ */
+export function createEffectivenessTracker(
+  baseDir: string
+): LearningEffectivenessTracker {
+  return new LearningEffectivenessTracker(baseDir);
+}

package/src/learning/index.ts

@@ -0,0 +1,58 @@
+export {
+  TrajectoryAnalyzer,
+  createAnalyzer,
+  simpleCreditAssignment,
+  outcomeCreditAssignment,
+  getCreditAssignmentFn,
+  type AnalysisResult,
+  type ErrorPattern,
+  type TrainingExample,
+  type CreditAssignmentFn,
+  type AnalyzerConfig,
+} from './analyzer.js';
+
+export {
+  LearningPipeline,
+  createLearningPipeline,
+  type ProcessResult,
+  type BatchResult,
+} from './pipeline.js';
+
+// Playbook-based learning
+export {
+  PlaybookExtractor,
+  createPlaybookExtractor,
+  type ExtractedPlaybooks,
+  type PlaybookUpdate,
+  type PlaybookExtractorConfig,
+} from './playbook-extractor.js';
+
+export {
+  MetaLearner,
+  createMetaLearner,
+  type MetaLearnerConfig,
+} from './meta-learner.js';
+
+export {
+  PlaybookUsageInference,
+  createUsageInference,
+  type PlaybookUsageResult,
+  type UsageInferenceConfig,
+} from './usage-inference.js';
+
+// LLM-based extraction
+export {
+  LLMPlaybookExtractor,
+  createLLMExtractor,
+  type LLMExtractorConfig,
+  type LLMExtractionResult,
+} from './llm-extractor.js';
+
+// Effectiveness tracking
+export {
+  LearningEffectivenessTracker,
+  createEffectivenessTracker,
+  type TaskAnnotation,
+  type EffectivenessMetrics,
+  type PlaybookEffectivenessEntry,
+} from './effectiveness.js';