cognitive-core 0.0.1 → 0.1.0

package/src/memory/meta.ts

@@ -0,0 +1,506 @@
+/**
+ * Meta Memory - Learning about learning
+ * Tracks routing decisions, memory effectiveness, and execution patterns
+ * to improve future learning and routing decisions
+ */
+
+import type {
+  MetaObservation,
+  MetaStrategy,
+  RoutingAnalysis,
+  RetrievalAnalysis,
+  MetaStats,
+  MetaContext,
+  RoutingDecisionType,
+} from '../types/index.js';
+import { createMetaStrategy } from '../types/index.js';
+import { JsonStore } from '../utils/storage.js';
+
+/**
+ * Filter options for querying observations
+ */
+export interface ObservationFilter {
+  routingDecision?: RoutingDecisionType;
+  outcome?: 'success' | 'failure';
+  minDate?: Date;
+  maxDate?: Date;
+  limit?: number;
+}
+
+/**
+ * Meta Memory - Store for meta-learning observations and strategies
+ */
+export class MetaMemory {
+  private observations: JsonStore<MetaObservation>;
+  private strategies: JsonStore<MetaStrategy>;
+  private initialized = false;
+
+  constructor(baseDir: string) {
+    this.observations = new JsonStore<MetaObservation>(
+      baseDir,
+      'meta-observations',
+      {
+        autoSaveInterval: 60000,
+        pretty: true,
+      }
+    );
+    this.strategies = new JsonStore<MetaStrategy>(baseDir, 'meta-strategies', {
+      autoSaveInterval: 60000,
+      pretty: true,
+    });
+  }
+
+  async init(): Promise<void> {
+    if (this.initialized) return;
+    await Promise.all([this.observations.init(), this.strategies.init()]);
+    this.initialized = true;
+  }
+
+  // === OBSERVATION ===
+
+  /**
+   * Record a meta-observation after task completion
+   */
+  async recordObservation(observation: MetaObservation): Promise<void> {
+    await this.init();
+    this.observations.set(observation.id, observation);
+    await this.observations.save(observation.id);
+  }
+
+  /**
+   * Get observations for analysis
+   */
+  async getObservations(filter?: ObservationFilter): Promise<MetaObservation[]> {
+    await this.init();
+
+    let observations = this.observations.values();
+
+    if (filter?.routingDecision) {
+      observations = observations.filter(
+        (o) => o.routing.decision === filter.routingDecision
+      );
+    }
+
+    if (filter?.outcome !== undefined) {
+      const isSuccess = filter.outcome === 'success';
+      observations = observations.filter((o) => o.outcome.success === isSuccess);
+    }
+
+    if (filter?.minDate) {
+      observations = observations.filter(
+        (o) => new Date(o.timestamp) >= filter.minDate!
+      );
+    }
+
+    if (filter?.maxDate) {
+      observations = observations.filter(
+        (o) => new Date(o.timestamp) <= filter.maxDate!
+      );
+    }
+
+    // Sort by timestamp descending (most recent first)
+    observations.sort(
+      (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()
+    );
+
+    if (filter?.limit) {
+      observations = observations.slice(0, filter.limit);
+    }
+
+    return observations;
+  }
+
+  /**
+   * Get observation by trajectory ID
+   */
+  async getByTrajectoryId(
+    trajectoryId: string
+  ): Promise<MetaObservation | undefined> {
+    await this.init();
+    return this.observations
+      .values()
+      .find((o) => o.trajectoryId === trajectoryId);
+  }
+
+  // === ANALYSIS ===
+
+  /**
+   * Analyze routing effectiveness
+   * Returns: which routing decisions lead to success in which contexts
+   */
+  async analyzeRoutingEffectiveness(): Promise<RoutingAnalysis> {
+    await this.init();
+
+    const byDecision: Record<string, { total: number; successful: number }> = {
+      adapt: { total: 0, successful: 0 },
+      direct: { total: 0, successful: 0 },
+      explore: { total: 0, successful: 0 },
+      fallback: { total: 0, successful: 0 },
+    };
+
+    for (const observation of this.observations.values()) {
+      const decision = observation.routing.decision;
+      byDecision[decision].total++;
+      if (observation.outcome.success) {
+        byDecision[decision].successful++;
+      }
+    }
+
+    const recommendations: string[] = [];
+
+    // Generate recommendations based on analysis
+    for (const [decision, stats] of Object.entries(byDecision)) {
+      if (stats.total >= 5) {
+        const successRate = stats.successful / stats.total;
+        if (successRate < 0.5) {
+          recommendations.push(
+            `Consider reducing reliance on '${decision}' routing (${Math.round(successRate * 100)}% success rate)`
+          );
+        } else if (successRate > 0.8) {
+          recommendations.push(
+            `'${decision}' routing is highly effective (${Math.round(successRate * 100)}% success rate)`
+          );
+        }
+      }
+    }
+
+    return { byDecision, recommendations };
+  }
+
+  /**
+   * Analyze memory retrieval quality
+   * Returns: which retrieval patterns help vs mislead
+   */
+  async analyzeRetrievalQuality(): Promise<RetrievalAnalysis> {
+    await this.init();
+
+    const helpfulPatterns: Map<string, number> = new Map();
+    const misleadingPatterns: Map<string, number> = new Map();
+
+    for (const observation of this.observations.values()) {
+      const quality = observation.memoryUsage.retrievalQuality;
+      const playbooks = observation.memoryUsage.playbooksApplied;
+
+      for (const playbookId of playbooks) {
+        if (quality === 'helpful') {
+          helpfulPatterns.set(
+            playbookId,
+            (helpfulPatterns.get(playbookId) ?? 0) + 1
+          );
+        } else if (quality === 'misleading') {
+          misleadingPatterns.set(
+            playbookId,
+            (misleadingPatterns.get(playbookId) ?? 0) + 1
+          );
+        }
+      }
+    }
+
+    const recommendations: string[] = [];
+
+    // Find consistently misleading playbooks
+    for (const [playbookId, count] of misleadingPatterns) {
+      const helpfulCount = helpfulPatterns.get(playbookId) ?? 0;
+      if (count > helpfulCount && count >= 3) {
+        recommendations.push(
+          `Playbook ${playbookId} may need refinement (${count} misleading vs ${helpfulCount} helpful uses)`
+        );
+      }
+    }
+
+    return {
+      helpfulPatterns: Array.from(helpfulPatterns.keys()),
+      misleadingPatterns: Array.from(misleadingPatterns.keys()),
+      recommendations,
+    };
+  }
+
+  /**
+   * Generate meta-strategies from observations
+   * Called periodically to distill learning patterns
+   */
+  async generateMetaStrategies(): Promise<MetaStrategy[]> {
+    await this.init();
+
+    const newStrategies: MetaStrategy[] = [];
+    const observations = this.observations.values();
+
+    // Pattern 1: Low-confidence routing tends to need exploration
+    const lowConfidenceObs = observations.filter(
+      (o) => o.routing.confidence < 0.5
+    );
+    const lowConfidenceSuccessRate =
+      lowConfidenceObs.filter((o) => o.outcome.success).length /
+      (lowConfidenceObs.length || 1);
+
+    if (lowConfidenceObs.length >= 5 && lowConfidenceSuccessRate < 0.4) {
+      const existing = this.strategies
+        .values()
+        .find((s) => s.name === 'low-confidence-explore');
+      if (!existing) {
+        newStrategies.push(
+          createMetaStrategy({
+            name: 'low-confidence-explore',
+            condition: {
+              taskCharacteristics: [],
+              memoryState: ['low confidence routing'],
+            },
+            adjustment: {
+              routingBias: { explore: 0.2 },
+              retrievalModification: 'expand query scope',
+              executionHint: 'prefer decomposition',
+            },
+          })
+        );
+      }
+    }
+
+    // Pattern 2: Multi-file changes often need decomposition
+    const multiFileObs = observations.filter((o) =>
+      o.execution.decompositionUsed
+    );
+    const multiFileSuccessRate =
+      multiFileObs.filter((o) => o.outcome.success).length /
+      (multiFileObs.length || 1);
+
+    if (multiFileObs.length >= 5 && multiFileSuccessRate > 0.7) {
+      const existing = this.strategies
+        .values()
+        .find((s) => s.name === 'decomposition-effective');
+      if (!existing) {
+        newStrategies.push(
+          createMetaStrategy({
+            name: 'decomposition-effective',
+            condition: {
+              taskCharacteristics: ['multi-file change', 'complex task'],
+              memoryState: [],
+            },
+            adjustment: {
+              routingBias: {},
+              retrievalModification: '',
+              executionHint: 'use decomposition',
+            },
+          })
+        );
+      }
+    }
+
+    // Pattern 3: Refinement iterations correlate with success
+    const refinedObs = observations.filter(
+      (o) => o.execution.refinementIterations > 0
+    );
+    const refinedSuccessRate =
+      refinedObs.filter((o) => o.outcome.success).length /
+      (refinedObs.length || 1);
+
+    if (refinedObs.length >= 5 && refinedSuccessRate > 0.6) {
+      const existing = this.strategies
+        .values()
+        .find((s) => s.name === 'refinement-beneficial');
+      if (!existing) {
+        newStrategies.push(
+          createMetaStrategy({
+            name: 'refinement-beneficial',
+            condition: {
+              taskCharacteristics: [],
+              memoryState: ['moderate confidence'],
+            },
+            adjustment: {
+              routingBias: {},
+              retrievalModification: '',
+              executionHint: 'allow refinement iterations',
+            },
+          })
+        );
+      }
+    }
+
+    // Save new strategies
+    for (const strategy of newStrategies) {
+      this.strategies.set(strategy.id, strategy);
+      await this.strategies.save(strategy.id);
+    }
+
+    return newStrategies;
+  }
+
+  // === APPLICATION ===
+
+  /**
+   * Get routing adjustments for current context
+   */
+  async getRoutingAdjustments(context: MetaContext): Promise<MetaStrategy[]> {
+    await this.init();
+
+    const applicable: MetaStrategy[] = [];
+
+    for (const strategy of this.strategies.values()) {
+      // Check if task characteristics match
+      const taskMatch =
+        strategy.condition.taskCharacteristics.length === 0 ||
+        strategy.condition.taskCharacteristics.some((tc) =>
+          context.taskCharacteristics.some(
+            (ctc) => ctc.toLowerCase().includes(tc.toLowerCase())
+          )
+        );
+
+      // Check if memory state matches
+      const memoryMatch =
+        strategy.condition.memoryState.length === 0 ||
+        strategy.condition.memoryState.some((ms) =>
+          context.memoryState.some(
+            (cms) => cms.toLowerCase().includes(ms.toLowerCase())
+          )
+        );
+
+      if (taskMatch && memoryMatch) {
+        applicable.push(strategy);
+      }
+    }
+
+    return applicable;
+  }
+
+  /**
+   * Record strategy application result
+   */
+  async recordStrategyApplication(
+    strategyId: string,
+    success: boolean
+  ): Promise<void> {
+    await this.init();
+    const strategy = this.strategies.get(strategyId);
+    if (!strategy) return;
+
+    strategy.applicationCount++;
+    const alpha = 0.1;
+    strategy.successRate =
+      strategy.successRate * (1 - alpha) + (success ? 1 : 0) * alpha;
+    strategy.updatedAt = new Date();
+
+    this.strategies.set(strategyId, strategy);
+    await this.strategies.save(strategyId);
+  }
+
+  // === LIFECYCLE ===
+
+  /**
+   * Add a meta-strategy
+   */
+  async addStrategy(strategy: MetaStrategy): Promise<void> {
+    await this.init();
+    this.strategies.set(strategy.id, strategy);
+    await this.strategies.save(strategy.id);
+  }
+
+  /**
+   * Get a meta-strategy by ID
+   */
+  async getStrategy(id: string): Promise<MetaStrategy | undefined> {
+    await this.init();
+    return this.strategies.get(id);
+  }
+
+  /**
+   * Get all meta-strategies
+   */
+  async getAllStrategies(): Promise<MetaStrategy[]> {
+    await this.init();
+    return this.strategies.values();
+  }
+
+  /**
+   * Get statistics about meta-learning
+   */
+  async getStats(): Promise<MetaStats> {
+    await this.init();
+
+    const observations = this.observations.values();
+    const totalObservations = observations.length;
+    const totalStrategies = this.strategies.size();
+
+    const observationsByRouting: Record<RoutingDecisionType, number> = {
+      adapt: 0,
+      direct: 0,
+      explore: 0,
+      fallback: 0,
+    };
+
+    let successCount = 0;
+    for (const obs of observations) {
+      observationsByRouting[obs.routing.decision]++;
+      if (obs.outcome.success) successCount++;
+    }
+
+    const averageSuccessRate =
+      totalObservations > 0 ? successCount / totalObservations : 0;
+
+    // Determine trend from recent observations
+    const recent = observations
+      .sort(
+        (a, b) =>
+          new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()
+      )
+      .slice(0, 10);
+    const older = observations
+      .sort(
+        (a, b) =>
+          new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()
+      )
+      .slice(10, 20);
+
+    let recentTrend: 'improving' | 'stable' | 'declining' = 'stable';
+    if (recent.length >= 5 && older.length >= 5) {
+      const recentSuccess =
+        recent.filter((o) => o.outcome.success).length / recent.length;
+      const olderSuccess =
+        older.filter((o) => o.outcome.success).length / older.length;
+      if (recentSuccess > olderSuccess + 0.1) {
+        recentTrend = 'improving';
+      } else if (recentSuccess < olderSuccess - 0.1) {
+        recentTrend = 'declining';
+      }
+    }
+
+    return {
+      totalObservations,
+      totalStrategies,
+      observationsByRouting,
+      averageSuccessRate,
+      recentTrend,
+    };
+  }
+
+  /**
+   * Prune old observations to save space
+   */
+  async pruneOldObservations(maxAge: number = 30 * 24 * 60 * 60 * 1000): Promise<number> {
+    await this.init();
+
+    const cutoff = new Date(Date.now() - maxAge);
+    let pruned = 0;
+
+    for (const observation of this.observations.values()) {
+      if (new Date(observation.timestamp) < cutoff) {
+        this.observations.delete(observation.id);
+        pruned++;
+      }
+    }
+
+    return pruned;
+  }
+
+  /**
+   * Close the stores
+   */
+  async close(): Promise<void> {
+    await Promise.all([this.observations.close(), this.strategies.close()]);
+  }
+}
+
+/**
+ * Create a meta memory store
+ */
+export function createMetaMemory(baseDir: string): MetaMemory {
+  return new MetaMemory(baseDir);
+}