cognitive-core 0.1.2 → 0.2.1

package/src/memory/meta.ts

@@ -2,6 +2,8 @@
  * Meta Memory - Learning about learning
  * Tracks routing decisions, memory effectiveness, and execution patterns
  * to improve future learning and routing decisions
+ *
+ * Backed by SqlitePersistence for unified database storage.
  */
 
 import type {
@@ -14,7 +16,8 @@ import type {
   RoutingDecisionType,
 } from '../types/index.js';
 import { createMetaStrategy } from '../types/index.js';
-import { JsonStore } from '../utils/storage.js';
+import type { SqlitePersistence, MetaObservationRow, MetaStrategyRow } from '../persistence/index.js';
+import { fromJSON, fromISO } from '../persistence/index.js';
 
 /**
  * Filter options for querying observations
@@ -27,32 +30,51 @@ export interface ObservationFilter {
   limit?: number;
 }
 
+/**
+ * Convert a MetaObservationRow to a MetaObservation.
+ */
+function rowToObservation(row: MetaObservationRow): MetaObservation {
+  return {
+    id: row.id,
+    trajectoryId: row.trajectory_id,
+    routing: fromJSON(row.routing, { decision: 'direct' as RoutingDecisionType, confidence: 0, reasoning: '' }),
+    memoryUsage: fromJSON(row.memory_usage, { experiencesRetrieved: 0, playbooksApplied: [], retrievalQuality: 'neutral' as const }),
+    execution: fromJSON(row.execution, { decompositionUsed: false, refinementIterations: 0, backtrackingOccurred: false, toolsUsed: [] }),
+    outcome: fromJSON(row.outcome, { success: false, quality: 'poor' as const, effortActual: 0, effortEstimate: 0 }),
+    lessons: fromJSON(row.lessons, { whatWorked: [], whatFailed: [], suggestions: [] }),
+    timestamp: fromISO(row.timestamp) ?? new Date(),
+  };
+}
+
+/**
+ * Convert a MetaStrategyRow to a MetaStrategy.
+ */
+function rowToStrategy(row: MetaStrategyRow): MetaStrategy {
+  return {
+    id: row.id,
+    name: row.name,
+    condition: fromJSON(row.condition, { taskCharacteristics: [], memoryState: [] }),
+    adjustment: fromJSON(row.adjustment, { routingBias: {}, retrievalModification: '', executionHint: '' }),
+    applicationCount: row.application_count,
+    successRate: row.success_rate,
+    createdAt: fromISO(row.created_at) ?? new Date(),
+    updatedAt: fromISO(row.updated_at) ?? new Date(),
+  };
+}
+
 /**
  * Meta Memory - Store for meta-learning observations and strategies
  */
 export class MetaMemory {
-  private observations: JsonStore<MetaObservation>;
-  private strategies: JsonStore<MetaStrategy>;
+  private persistence: SqlitePersistence;
   private initialized = false;
 
-  constructor(baseDir: string) {
-    this.observations = new JsonStore<MetaObservation>(
-      baseDir,
-      'meta-observations',
-      {
-        autoSaveInterval: 60000,
-        pretty: true,
-      }
-    );
-    this.strategies = new JsonStore<MetaStrategy>(baseDir, 'meta-strategies', {
-      autoSaveInterval: 60000,
-      pretty: true,
-    });
+  constructor(persistence: SqlitePersistence) {
+    this.persistence = persistence;
   }
 
   async init(): Promise<void> {
     if (this.initialized) return;
-    await Promise.all([this.observations.init(), this.strategies.init()]);
     this.initialized = true;
   }
 
@@ -63,8 +85,7 @@ export class MetaMemory {
    */
   async recordObservation(observation: MetaObservation): Promise<void> {
     await this.init();
-    this.observations.set(observation.id, observation);
-    await this.observations.save(observation.id);
+    this.persistence.insertMetaObservation(observation);
   }
 
   /**
@@ -73,7 +94,7 @@ export class MetaMemory {
   async getObservations(filter?: ObservationFilter): Promise<MetaObservation[]> {
     await this.init();
 
-    let observations = this.observations.values();
+    let observations = this.persistence.getAllMetaObservations().map(rowToObservation);
 
     if (filter?.routingDecision) {
       observations = observations.filter(
@@ -98,11 +119,6 @@ export class MetaMemory {
       );
     }
 
-    // Sort by timestamp descending (most recent first)
-    observations.sort(
-      (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()
-    );
-
     if (filter?.limit) {
       observations = observations.slice(0, filter.limit);
     }
@@ -117,8 +133,8 @@ export class MetaMemory {
     trajectoryId: string
   ): Promise<MetaObservation | undefined> {
     await this.init();
-    return this.observations
-      .values()
+    return this.persistence.getAllMetaObservations()
+      .map(rowToObservation)
       .find((o) => o.trajectoryId === trajectoryId);
   }
 
@@ -126,29 +142,23 @@ export class MetaMemory {
 
   /**
    * Analyze routing effectiveness
-   * Returns: which routing decisions lead to success in which contexts
    */
   async analyzeRoutingEffectiveness(): Promise<RoutingAnalysis> {
     await this.init();
 
+    // Use SQL aggregation for efficiency
+    const raw = this.persistence.aggregateRoutingEffectiveness();
+
     const byDecision: Record<string, { total: number; successful: number }> = {
       adapt: { total: 0, successful: 0 },
       direct: { total: 0, successful: 0 },
       explore: { total: 0, successful: 0 },
       fallback: { total: 0, successful: 0 },
+      ...raw,
     };
 
-    for (const observation of this.observations.values()) {
-      const decision = observation.routing.decision;
-      byDecision[decision].total++;
-      if (observation.outcome.success) {
-        byDecision[decision].successful++;
-      }
-    }
-
     const recommendations: string[] = [];
 
-    // Generate recommendations based on analysis
     for (const [decision, stats] of Object.entries(byDecision)) {
       if (stats.total >= 5) {
         const successRate = stats.successful / stats.total;
@@ -169,7 +179,6 @@ export class MetaMemory {
 
   /**
    * Analyze memory retrieval quality
-   * Returns: which retrieval patterns help vs mislead
    */
   async analyzeRetrievalQuality(): Promise<RetrievalAnalysis> {
     await this.init();
@@ -177,9 +186,10 @@ export class MetaMemory {
     const helpfulPatterns: Map<string, number> = new Map();
     const misleadingPatterns: Map<string, number> = new Map();
 
-    for (const observation of this.observations.values()) {
-      const quality = observation.memoryUsage.retrievalQuality;
-      const playbooks = observation.memoryUsage.playbooksApplied;
+    for (const row of this.persistence.getAllMetaObservations()) {
+      const obs = rowToObservation(row);
+      const quality = obs.memoryUsage.retrievalQuality;
+      const playbooks = obs.memoryUsage.playbooksApplied;
 
       for (const playbookId of playbooks) {
         if (quality === 'helpful') {
@@ -198,7 +208,6 @@ export class MetaMemory {
 
     const recommendations: string[] = [];
 
-    // Find consistently misleading playbooks
     for (const [playbookId, count] of misleadingPatterns) {
       const helpfulCount = helpfulPatterns.get(playbookId) ?? 0;
       if (count > helpfulCount && count >= 3) {
@@ -217,13 +226,13 @@ export class MetaMemory {
 
   /**
    * Generate meta-strategies from observations
-   * Called periodically to distill learning patterns
    */
   async generateMetaStrategies(): Promise<MetaStrategy[]> {
     await this.init();
 
     const newStrategies: MetaStrategy[] = [];
-    const observations = this.observations.values();
+    const observations = this.persistence.getAllMetaObservations().map(rowToObservation);
+    const existingStrategies = this.persistence.getAllMetaStrategies().map(rowToStrategy);
 
     // Pattern 1: Low-confidence routing tends to need exploration
     const lowConfidenceObs = observations.filter(
@@ -234,9 +243,7 @@ export class MetaMemory {
       (lowConfidenceObs.length || 1);
 
     if (lowConfidenceObs.length >= 5 && lowConfidenceSuccessRate < 0.4) {
-      const existing = this.strategies
-        .values()
-        .find((s) => s.name === 'low-confidence-explore');
+      const existing = existingStrategies.find((s) => s.name === 'low-confidence-explore');
       if (!existing) {
         newStrategies.push(
           createMetaStrategy({
@@ -264,9 +271,7 @@ export class MetaMemory {
       (multiFileObs.length || 1);
 
     if (multiFileObs.length >= 5 && multiFileSuccessRate > 0.7) {
-      const existing = this.strategies
-        .values()
-        .find((s) => s.name === 'decomposition-effective');
+      const existing = existingStrategies.find((s) => s.name === 'decomposition-effective');
       if (!existing) {
         newStrategies.push(
           createMetaStrategy({
@@ -294,9 +299,7 @@ export class MetaMemory {
       (refinedObs.length || 1);
 
     if (refinedObs.length >= 5 && refinedSuccessRate > 0.6) {
-      const existing = this.strategies
-        .values()
-        .find((s) => s.name === 'refinement-beneficial');
+      const existing = existingStrategies.find((s) => s.name === 'refinement-beneficial');
       if (!existing) {
         newStrategies.push(
           createMetaStrategy({
@@ -317,8 +320,7 @@ export class MetaMemory {
 
     // Save new strategies
     for (const strategy of newStrategies) {
-      this.strategies.set(strategy.id, strategy);
-      await this.strategies.save(strategy.id);
+      this.persistence.insertMetaStrategy(strategy);
     }
 
     return newStrategies;
@@ -334,8 +336,9 @@ export class MetaMemory {
 
     const applicable: MetaStrategy[] = [];
 
-    for (const strategy of this.strategies.values()) {
-      // Check if task characteristics match
+    for (const row of this.persistence.getAllMetaStrategies()) {
+      const strategy = rowToStrategy(row);
+
       const taskMatch =
         strategy.condition.taskCharacteristics.length === 0 ||
         strategy.condition.taskCharacteristics.some((tc) =>
@@ -344,7 +347,6 @@ export class MetaMemory {
           )
         );
 
-      // Check if memory state matches
       const memoryMatch =
         strategy.condition.memoryState.length === 0 ||
         strategy.condition.memoryState.some((ms) =>
@@ -369,17 +371,17 @@ export class MetaMemory {
     success: boolean
   ): Promise<void> {
     await this.init();
-    const strategy = this.strategies.get(strategyId);
-    if (!strategy) return;
+    const row = this.persistence.getMetaStrategy(strategyId);
+    if (!row) return;
 
+    const strategy = rowToStrategy(row);
     strategy.applicationCount++;
     const alpha = 0.1;
     strategy.successRate =
       strategy.successRate * (1 - alpha) + (success ? 1 : 0) * alpha;
     strategy.updatedAt = new Date();
 
-    this.strategies.set(strategyId, strategy);
-    await this.strategies.save(strategyId);
+    this.persistence.insertMetaStrategy(strategy); // Write-through
   }
 
   // === LIFECYCLE ===
@@ -389,8 +391,7 @@ export class MetaMemory {
    */
   async addStrategy(strategy: MetaStrategy): Promise<void> {
     await this.init();
-    this.strategies.set(strategy.id, strategy);
-    await this.strategies.save(strategy.id);
+    this.persistence.insertMetaStrategy(strategy);
   }
 
   /**
@@ -398,7 +399,8 @@ export class MetaMemory {
    */
   async getStrategy(id: string): Promise<MetaStrategy | undefined> {
     await this.init();
-    return this.strategies.get(id);
+    const row = this.persistence.getMetaStrategy(id);
+    return row ? rowToStrategy(row) : undefined;
   }
 
   /**
@@ -406,7 +408,7 @@ export class MetaMemory {
    */
   async getAllStrategies(): Promise<MetaStrategy[]> {
     await this.init();
-    return this.strategies.values();
+    return this.persistence.getAllMetaStrategies().map(rowToStrategy);
   }
 
   /**
@@ -415,9 +417,9 @@ export class MetaMemory {
   async getStats(): Promise<MetaStats> {
     await this.init();
 
-    const observations = this.observations.values();
+    const observations = this.persistence.getAllMetaObservations().map(rowToObservation);
     const totalObservations = observations.length;
-    const totalStrategies = this.strategies.size();
+    const totalStrategies = this.persistence.getAllMetaStrategies().length;
 
     const observationsByRouting: Record<RoutingDecisionType, number> = {
       adapt: 0,
@@ -435,19 +437,9 @@ export class MetaMemory {
     const averageSuccessRate =
       totalObservations > 0 ? successCount / totalObservations : 0;
 
-    // Determine trend from recent observations
-    const recent = observations
-      .sort(
-        (a, b) =>
-          new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()
-      )
-      .slice(0, 10);
-    const older = observations
-      .sort(
-        (a, b) =>
-          new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()
-      )
-      .slice(10, 20);
+    // Determine trend from recent observations (already sorted by timestamp DESC from SQL)
+    const recent = observations.slice(0, 10);
+    const older = observations.slice(10, 20);
 
     let recentTrend: 'improving' | 'stable' | 'declining' = 'stable';
     if (recent.length >= 5 && older.length >= 5) {
@@ -476,31 +468,21 @@ export class MetaMemory {
    */
   async pruneOldObservations(maxAge: number = 30 * 24 * 60 * 60 * 1000): Promise<number> {
     await this.init();
-
     const cutoff = new Date(Date.now() - maxAge);
-    let pruned = 0;
-
-    for (const observation of this.observations.values()) {
-      if (new Date(observation.timestamp) < cutoff) {
-        this.observations.delete(observation.id);
-        pruned++;
-      }
-    }
-
-    return pruned;
+    return this.persistence.deleteMetaObservationsBefore(cutoff);
   }
 
   /**
-   * Close the stores
+   * Close the stores (no-op — SqlitePersistence lifecycle is managed externally)
    */
   async close(): Promise<void> {
-    await Promise.all([this.observations.close(), this.strategies.close()]);
+    // SqlitePersistence lifecycle is managed by MemorySystem/Atlas
   }
 }
 
 /**
  * Create a meta memory store
  */
-export function createMetaMemory(baseDir: string): MetaMemory {
-  return new MetaMemory(baseDir);
+export function createMetaMemory(persistence: SqlitePersistence): MetaMemory {
+  return new MetaMemory(persistence);
 }