cognitive-core 0.2.5 → 0.3.1

package/src/memory/system.ts

@@ -1,620 +0,0 @@
-import type {
-  Experience,
-  CodeConcept,
-  Strategy,
-  Skill,
-  Playbook,
-  MetaObservation,
-  MetaStrategy,
-  MetaContext,
-  MemoryConfig,
-  MemoryCapacityConfig,
-  Trajectory,
-  TeamTrajectory,
-  KnowledgeBankConfig,
-} from '../types/index.js';
-import { createExperience } from '../types/index.js';
-import type { SqlitePersistence } from '../persistence/index.js';
-import { ExperienceMemory, type PruneResult } from './experience.js';
-import { PlaybookLibrary, type PlaybookMatch } from './playbook.js';
-import { MetaMemory } from './meta.js';
-import {
-  TeamExperienceMemory,
-  type TeamExperience,
-} from './team-experience.js';
-import {
-  LearningEffectivenessTracker,
-  type EffectivenessMetrics,
-} from '../learning/effectiveness.js';
-import { KnowledgeBank } from './knowledge-bank.js';
-import { ReflexionMemory } from './reflexion.js';
-import { CausalStore } from './causal-store.js';
-import { join } from 'node:path';
-
-/**
- * Default capacity configuration
- */
-const DEFAULT_CAPACITY: MemoryCapacityConfig = {
-  maxExperiences: 0,
-  maxPlaybooks: 0,
-  maxExperienceAgeDays: 0,
-  maxFailedExperienceAgeDays: 30,
-  autoPrune: true,
-  preserveDomainCoverage: true,
-  deduplicationThreshold: 0,
-};
-
-/**
- * Result from a memory query (legacy format)
- * @deprecated Use MemoryQueryResultV2 for new code
- */
-export interface MemoryQueryResult {
-  experiences: Array<{ experience: Experience; score: number }>;
-  /** @deprecated Always returns empty array - use playbooks instead */
-  concepts: Array<{ concept: CodeConcept; score: number }>;
-  /** @deprecated Always returns empty array - use playbooks instead */
-  strategies: Array<{ strategy: Strategy; score: number }>;
-  /** @deprecated Always returns empty array - use playbooks instead */
-  skills: Array<{ skill: Skill; score: number }>;
-}
-
-/**
- * Result from a memory query (new format with playbooks)
- */
-export interface MemoryQueryResultV2 {
-  experiences: Array<{ experience: Experience; score: number }>;
-  playbooks: PlaybookMatch[];
-}
-
-/**
- * Unified Memory System
- * Aggregates all memory components with parallel querying
- */
-export class MemorySystem {
-  public readonly experiences: ExperienceMemory;
-  public readonly playbooks: PlaybookLibrary;
-  public readonly meta: MetaMemory;
-  public readonly effectiveness: LearningEffectivenessTracker;
-  public readonly teamExperiences: TeamExperienceMemory;
-  public readonly reflexion: ReflexionMemory;
-  public readonly causal: CausalStore;
-  public readonly knowledgeBank: KnowledgeBank | null;
-
-  private config: MemoryConfig;
-  private capacityConfig: MemoryCapacityConfig;
-  private initialized = false;
-  private lastCapacityCheck = 0;
-  private capacityCheckInterval = 60000; // Check every minute
-
-  constructor(
-    persistence: SqlitePersistence,
-    baseDir: string,
-    config?: Partial<MemoryConfig>,
-    knowledgeBankConfig?: KnowledgeBankConfig
-  ) {
-    this.config = {
-      maxExperiences: config?.maxExperiences ?? 4,
-      maxStrategies: config?.maxStrategies ?? 3,
-      maxConcepts: config?.maxConcepts ?? 5,
-      maxSkills: config?.maxSkills ?? 3,
-      maxContextTokens: config?.maxContextTokens ?? 4000,
-      capacity: config?.capacity ?? DEFAULT_CAPACITY,
-    };
-
-    this.capacityConfig = { ...DEFAULT_CAPACITY, ...config?.capacity };
-
-    // Core stores backed by SqlitePersistence
-    this.experiences = new ExperienceMemory(persistence);
-    this.playbooks = new PlaybookLibrary(persistence);
-    this.meta = new MetaMemory(persistence);
-    this.reflexion = new ReflexionMemory(persistence);
-    this.causal = new CausalStore(persistence);
-
-    // File-backed stores (not yet migrated to SqlitePersistence)
-    this.effectiveness = new LearningEffectivenessTracker(baseDir);
-    this.teamExperiences = new TeamExperienceMemory(baseDir);
-
-    // Initialize knowledge bank if enabled (stays as Markdown files)
-    if (knowledgeBankConfig?.enabled) {
-      const memoryDir = join(baseDir, knowledgeBankConfig.memoryDir ?? 'memory');
-      this.knowledgeBank = new KnowledgeBank(memoryDir, knowledgeBankConfig);
-    } else {
-      this.knowledgeBank = null;
-    }
-  }
-
-  /**
-   * Initialize all memory components
-   */
-  async init(): Promise<void> {
-    if (this.initialized) return;
-
-    const initPromises: Promise<void>[] = [
-      this.experiences.init(),
-      this.playbooks.init(),
-      this.meta.init(),
-      this.effectiveness.init(),
-      this.teamExperiences.init(),
-      this.reflexion.init(),
-      this.causal.init(),
-    ];
-    if (this.knowledgeBank) {
-      initPromises.push(this.knowledgeBank.init());
-    }
-    await Promise.all(initPromises);
-
-    this.initialized = true;
-  }
-
-  /**
-   * Query memory (legacy format)
-   * @deprecated Use queryV2 for new code - this method returns empty arrays for deprecated fields
-   */
-  async query(
-    input: string,
-    options?: {
-      domain?: string;
-      includeExperiences?: boolean;
-    }
-  ): Promise<MemoryQueryResult> {
-    await this.init();
-
-    const experiences = await this.experiences.findSimilar(input, {
-      k: this.config.maxExperiences,
-      domain: options?.domain,
-    });
-
-    // Return empty arrays for deprecated fields
-    return {
-      experiences,
-      concepts: [],
-      strategies: [],
-      skills: [],
-    };
-  }
-
-  /**
-   * Store a trajectory and extract experience from it
-   * Preserves trajectory structure in metadata for richer learning
-   */
-  async storeTrajectory(trajectory: Trajectory): Promise<Experience> {
-    await this.init();
-
-    // Create experience from trajectory
-    // Handle undefined solution (e.g., from failed trajectories)
-    const solutionOutput = trajectory.outcome.solution !== undefined
-      ? JSON.stringify(trajectory.outcome.solution)
-      : trajectory.outcome.errorInfo ?? 'No solution (task failed)';
-
-    // Preserve trajectory structure in metadata for learning from intermediate steps
-    const trajectoryMetadata: Record<string, unknown> = {
-      // Preserve steps with key information (limited to prevent bloat)
-      steps: trajectory.steps.map((step, index) => ({
-        index,
-        thought: step.thought?.slice(0, 500),
-        action: step.action?.slice(0, 200),
-        observation: step.observation?.slice(0, 300),
-        attributionScore: step.attributionScore,
-      })),
-      // Preserve outcome details
-      outcome: {
-        partialScore: trajectory.outcome.partialScore,
-        errorInfo: trajectory.outcome.errorInfo,
-      },
-      // Preserve agent and timing info
-      agentId: trajectory.agentId,
-      wallTimeSeconds: trajectory.wallTimeSeconds,
-      llmCalls: trajectory.llmCalls,
-      stepCount: trajectory.steps.length,
-      // Preserve task context
-      taskContext: trajectory.task.context,
-      // Original trajectory metadata
-      ...trajectory.metadata,
-    };
-
-    const experience = createExperience({
-      taskInput: trajectory.task.description,
-      solutionOutput,
-      feedback: trajectory.outcome.success
-        ? 'Success'
-        : trajectory.outcome.errorInfo ?? 'Failed',
-      success: trajectory.outcome.success,
-      domain: trajectory.task.domain,
-      trajectoryId: trajectory.id,
-      metadata: trajectoryMetadata,
-    });
-
-    await this.experiences.add(experience);
-
-    // Check capacity limits
-    await this.maybeEnforceCapacity();
-
-    return experience;
-  }
-
-  /**
-   * Record usage of an experience
-   */
-  async recordExperienceUsage(id: string): Promise<void> {
-    await this.init();
-    await this.experiences.recordUsage(id);
-  }
-
-  /**
-   * Query memory with new playbook-based format
-   */
-  async queryV2(
-    input: string,
-    options?: {
-      domains?: string[];
-      includeExperiences?: boolean;
-      includePlaybooks?: boolean;
-      minPlaybookConfidence?: number;
-    }
-  ): Promise<MemoryQueryResultV2> {
-    await this.init();
-
-    const includeAll =
-      !options ||
-      (options.includeExperiences === undefined &&
-        options.includePlaybooks === undefined);
-
-    const [experiences, playbooks] = await Promise.all([
-      includeAll || options?.includeExperiences
-        ? this.experiences.findSimilar(input, {
-            k: this.config.maxExperiences,
-            domain: options?.domains?.[0],
-          })
-        : Promise.resolve([]),
-      includeAll || options?.includePlaybooks
-        ? this.playbooks.findMatching(input, {
-            k: this.config.maxStrategies + this.config.maxSkills, // Combined count
-            domains: options?.domains,
-            minConfidence: options?.minPlaybookConfidence,
-            excludeAntiPatterns: true,
-          })
-        : Promise.resolve([]),
-    ]);
-
-    return { experiences, playbooks };
-  }
-
-  /**
-   * Format memory for context injection (new playbook format)
-   */
-  formatForPromptV2(
-    result: MemoryQueryResultV2,
-    options?: {
-      maxTokens?: number;
-      includeRefinements?: boolean;
-    }
-  ): string {
-    const sections: string[] = [];
-    const includeRefinements = options?.includeRefinements ?? true;
-
-    // Format experiences
-    if (result.experiences.length > 0) {
-      sections.push('## Similar Experiences\n');
-      for (const { experience, score } of result.experiences) {
-        sections.push(
-          `### Task (${Math.round(score * 100)}% match)`
-        );
-        sections.push(`${experience.taskInput.slice(0, 200)}`);
-        sections.push(`**Approach**: ${experience.solutionOutput.slice(0, 300)}`);
-        sections.push(
-          `**Outcome**: ${experience.success ? 'Succeeded' : 'Failed'}`
-        );
-        sections.push('');
-      }
-    }
-
-    // Format playbooks
-    if (result.playbooks.length > 0) {
-      sections.push('## Applicable Playbooks\n');
-      for (const { playbook, score: _score, matchType } of result.playbooks) {
-        sections.push(
-          `### ${playbook.name} (${Math.round(playbook.confidence * 100)}% confidence, matched on ${matchType})`
-        );
-        sections.push(`**Strategy**: ${playbook.guidance.strategy}`);
-
-        if (playbook.guidance.tactics.length > 0) {
-          sections.push('**Tactics**:');
-          for (const tactic of playbook.guidance.tactics) {
-            sections.push(`  - ${tactic}`);
-          }
-        }
-
-        if (playbook.guidance.steps && playbook.guidance.steps.length > 0) {
-          sections.push('**Steps**:');
-          for (let i = 0; i < playbook.guidance.steps.length; i++) {
-            sections.push(`  ${i + 1}. ${playbook.guidance.steps[i]}`);
-          }
-        }
-
-        if (playbook.verification.successIndicators.length > 0) {
-          sections.push(
-            `**Success indicators**: ${playbook.verification.successIndicators.join(', ')}`
-          );
-        }
-
-        // Include refinements if requested
-        if (includeRefinements && playbook.evolution.refinements.length > 0) {
-          sections.push('**Context-specific notes**:');
-          for (const refinement of playbook.evolution.refinements.slice(0, 3)) {
-            sections.push(`  - In ${refinement.context}: ${refinement.addition}`);
-          }
-        }
-
-        sections.push('');
-      }
-    }
-
-    return sections.join('\n');
-  }
-
-  // === META-LEARNING METHODS ===
-
-  /**
-   * Record a meta-observation after task completion
-   */
-  async recordMetaObservation(observation: MetaObservation): Promise<void> {
-    await this.init();
-    await this.meta.recordObservation(observation);
-  }
-
-  /**
-   * Get meta-learning adjustments for routing
-   */
-  async getMetaAdjustments(context: MetaContext): Promise<MetaStrategy[]> {
-    await this.init();
-    return this.meta.getRoutingAdjustments(context);
-  }
-
-  /**
-   * Add a playbook to the library
-   */
-  async addPlaybook(playbook: Playbook): Promise<void> {
-    await this.init();
-    await this.playbooks.add(playbook);
-  }
-
-  /**
-   * Record playbook usage
-   */
-  async recordPlaybookUsage(
-    id: string,
-    trajectoryId: string,
-    success: boolean,
-    failureContext?: string,
-    failureMode?: string
-  ): Promise<void> {
-    await this.init();
-    if (success) {
-      await this.playbooks.recordSuccess(id, trajectoryId);
-    } else {
-      await this.playbooks.recordFailure(
-        id,
-        trajectoryId,
-        failureContext ?? 'Unknown context',
-        failureMode ?? 'Unknown failure mode'
-      );
-    }
-  }
-
-  /**
-   * Get statistics about memory contents
-   */
-  async getStats(): Promise<{
-    experienceCount: number;
-    playbookCount: number;
-    metaObservationCount: number;
-    annotationCount: number;
-    teamExperienceCount: number;
-    teamPlaybookCount: number;
-    rolePlaybookCount: number;
-    teamMetaObservationCount: number;
-  }> {
-    await this.init();
-
-    const [experienceCount, playbookCount, metaStats, annotationCount, teamStats] = await Promise.all([
-      this.experiences.count(),
-      this.playbooks.count(),
-      this.meta.getStats(),
-      this.effectiveness.count(),
-      this.teamExperiences.getStats(),
-    ]);
-
-    return {
-      experienceCount,
-      playbookCount,
-      metaObservationCount: metaStats.totalObservations,
-      annotationCount,
-      teamExperienceCount: teamStats.teamExperienceCount,
-      teamPlaybookCount: teamStats.teamPlaybookCount,
-      rolePlaybookCount: teamStats.rolePlaybookCount,
-      teamMetaObservationCount: teamStats.metaObservationCount,
-    };
-  }
-
-  /**
-   * Get learning effectiveness metrics
-   */
-  async getEffectivenessMetrics(options?: {
-    since?: Date;
-    until?: Date;
-  }): Promise<EffectivenessMetrics> {
-    await this.init();
-    return this.effectiveness.computeMetrics(options);
-  }
-
-  // === CAPACITY MANAGEMENT ===
-
-  /**
-   * Check and enforce capacity limits (called automatically if autoPrune is enabled)
-   */
-  async enforceCapacityLimits(): Promise<{
-    experiencesPruned: PruneResult | null;
-    playbooksPruned: number;
-  }> {
-    await this.init();
-
-    const result: {
-      experiencesPruned: PruneResult | null;
-      playbooksPruned: number;
-    } = {
-      experiencesPruned: null,
-      playbooksPruned: 0,
-    };
-
-    const capacity = this.capacityConfig;
-
-    // Check if we need to prune experiences
-    const experienceCount = await this.experiences.count();
-    const needsExperiencePrune =
-      (capacity.maxExperiences > 0 && experienceCount > capacity.maxExperiences) ||
-      capacity.maxExperienceAgeDays > 0 ||
-      capacity.maxFailedExperienceAgeDays > 0 ||
-      capacity.deduplicationThreshold > 0;
-
-    if (needsExperiencePrune) {
-      result.experiencesPruned = await this.experiences.prune({
-        maxCount: capacity.maxExperiences > 0 ? capacity.maxExperiences : undefined,
-        maxAgeDays: capacity.maxExperienceAgeDays > 0 ? capacity.maxExperienceAgeDays : undefined,
-        removeFailedOlderThanDays:
-          capacity.maxFailedExperienceAgeDays > 0
-            ? capacity.maxFailedExperienceAgeDays
-            : undefined,
-        similarityThreshold:
-          capacity.deduplicationThreshold > 0
-            ? capacity.deduplicationThreshold
-            : undefined,
-        preserveDomainCoverage: capacity.preserveDomainCoverage,
-      });
-    }
-
-    // Check if we need to prune playbooks
-    if (capacity.maxPlaybooks > 0) {
-      const playbookCount = await this.playbooks.count();
-      if (playbookCount > capacity.maxPlaybooks) {
-        // Prune lowest confidence playbooks
-        const allPlaybooks = await this.playbooks.getAll();
-        const sorted = allPlaybooks.sort((a, b) => b.confidence - a.confidence);
-        const toRemove = sorted.slice(capacity.maxPlaybooks);
-
-        for (const playbook of toRemove) {
-          await this.playbooks.delete(playbook.id);
-          result.playbooksPruned++;
-        }
-      }
-    }
-
-    return result;
-  }
-
-  /**
-   * Check capacity and prune if necessary (throttled)
-   * Called automatically after adding experiences
-   */
-  private async maybeEnforceCapacity(): Promise<void> {
-    if (!this.capacityConfig.autoPrune) return;
-
-    const now = Date.now();
-    if (now - this.lastCapacityCheck < this.capacityCheckInterval) return;
-
-    this.lastCapacityCheck = now;
-    await this.enforceCapacityLimits();
-  }
-
-  /**
-   * Get capacity status
-   */
-  async getCapacityStatus(): Promise<{
-    experiences: {
-      current: number;
-      limit: number;
-      utilizationPercent: number;
-    };
-    playbooks: {
-      current: number;
-      limit: number;
-      utilizationPercent: number;
-    };
-    pruningEnabled: boolean;
-    lastCheck: Date | null;
-  }> {
-    await this.init();
-
-    const experienceCount = await this.experiences.count();
-    const playbookCount = await this.playbooks.count();
-    const capacity = this.capacityConfig;
-
-    return {
-      experiences: {
-        current: experienceCount,
-        limit: capacity.maxExperiences,
-        utilizationPercent:
-          capacity.maxExperiences > 0
-            ? Math.round((experienceCount / capacity.maxExperiences) * 100)
-            : 0,
-      },
-      playbooks: {
-        current: playbookCount,
-        limit: capacity.maxPlaybooks,
-        utilizationPercent:
-          capacity.maxPlaybooks > 0
-            ? Math.round((playbookCount / capacity.maxPlaybooks) * 100)
-            : 0,
-      },
-      pruningEnabled: capacity.autoPrune,
-      lastCheck: this.lastCapacityCheck > 0 ? new Date(this.lastCapacityCheck) : null,
-    };
-  }
-
-  /**
-   * Update capacity configuration
-   */
-  updateCapacityConfig(config: Partial<MemoryCapacityConfig>): void {
-    this.capacityConfig = { ...this.capacityConfig, ...config };
-  }
-
-  /**
-   * Close all stores
-   */
-  /**
-   * Store a team trajectory and create a team experience
-   */
-  async storeTeamTrajectory(trajectory: TeamTrajectory): Promise<TeamExperience> {
-    await this.init();
-    return this.teamExperiences.storeTeamExperience({
-      teamTrajectory: trajectory,
-    });
-  }
-
-  async close(): Promise<void> {
-    const closePromises: Promise<void>[] = [
-      this.experiences.close(),
-      this.playbooks.close(),
-      this.meta.close(),
-      this.effectiveness.close(),
-      this.teamExperiences.close(),
-      this.reflexion.close(),
-      this.causal.close(),
-    ];
-    if (this.knowledgeBank) {
-      closePromises.push(this.knowledgeBank.close());
-    }
-    await Promise.all(closePromises);
-  }
-}
-
-/**
- * Create a memory system instance
- */
-export function createMemorySystem(
-  persistence: SqlitePersistence,
-  baseDir: string,
-  config?: Partial<MemoryConfig>,
-  knowledgeBankConfig?: KnowledgeBankConfig
-): MemorySystem {
-  return new MemorySystem(persistence, baseDir, config, knowledgeBankConfig);
-}