cognitive-core 0.1.2 → 0.2.0

package/src/memory/temporal-compressor.ts

@@ -0,0 +1,409 @@
+/**
+ * TemporalCompressor — Adaptive memory compression.
+ *
+ * Experiences flow through Hot → Warm → Cold → Evicted tiers
+ * based on their accessScore (from Phase 0).
+ *
+ * Compression levels:
+ * - Hot:  Full fidelity (all data preserved)
+ * - Warm: Strip solutionOutput to key steps only
+ * - Cold: Replace key steps with single-paragraph summary
+ * - Evicted: Remove entirely (after ensuring pattern captured in ReasoningBank)
+ *
+ * Hysteresis: An experience must stay below a threshold for 2 consecutive
+ * compression passes before demotion, preventing oscillation from a
+ * single retrieval hit.
+ */
+
+import type { Experience } from '../types/index.js';
+import type { SqlitePersistence } from '../persistence/index.js';
+
+/**
+ * Configuration for temporal compression
+ */
+export interface TemporalCompressorConfig {
+  /** accessScore above this = hot (default: 0.5) */
+  hotThreshold: number;
+  /** accessScore above this = warm (default: 0.2) */
+  warmThreshold: number;
+  /** accessScore below this = evict (default: 0.05) */
+  evictionThreshold: number;
+  /** Prevent oscillation between tiers (default: 0.05) */
+  hysteresis: number;
+}
+
+/**
+ * Result of a compression pass
+ */
+export interface CompressionResult {
+  /** IDs moved to a hotter tier (rare, via hysteresis) */
+  promoted: string[];
+  /** IDs moved to a colder tier */
+  demoted: string[];
+  /** IDs recommended for eviction */
+  evicted: string[];
+  /** IDs that stayed in the same tier */
+  unchanged: string[];
+  /** The modified experiences (for batch updating) */
+  updates: Map<string, Partial<Experience>>;
+}
+
+const DEFAULT_CONFIG: TemporalCompressorConfig = {
+  hotThreshold: 0.5,
+  warmThreshold: 0.2,
+  evictionThreshold: 0.05,
+  hysteresis: 0.05,
+};
+
+/**
+ * TemporalCompressor — evaluates and compresses experiences based on access scores
+ */
+export class TemporalCompressor {
+  private config: TemporalCompressorConfig;
+  private persistence?: SqlitePersistence;
+
+  /**
+   * Track experiences that were below demotion threshold on the previous pass.
+   * Hysteresis: only demote after 2 consecutive passes below threshold.
+   */
+  private pendingDemotion: Set<string> = new Set();
+
+  constructor(options?: Partial<TemporalCompressorConfig>, persistence?: SqlitePersistence) {
+    this.config = { ...DEFAULT_CONFIG, ...options };
+    this.persistence = persistence;
+
+    // Load persisted pending demotions
+    if (persistence?.isInitialized) {
+      const saved = persistence.getSystemState<string[]>('temporal_compressor');
+      if (saved) {
+        this.pendingDemotion = new Set(saved);
+      }
+    }
+  }
+
+  /**
+   * Evaluate and compress experiences based on their access scores.
+   * Returns a CompressionResult describing all tier changes.
+   */
+  async compress(experiences: Experience[]): Promise<CompressionResult> {
+    const result: CompressionResult = {
+      promoted: [],
+      demoted: [],
+      evicted: [],
+      unchanged: [],
+      updates: new Map(),
+    };
+
+    const newPendingDemotion = new Set<string>();
+
+    for (const exp of experiences) {
+      const currentTier = exp.tier ?? 'hot';
+      const targetTier = this.computeTargetTier(exp.accessScore);
+
+      if (targetTier === 'evict') {
+        // Cold → Evicted
+        if (currentTier === 'cold') {
+          if (this.pendingDemotion.has(exp.id)) {
+            // Second consecutive pass below eviction threshold → evict
+            result.evicted.push(exp.id);
+          } else {
+            // First pass below → mark pending
+            newPendingDemotion.add(exp.id);
+            result.unchanged.push(exp.id);
+          }
+        } else {
+          // Not yet cold — demote first (can't skip tiers)
+          this.handleDemotion(exp, currentTier, result, newPendingDemotion);
+        }
+      } else if (this.tierOrder(targetTier) < this.tierOrder(currentTier)) {
+        // Demotion (hot → warm, warm → cold, hot → cold)
+        this.handleDemotion(exp, currentTier, result, newPendingDemotion);
+      } else if (this.tierOrder(targetTier) > this.tierOrder(currentTier)) {
+        // Promotion (cold → warm, warm → hot) — requires score above threshold + hysteresis
+        const effectiveScore = exp.accessScore;
+        const promotionThreshold = this.getPromotionThreshold(currentTier);
+
+        if (effectiveScore >= promotionThreshold) {
+          const nextTier = this.nextHotterTier(currentTier);
+          if (nextTier) {
+            const promoted = this.promoteTier(exp, nextTier);
+            result.promoted.push(exp.id);
+            result.updates.set(exp.id, promoted);
+          } else {
+            result.unchanged.push(exp.id);
+          }
+        } else {
+          result.unchanged.push(exp.id);
+        }
+      } else {
+        result.unchanged.push(exp.id);
+      }
+    }
+
+    this.pendingDemotion = newPendingDemotion;
+
+    // Persist pending demotions
+    if (this.persistence) {
+      this.persistence.setSystemState('temporal_compressor', [...this.pendingDemotion]);
+    }
+
+    return result;
+  }
+
+  /**
+   * Load persisted pending demotions. Call after persistence is initialized.
+   */
+  loadPersistedState(): void {
+    if (this.persistence?.isInitialized) {
+      const saved = this.persistence.getSystemState<string[]>('temporal_compressor');
+      if (saved) {
+        this.pendingDemotion = new Set(saved);
+      }
+    }
+  }
+
+  /**
+   * Compress a single experience from hot → warm.
+   * Strips solutionOutput to key steps only.
+   */
+  compressToWarm(experience: Experience): Experience {
+    const keySteps = this.extractKeySteps(experience);
+    const originalStepCount = this.estimateStepCount(experience);
+
+    return {
+      ...experience,
+      tier: 'warm',
+      solutionOutput: keySteps.join('\n---\n'),
+      compressed: {
+        keySteps,
+        summary: '',
+        originalStepCount,
+      },
+    };
+  }
+
+  /**
+   * Compress a single experience from warm → cold.
+   * Replaces key steps with a single-paragraph summary.
+   */
+  compressToCold(experience: Experience): Experience {
+    const summary = this.generateSummary(experience);
+
+    return {
+      ...experience,
+      tier: 'cold',
+      taskInput: experience.taskInput.slice(0, 200),
+      solutionOutput: summary,
+      feedback: experience.feedback.slice(0, 100),
+      compressed: {
+        keySteps: experience.compressed?.keySteps ?? [],
+        summary,
+        originalStepCount: experience.compressed?.originalStepCount ?? this.estimateStepCount(experience),
+      },
+    };
+  }
+
+  // === Internal helpers ===
+
+  /**
+   * Determine the target tier for a given access score.
+   */
+  private computeTargetTier(accessScore: number): 'hot' | 'warm' | 'cold' | 'evict' {
+    if (accessScore >= this.config.hotThreshold) return 'hot';
+    if (accessScore >= this.config.warmThreshold) return 'warm';
+    if (accessScore >= this.config.evictionThreshold) return 'cold';
+    return 'evict';
+  }
+
+  /**
+   * Numeric ordering for tiers (higher = hotter).
+   */
+  private tierOrder(tier: 'hot' | 'warm' | 'cold'): number {
+    switch (tier) {
+      case 'hot': return 2;
+      case 'warm': return 1;
+      case 'cold': return 0;
+    }
+  }
+
+  /**
+   * Get the next colder tier.
+   */
+  private nextColderTier(tier: 'hot' | 'warm' | 'cold'): 'warm' | 'cold' | undefined {
+    switch (tier) {
+      case 'hot': return 'warm';
+      case 'warm': return 'cold';
+      case 'cold': return undefined;
+    }
+  }
+
+  /**
+   * Get the next hotter tier.
+   */
+  private nextHotterTier(tier: 'hot' | 'warm' | 'cold'): 'hot' | 'warm' | undefined {
+    switch (tier) {
+      case 'cold': return 'warm';
+      case 'warm': return 'hot';
+      case 'hot': return undefined;
+    }
+  }
+
+  /**
+   * Get the threshold needed for promotion from a tier (with hysteresis).
+   */
+  private getPromotionThreshold(currentTier: 'hot' | 'warm' | 'cold'): number {
+    switch (currentTier) {
+      case 'cold': return this.config.warmThreshold + this.config.hysteresis;
+      case 'warm': return this.config.hotThreshold + this.config.hysteresis;
+      case 'hot': return Infinity;
+    }
+  }
+
+  /**
+   * Handle a demotion with hysteresis.
+   * Only demote after 2 consecutive passes below threshold.
+   */
+  private handleDemotion(
+    exp: Experience,
+    currentTier: 'hot' | 'warm' | 'cold',
+    result: CompressionResult,
+    newPendingDemotion: Set<string>
+  ): void {
+    if (this.pendingDemotion.has(exp.id)) {
+      // Second consecutive pass below threshold → demote one step
+      const nextTier = this.nextColderTier(currentTier);
+      if (nextTier) {
+        const compressed = nextTier === 'warm'
+          ? this.compressToWarm(exp)
+          : this.compressToCold(exp);
+
+        result.demoted.push(exp.id);
+        result.updates.set(exp.id, {
+          tier: compressed.tier,
+          solutionOutput: compressed.solutionOutput,
+          taskInput: compressed.taskInput,
+          feedback: compressed.feedback,
+          compressed: compressed.compressed,
+        });
+      } else {
+        result.unchanged.push(exp.id);
+      }
+    } else {
+      // First pass below threshold → mark pending
+      newPendingDemotion.add(exp.id);
+      result.unchanged.push(exp.id);
+    }
+  }
+
+  /**
+   * Promote an experience to a hotter tier.
+   * Note: data can't be un-compressed, but the tier changes.
+   */
+  private promoteTier(
+    _exp: Experience,
+    targetTier: 'hot' | 'warm'
+  ): Partial<Experience> {
+    return { tier: targetTier };
+  }
+
+  /**
+   * Extract key steps from a solution output.
+   * Looks for tool calls, error messages, and outcome-relevant actions.
+   */
+  private extractKeySteps(experience: Experience): string[] {
+    const output = experience.solutionOutput;
+    const keySteps: string[] = [];
+
+    // If metadata has trajectory steps, extract from there
+    const metadata = experience.metadata as Record<string, unknown>;
+    const steps = metadata?.steps as Array<{
+      thought?: string;
+      action?: string;
+      observation?: string;
+      attributionScore?: number;
+    }> | undefined;
+
+    if (steps && Array.isArray(steps)) {
+      for (const step of steps) {
+        // Keep steps with high attribution or significant actions
+        const isKey =
+          (step.attributionScore !== undefined && step.attributionScore > 0.5) ||
+          step.action?.match(/^(Read|Write|Edit|Bash|Grep|Glob)\s/i) ||
+          step.observation?.match(/error|fail|success|created|updated/i);
+
+        if (isKey) {
+          const parts: string[] = [];
+          if (step.action) parts.push(`Action: ${step.action.slice(0, 150)}`);
+          if (step.observation) parts.push(`Result: ${step.observation.slice(0, 150)}`);
+          keySteps.push(parts.join(' → '));
+        }
+      }
+    }
+
+    // Fallback: split output into paragraphs and keep first/last + error lines
+    if (keySteps.length === 0) {
+      const lines = output.split('\n').filter((l) => l.trim());
+      if (lines.length <= 5) {
+        return lines;
+      }
+
+      // Keep first line (often the approach), error lines, and last line (outcome)
+      keySteps.push(lines[0]);
+      for (const line of lines) {
+        if (line.match(/error|exception|fail|warning/i) && keySteps.length < 8) {
+          keySteps.push(line.slice(0, 200));
+        }
+      }
+      keySteps.push(lines[lines.length - 1]);
+    }
+
+    return keySteps.slice(0, 10);
+  }
+
+  /**
+   * Estimate the original step count from an experience.
+   */
+  private estimateStepCount(experience: Experience): number {
+    const metadata = experience.metadata as Record<string, unknown>;
+    if (typeof metadata?.stepCount === 'number') {
+      return metadata.stepCount;
+    }
+    const steps = metadata?.steps;
+    if (Array.isArray(steps)) {
+      return steps.length;
+    }
+    // Fallback: estimate from output length
+    return Math.max(1, Math.ceil(experience.solutionOutput.length / 200));
+  }
+
+  /**
+   * Generate a one-paragraph summary for cold tier.
+   */
+  private generateSummary(experience: Experience): string {
+    const success = experience.success ? 'Successfully' : 'Failed to';
+    const domain = experience.domain;
+    const taskSnippet = experience.taskInput.slice(0, 100);
+
+    // Use key steps if available
+    const keySteps = experience.compressed?.keySteps;
+    if (keySteps && keySteps.length > 0) {
+      const stepsSnippet = keySteps
+        .slice(0, 3)
+        .map((s) => s.slice(0, 80))
+        .join('; ');
+      return `[${domain}] ${success} complete task: "${taskSnippet}". Key steps: ${stepsSnippet}. ${experience.feedback.slice(0, 100)}`;
+    }
+
+    return `[${domain}] ${success} complete task: "${taskSnippet}". ${experience.feedback.slice(0, 150)}`;
+  }
+}
+
+/**
+ * Create a TemporalCompressor
+ */
+export function createTemporalCompressor(
+  options?: Partial<TemporalCompressorConfig>,
+  persistence?: SqlitePersistence
+): TemporalCompressor {
+  return new TemporalCompressor(options, persistence);
+}

package/src/persistence/index.ts

@@ -0,0 +1,37 @@
+/**
+ * Persistence module — Unified SQLite database for system-internal state.
+ *
+ * Agent-accessible data (knowledge notes, published skills) stays as files.
+ * This module handles everything else: experiences, playbooks, causal edges,
+ * meta state, routing weights, vector embeddings.
+ */
+
+export {
+  SqlitePersistence,
+  createSqlitePersistence,
+  type SqlitePersistenceConfig,
+  type ExperienceRow,
+  type PlaybookRow,
+  type ReflexionRow,
+  type CausalEdgeRow,
+  type MetaObservationRow,
+  type MetaStrategyRow,
+  type SystemStateRow,
+} from './sqlite-persistence.js';
+
+export {
+  toISO,
+  fromISO,
+  toJSON,
+  fromJSON,
+  toInt,
+  fromInt,
+  embeddingToBlob,
+  blobToEmbedding,
+} from './serializers.js';
+
+export {
+  JsonToSqliteMigrator,
+  runMigration,
+  type MigrationResult,
+} from './migrator.js';