neuronlayer 0.1.0

package/src/core/context-rot/compaction.ts

@@ -0,0 +1,284 @@
+import type {
+  ContextChunk,
+  CompactionResult,
+  CompactionOptions,
+  CompactionSuggestion
+} from '../../types/documentation.js';
+import { ContextHealthMonitor } from './context-health.js';
+import { CriticalContextManager } from './critical-context.js';
+
+// Thresholds for different strategies
+const RELEVANCE_THRESHOLD_SUMMARIZE = 0.5;
+const RELEVANCE_THRESHOLD_SELECTIVE = 0.3;
+const RELEVANCE_THRESHOLD_AGGRESSIVE = 0.2;
+
+export class CompactionEngine {
+  private healthMonitor: ContextHealthMonitor;
+  private criticalManager: CriticalContextManager;
+
+  constructor(healthMonitor: ContextHealthMonitor, criticalManager: CriticalContextManager) {
+    this.healthMonitor = healthMonitor;
+    this.criticalManager = criticalManager;
+  }
+
+  suggestCompaction(): CompactionSuggestion {
+    const chunks = this.healthMonitor.getChunks();
+    const tokenLimit = this.healthMonitor.getTokenLimit();
+    const currentTokens = this.healthMonitor.getCurrentTokens();
+
+    const critical: ContextChunk[] = [];
+    const summarizable: ContextChunk[] = [];
+    const removable: ContextChunk[] = [];
+
+    for (const chunk of chunks) {
+      if (chunk.isCritical || chunk.relevanceScore >= RELEVANCE_THRESHOLD_SUMMARIZE) {
+        critical.push(chunk);
+      } else if (chunk.relevanceScore >= RELEVANCE_THRESHOLD_SELECTIVE) {
+        summarizable.push(chunk);
+      } else {
+        removable.push(chunk);
+      }
+    }
+
+    const removableTokens = removable.reduce((sum, c) => sum + c.tokens, 0);
+    const summarizableTokens = summarizable.reduce((sum, c) => sum + c.tokens, 0);
+
+    // Estimate tokens after summarization (assume 70% compression)
+    const summarizedTokens = Math.ceil(summarizableTokens * 0.3);
+    const tokensSaved = removableTokens + (summarizableTokens - summarizedTokens);
+
+    const newTokens = currentTokens - tokensSaved;
+    const newUtilization = (newTokens / tokenLimit) * 100;
+
+    return {
+      critical,
+      summarizable,
+      removable,
+      tokensSaved,
+      newUtilization: Math.round(newUtilization * 10) / 10
+    };
+  }
+
+  compact(options: CompactionOptions): CompactionResult {
+    const { strategy, preserveRecent = 5, targetUtilization, preserveCritical = true } = options;
+
+    const chunks = this.healthMonitor.getChunks();
+    const tokensBefore = this.healthMonitor.getCurrentTokens();
+
+    // Separate chunks by type
+    const recentChunks = chunks.slice(-preserveRecent);
+    const olderChunks = chunks.slice(0, -preserveRecent);
+
+    let relevanceThreshold: number;
+    switch (strategy) {
+      case 'aggressive':
+        relevanceThreshold = RELEVANCE_THRESHOLD_AGGRESSIVE;
+        break;
+      case 'selective':
+        relevanceThreshold = RELEVANCE_THRESHOLD_SELECTIVE;
+        break;
+      case 'summarize':
+      default:
+        relevanceThreshold = RELEVANCE_THRESHOLD_SUMMARIZE;
+    }
+
+    const toKeep: ContextChunk[] = [];
+    const toSummarize: ContextChunk[] = [];
+    const toRemove: ContextChunk[] = [];
+
+    for (const chunk of olderChunks) {
+      // Always preserve critical if flag is set
+      if (preserveCritical && chunk.isCritical) {
+        toKeep.push(chunk);
+      } else if (chunk.relevanceScore >= relevanceThreshold) {
+        if (strategy === 'aggressive') {
+          toSummarize.push(chunk);
+        } else {
+          toKeep.push(chunk);
+        }
+      } else if (chunk.relevanceScore >= relevanceThreshold * 0.5 && strategy !== 'aggressive') {
+        toSummarize.push(chunk);
+      } else {
+        toRemove.push(chunk);
+      }
+    }
+
+    // Generate summaries for chunks to summarize
+    const summaries = this.generateSummaries(toSummarize);
+
+    // Calculate new token count
+    const keptTokens = toKeep.reduce((sum, c) => sum + c.tokens, 0);
+    const recentTokens = recentChunks.reduce((sum, c) => sum + c.tokens, 0);
+    const summaryTokens = summaries.reduce((sum, s) => sum + this.estimateTokens(s), 0);
+
+    const tokensAfter = keptTokens + recentTokens + summaryTokens;
+    const tokensSaved = tokensBefore - tokensAfter;
+
+    // Update the health monitor with new chunks
+    this.healthMonitor.clearChunks();
+
+    // Re-add kept chunks
+    for (const chunk of toKeep) {
+      this.healthMonitor.addChunk(chunk);
+    }
+
+    // Add summary chunks
+    for (const summary of summaries) {
+      this.healthMonitor.addChunk({
+        content: summary,
+        tokens: this.estimateTokens(summary),
+        timestamp: new Date(),
+        type: 'message'
+      });
+    }
+
+    // Re-add recent chunks
+    for (const chunk of recentChunks) {
+      this.healthMonitor.addChunk(chunk);
+    }
+
+    // Check if we hit target utilization
+    if (targetUtilization) {
+      const currentUtilization = (tokensAfter / this.healthMonitor.getTokenLimit()) * 100;
+      if (currentUtilization > targetUtilization && strategy !== 'aggressive') {
+        // Recursively compact with more aggressive strategy
+        return this.compact({
+          ...options,
+          strategy: strategy === 'summarize' ? 'selective' : 'aggressive'
+        });
+      }
+    }
+
+    return {
+      success: true,
+      strategy,
+      tokensBefore,
+      tokensAfter,
+      tokensSaved,
+      preservedCritical: toKeep.filter(c => c.isCritical).length,
+      summarizedChunks: toSummarize.length,
+      removedChunks: toRemove.length,
+      summaries
+    };
+  }
+
+  private generateSummaries(chunks: ContextChunk[]): string[] {
+    if (chunks.length === 0) return [];
+
+    // Group chunks by type
+    const grouped: Record<string, ContextChunk[]> = {};
+    for (const chunk of chunks) {
+      if (!grouped[chunk.type]) {
+        grouped[chunk.type] = [];
+      }
+      grouped[chunk.type]!.push(chunk);
+    }
+
+    const summaries: string[] = [];
+
+    for (const [type, typeChunks] of Object.entries(grouped)) {
+      if (typeChunks.length === 0) continue;
+
+      // Simple extractive summary: take key sentences
+      const allContent = typeChunks.map(c => c.content).join(' ');
+      const summary = this.extractiveSummarize(allContent, type);
+      summaries.push(summary);
+    }
+
+    return summaries;
+  }
+
+  private extractiveSummarize(content: string, type: string): string {
+    // Split into sentences
+    const sentences = content.split(/[.!?]+/).map(s => s.trim()).filter(s => s.length > 10);
+
+    if (sentences.length === 0) {
+      return `[${type}]: ${content.slice(0, 100)}`;
+    }
+
+    // Score sentences by importance
+    const scored = sentences.map(sentence => ({
+      sentence,
+      score: this.scoreSentence(sentence)
+    }));
+
+    // Sort by score and take top sentences
+    scored.sort((a, b) => b.score - a.score);
+    const topSentences = scored.slice(0, Math.min(3, sentences.length));
+
+    // Sort back by original order for coherence
+    const originalOrder = topSentences.sort((a, b) => {
+      return sentences.indexOf(a.sentence) - sentences.indexOf(b.sentence);
+    });
+
+    const summary = originalOrder.map(s => s.sentence).join('. ') + '.';
+
+    return `[Summary - ${type}]: ${summary}`;
+  }
+
+  private scoreSentence(sentence: string): number {
+    let score = 0;
+
+    // Longer sentences might have more info (but not too long)
+    const wordCount = sentence.split(/\s+/).length;
+    if (wordCount >= 5 && wordCount <= 30) {
+      score += 1;
+    }
+
+    // Contains important keywords
+    const importantWords = [
+      'decided', 'choose', 'use', 'implement', 'because', 'important',
+      'must', 'should', 'require', 'need', 'critical', 'key'
+    ];
+    for (const word of importantWords) {
+      if (sentence.toLowerCase().includes(word)) {
+        score += 0.5;
+      }
+    }
+
+    // Contains technical terms (likely important)
+    const technicalPatterns = [
+      /\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\b/, // CamelCase
+      /\b\w+\(\)/, // Function calls
+      /`[^`]+`/ // Code markers
+    ];
+    for (const pattern of technicalPatterns) {
+      if (pattern.test(sentence)) {
+        score += 0.3;
+      }
+    }
+
+    return score;
+  }
+
+  private estimateTokens(text: string): number {
+    // Rough estimation: ~4 characters per token
+    return Math.ceil(text.length / 4);
+  }
+
+  autoCompact(): CompactionResult {
+    const health = this.healthMonitor.getHealth();
+
+    // Determine strategy based on health
+    let strategy: CompactionOptions['strategy'];
+    let targetUtilization: number;
+
+    if (health.health === 'critical') {
+      strategy = 'aggressive';
+      targetUtilization = 40;
+    } else if (health.health === 'warning') {
+      strategy = 'selective';
+      targetUtilization = 50;
+    } else {
+      strategy = 'summarize';
+      targetUtilization = 60;
+    }
+
+    return this.compact({
+      strategy,
+      preserveRecent: 10,
+      targetUtilization,
+      preserveCritical: true
+    });
+  }
+}

package/src/core/context-rot/context-health.ts

@@ -0,0 +1,243 @@
+import type Database from 'better-sqlite3';
+import type { ContextHealth, ContextChunk } from '../../types/documentation.js';
+import { CriticalContextManager } from './critical-context.js';
+
+// Default token limits (can be overridden)
+const DEFAULT_TOKEN_LIMIT = 100000;
+
+// Health thresholds
+const UTILIZATION_WARNING = 0.7;   // 70%
+const UTILIZATION_CRITICAL = 0.85; // 85%
+const DRIFT_WARNING = 0.3;
+const DRIFT_CRITICAL = 0.5;
+
+export class ContextHealthMonitor {
+  private db: Database.Database;
+  private criticalManager: CriticalContextManager;
+  private tokenLimit: number;
+
+  // In-memory tracking for current session
+  private contextChunks: ContextChunk[] = [];
+  private currentTokens: number = 0;
+
+  constructor(db: Database.Database, criticalManager: CriticalContextManager, tokenLimit?: number) {
+    this.db = db;
+    this.criticalManager = criticalManager;
+    this.tokenLimit = tokenLimit || DEFAULT_TOKEN_LIMIT;
+  }
+
+  setTokenLimit(limit: number): void {
+    this.tokenLimit = limit;
+  }
+
+  getTokenLimit(): number {
+    return this.tokenLimit;
+  }
+
+  addChunk(chunk: Omit<ContextChunk, 'id' | 'relevanceScore' | 'isCritical'>): ContextChunk {
+    const id = `chunk_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    const isCritical = this.criticalManager.isCritical(chunk.content);
+
+    const fullChunk: ContextChunk = {
+      ...chunk,
+      id,
+      relevanceScore: 1.0, // New chunks start with full relevance
+      isCritical
+    };
+
+    this.contextChunks.push(fullChunk);
+    this.currentTokens += chunk.tokens;
+
+    // Decay relevance of older chunks
+    this.decayRelevance();
+
+    return fullChunk;
+  }
+
+  removeChunk(id: string): boolean {
+    const index = this.contextChunks.findIndex(c => c.id === id);
+    if (index === -1) return false;
+
+    const chunk = this.contextChunks[index]!;
+    this.currentTokens -= chunk.tokens;
+    this.contextChunks.splice(index, 1);
+
+    return true;
+  }
+
+  getChunks(): ContextChunk[] {
+    return [...this.contextChunks];
+  }
+
+  clearChunks(): void {
+    this.contextChunks = [];
+    this.currentTokens = 0;
+  }
+
+  setCurrentTokens(tokens: number): void {
+    this.currentTokens = tokens;
+  }
+
+  getCurrentTokens(): number {
+    return this.currentTokens;
+  }
+
+  getHealth(driftScore: number = 0): ContextHealth {
+    const tokensUsed = this.currentTokens;
+    const utilizationPercent = (tokensUsed / this.tokenLimit) * 100;
+
+    // Calculate relevance score (average of all chunks)
+    const relevanceScore = this.contextChunks.length > 0
+      ? this.contextChunks.reduce((sum, c) => sum + c.relevanceScore, 0) / this.contextChunks.length
+      : 1.0;
+
+    // Determine health status
+    let health: ContextHealth['health'] = 'good';
+    if (utilizationPercent >= UTILIZATION_CRITICAL * 100 || driftScore >= DRIFT_CRITICAL) {
+      health = 'critical';
+    } else if (utilizationPercent >= UTILIZATION_WARNING * 100 || driftScore >= DRIFT_WARNING) {
+      health = 'warning';
+    }
+
+    // Check if compaction is needed
+    const compactionNeeded = health !== 'good';
+    const driftDetected = driftScore >= DRIFT_WARNING;
+
+    // Generate suggestions
+    const suggestions = this.generateSuggestions(health, utilizationPercent, driftScore);
+
+    // Get critical context count
+    const criticalContextCount = this.criticalManager.getCriticalCount();
+
+    const healthResult: ContextHealth = {
+      tokensUsed,
+      tokensLimit: this.tokenLimit,
+      utilizationPercent: Math.round(utilizationPercent * 10) / 10,
+      health,
+      relevanceScore: Math.round(relevanceScore * 100) / 100,
+      driftScore: Math.round(driftScore * 100) / 100,
+      criticalContextCount,
+      driftDetected,
+      compactionNeeded,
+      suggestions
+    };
+
+    // Log to history
+    this.logHealthCheck(healthResult);
+
+    return healthResult;
+  }
+
+  private generateSuggestions(
+    health: ContextHealth['health'],
+    utilization: number,
+    driftScore: number
+  ): string[] {
+    const suggestions: string[] = [];
+
+    if (health === 'good') {
+      suggestions.push('Context is healthy, no action needed');
+      return suggestions;
+    }
+
+    if (utilization >= UTILIZATION_CRITICAL * 100) {
+      suggestions.push('Context nearly full - compaction strongly recommended');
+      suggestions.push('Consider using "aggressive" compaction strategy');
+    } else if (utilization >= UTILIZATION_WARNING * 100) {
+      suggestions.push('Context getting large - consider compaction');
+      suggestions.push('Use "summarize" strategy to compress old context');
+    }
+
+    if (driftScore >= DRIFT_CRITICAL) {
+      suggestions.push('Significant drift detected - AI may be ignoring earlier instructions');
+      suggestions.push('Review critical context and add reminders if needed');
+    } else if (driftScore >= DRIFT_WARNING) {
+      suggestions.push('Some drift detected - consider marking critical items');
+    }
+
+    const criticalCount = this.criticalManager.getCriticalCount();
+    if (criticalCount === 0) {
+      suggestions.push('No critical context marked - consider marking important decisions/requirements');
+    }
+
+    return suggestions;
+  }
+
+  private decayRelevance(): void {
+    // Decay relevance of chunks based on position (older = lower relevance)
+    const totalChunks = this.contextChunks.length;
+
+    for (let i = 0; i < totalChunks; i++) {
+      const chunk = this.contextChunks[i]!;
+
+      // Critical chunks decay slower
+      const decayRate = chunk.isCritical ? 0.98 : 0.95;
+
+      // Position-based decay (older chunks have lower position)
+      const positionFactor = (i + 1) / totalChunks;
+
+      // New relevance is combination of decay and position
+      chunk.relevanceScore = Math.max(0.1, chunk.relevanceScore * decayRate * (0.5 + 0.5 * positionFactor));
+    }
+  }
+
+  private logHealthCheck(health: ContextHealth): void {
+    try {
+      const stmt = this.db.prepare(`
+        INSERT INTO context_health_history
+        (tokens_used, tokens_limit, utilization_percent, drift_score, relevance_score, health, compaction_triggered)
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+      `);
+
+      stmt.run(
+        health.tokensUsed,
+        health.tokensLimit,
+        health.utilizationPercent,
+        health.driftScore,
+        health.relevanceScore,
+        health.health,
+        0
+      );
+    } catch {
+      // Ignore logging errors
+    }
+  }
+
+  getHealthHistory(limit: number = 20): Array<{
+    timestamp: Date;
+    health: ContextHealth['health'];
+    utilizationPercent: number;
+    driftScore: number;
+  }> {
+    try {
+      const stmt = this.db.prepare(`
+        SELECT timestamp, health, utilization_percent, drift_score
+        FROM context_health_history
+        ORDER BY timestamp DESC
+        LIMIT ?
+      `);
+
+      const rows = stmt.all(limit) as Array<{
+        timestamp: number;
+        health: string;
+        utilization_percent: number;
+        drift_score: number;
+      }>;
+
+      return rows.map(row => ({
+        timestamp: new Date(row.timestamp * 1000),
+        health: row.health as ContextHealth['health'],
+        utilizationPercent: row.utilization_percent,
+        driftScore: row.drift_score
+      }));
+    } catch {
+      return [];
+    }
+  }
+
+  estimateTokens(text: string): number {
+    // Rough estimation: ~4 characters per token for English
+    // This is a simple heuristic; real implementation would use a proper tokenizer
+    return Math.ceil(text.length / 4);
+  }
+}