neuronlayer 0.1.9 → 0.2.1

package/src/core/context-rot/compaction.ts

@@ -1,284 +0,0 @@
-import type {
-  ContextChunk,
-  CompactionResult,
-  CompactionOptions,
-  CompactionSuggestion
-} from '../../types/documentation.js';
-import { ContextHealthMonitor } from './context-health.js';
-import { CriticalContextManager } from './critical-context.js';
-
-// Thresholds for different strategies
-const RELEVANCE_THRESHOLD_SUMMARIZE = 0.5;
-const RELEVANCE_THRESHOLD_SELECTIVE = 0.3;
-const RELEVANCE_THRESHOLD_AGGRESSIVE = 0.2;
-
-export class CompactionEngine {
-  private healthMonitor: ContextHealthMonitor;
-  private criticalManager: CriticalContextManager;
-
-  constructor(healthMonitor: ContextHealthMonitor, criticalManager: CriticalContextManager) {
-    this.healthMonitor = healthMonitor;
-    this.criticalManager = criticalManager;
-  }
-
-  suggestCompaction(): CompactionSuggestion {
-    const chunks = this.healthMonitor.getChunks();
-    const tokenLimit = this.healthMonitor.getTokenLimit();
-    const currentTokens = this.healthMonitor.getCurrentTokens();
-
-    const critical: ContextChunk[] = [];
-    const summarizable: ContextChunk[] = [];
-    const removable: ContextChunk[] = [];
-
-    for (const chunk of chunks) {
-      if (chunk.isCritical || chunk.relevanceScore >= RELEVANCE_THRESHOLD_SUMMARIZE) {
-        critical.push(chunk);
-      } else if (chunk.relevanceScore >= RELEVANCE_THRESHOLD_SELECTIVE) {
-        summarizable.push(chunk);
-      } else {
-        removable.push(chunk);
-      }
-    }
-
-    const removableTokens = removable.reduce((sum, c) => sum + c.tokens, 0);
-    const summarizableTokens = summarizable.reduce((sum, c) => sum + c.tokens, 0);
-
-    // Estimate tokens after summarization (assume 70% compression)
-    const summarizedTokens = Math.ceil(summarizableTokens * 0.3);
-    const tokensSaved = removableTokens + (summarizableTokens - summarizedTokens);
-
-    const newTokens = currentTokens - tokensSaved;
-    const newUtilization = (newTokens / tokenLimit) * 100;
-
-    return {
-      critical,
-      summarizable,
-      removable,
-      tokensSaved,
-      newUtilization: Math.round(newUtilization * 10) / 10
-    };
-  }
-
-  compact(options: CompactionOptions): CompactionResult {
-    const { strategy, preserveRecent = 5, targetUtilization, preserveCritical = true } = options;
-
-    const chunks = this.healthMonitor.getChunks();
-    const tokensBefore = this.healthMonitor.getCurrentTokens();
-
-    // Separate chunks by type
-    const recentChunks = chunks.slice(-preserveRecent);
-    const olderChunks = chunks.slice(0, -preserveRecent);
-
-    let relevanceThreshold: number;
-    switch (strategy) {
-      case 'aggressive':
-        relevanceThreshold = RELEVANCE_THRESHOLD_AGGRESSIVE;
-        break;
-      case 'selective':
-        relevanceThreshold = RELEVANCE_THRESHOLD_SELECTIVE;
-        break;
-      case 'summarize':
-      default:
-        relevanceThreshold = RELEVANCE_THRESHOLD_SUMMARIZE;
-    }
-
-    const toKeep: ContextChunk[] = [];
-    const toSummarize: ContextChunk[] = [];
-    const toRemove: ContextChunk[] = [];
-
-    for (const chunk of olderChunks) {
-      // Always preserve critical if flag is set
-      if (preserveCritical && chunk.isCritical) {
-        toKeep.push(chunk);
-      } else if (chunk.relevanceScore >= relevanceThreshold) {
-        if (strategy === 'aggressive') {
-          toSummarize.push(chunk);
-        } else {
-          toKeep.push(chunk);
-        }
-      } else if (chunk.relevanceScore >= relevanceThreshold * 0.5 && strategy !== 'aggressive') {
-        toSummarize.push(chunk);
-      } else {
-        toRemove.push(chunk);
-      }
-    }
-
-    // Generate summaries for chunks to summarize
-    const summaries = this.generateSummaries(toSummarize);
-
-    // Calculate new token count
-    const keptTokens = toKeep.reduce((sum, c) => sum + c.tokens, 0);
-    const recentTokens = recentChunks.reduce((sum, c) => sum + c.tokens, 0);
-    const summaryTokens = summaries.reduce((sum, s) => sum + this.estimateTokens(s), 0);
-
-    const tokensAfter = keptTokens + recentTokens + summaryTokens;
-    const tokensSaved = tokensBefore - tokensAfter;
-
-    // Update the health monitor with new chunks
-    this.healthMonitor.clearChunks();
-
-    // Re-add kept chunks
-    for (const chunk of toKeep) {
-      this.healthMonitor.addChunk(chunk);
-    }
-
-    // Add summary chunks
-    for (const summary of summaries) {
-      this.healthMonitor.addChunk({
-        content: summary,
-        tokens: this.estimateTokens(summary),
-        timestamp: new Date(),
-        type: 'message'
-      });
-    }
-
-    // Re-add recent chunks
-    for (const chunk of recentChunks) {
-      this.healthMonitor.addChunk(chunk);
-    }
-
-    // Check if we hit target utilization
-    if (targetUtilization) {
-      const currentUtilization = (tokensAfter / this.healthMonitor.getTokenLimit()) * 100;
-      if (currentUtilization > targetUtilization && strategy !== 'aggressive') {
-        // Recursively compact with more aggressive strategy
-        return this.compact({
-          ...options,
-          strategy: strategy === 'summarize' ? 'selective' : 'aggressive'
-        });
-      }
-    }
-
-    return {
-      success: true,
-      strategy,
-      tokensBefore,
-      tokensAfter,
-      tokensSaved,
-      preservedCritical: toKeep.filter(c => c.isCritical).length,
-      summarizedChunks: toSummarize.length,
-      removedChunks: toRemove.length,
-      summaries
-    };
-  }
-
-  private generateSummaries(chunks: ContextChunk[]): string[] {
-    if (chunks.length === 0) return [];
-
-    // Group chunks by type
-    const grouped: Record<string, ContextChunk[]> = {};
-    for (const chunk of chunks) {
-      if (!grouped[chunk.type]) {
-        grouped[chunk.type] = [];
-      }
-      grouped[chunk.type]!.push(chunk);
-    }
-
-    const summaries: string[] = [];
-
-    for (const [type, typeChunks] of Object.entries(grouped)) {
-      if (typeChunks.length === 0) continue;
-
-      // Simple extractive summary: take key sentences
-      const allContent = typeChunks.map(c => c.content).join(' ');
-      const summary = this.extractiveSummarize(allContent, type);
-      summaries.push(summary);
-    }
-
-    return summaries;
-  }
-
-  private extractiveSummarize(content: string, type: string): string {
-    // Split into sentences
-    const sentences = content.split(/[.!?]+/).map(s => s.trim()).filter(s => s.length > 10);
-
-    if (sentences.length === 0) {
-      return `[${type}]: ${content.slice(0, 100)}`;
-    }
-
-    // Score sentences by importance
-    const scored = sentences.map(sentence => ({
-      sentence,
-      score: this.scoreSentence(sentence)
-    }));
-
-    // Sort by score and take top sentences
-    scored.sort((a, b) => b.score - a.score);
-    const topSentences = scored.slice(0, Math.min(3, sentences.length));
-
-    // Sort back by original order for coherence
-    const originalOrder = topSentences.sort((a, b) => {
-      return sentences.indexOf(a.sentence) - sentences.indexOf(b.sentence);
-    });
-
-    const summary = originalOrder.map(s => s.sentence).join('. ') + '.';
-
-    return `[Summary - ${type}]: ${summary}`;
-  }
-
-  private scoreSentence(sentence: string): number {
-    let score = 0;
-
-    // Longer sentences might have more info (but not too long)
-    const wordCount = sentence.split(/\s+/).length;
-    if (wordCount >= 5 && wordCount <= 30) {
-      score += 1;
-    }
-
-    // Contains important keywords
-    const importantWords = [
-      'decided', 'choose', 'use', 'implement', 'because', 'important',
-      'must', 'should', 'require', 'need', 'critical', 'key'
-    ];
-    for (const word of importantWords) {
-      if (sentence.toLowerCase().includes(word)) {
-        score += 0.5;
-      }
-    }
-
-    // Contains technical terms (likely important)
-    const technicalPatterns = [
-      /\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\b/, // CamelCase
-      /\b\w+\(\)/, // Function calls
-      /`[^`]+`/ // Code markers
-    ];
-    for (const pattern of technicalPatterns) {
-      if (pattern.test(sentence)) {
-        score += 0.3;
-      }
-    }
-
-    return score;
-  }
-
-  private estimateTokens(text: string): number {
-    // Rough estimation: ~4 characters per token
-    return Math.ceil(text.length / 4);
-  }
-
-  autoCompact(): CompactionResult {
-    const health = this.healthMonitor.getHealth();
-
-    // Determine strategy based on health
-    let strategy: CompactionOptions['strategy'];
-    let targetUtilization: number;
-
-    if (health.health === 'critical') {
-      strategy = 'aggressive';
-      targetUtilization = 40;
-    } else if (health.health === 'warning') {
-      strategy = 'selective';
-      targetUtilization = 50;
-    } else {
-      strategy = 'summarize';
-      targetUtilization = 60;
-    }
-
-    return this.compact({
-      strategy,
-      preserveRecent: 10,
-      targetUtilization,
-      preserveCritical: true
-    });
-  }
-}

package/src/core/context-rot/context-health.ts

@@ -1,243 +0,0 @@
-import type Database from 'better-sqlite3';
-import type { ContextHealth, ContextChunk } from '../../types/documentation.js';
-import { CriticalContextManager } from './critical-context.js';
-
-// Default token limits (can be overridden)
-const DEFAULT_TOKEN_LIMIT = 100000;
-
-// Health thresholds
-const UTILIZATION_WARNING = 0.7;   // 70%
-const UTILIZATION_CRITICAL = 0.85; // 85%
-const DRIFT_WARNING = 0.3;
-const DRIFT_CRITICAL = 0.5;
-
-export class ContextHealthMonitor {
-  private db: Database.Database;
-  private criticalManager: CriticalContextManager;
-  private tokenLimit: number;
-
-  // In-memory tracking for current session
-  private contextChunks: ContextChunk[] = [];
-  private currentTokens: number = 0;
-
-  constructor(db: Database.Database, criticalManager: CriticalContextManager, tokenLimit?: number) {
-    this.db = db;
-    this.criticalManager = criticalManager;
-    this.tokenLimit = tokenLimit || DEFAULT_TOKEN_LIMIT;
-  }
-
-  setTokenLimit(limit: number): void {
-    this.tokenLimit = limit;
-  }
-
-  getTokenLimit(): number {
-    return this.tokenLimit;
-  }
-
-  addChunk(chunk: Omit<ContextChunk, 'id' | 'relevanceScore' | 'isCritical'>): ContextChunk {
-    const id = `chunk_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
-    const isCritical = this.criticalManager.isCritical(chunk.content);
-
-    const fullChunk: ContextChunk = {
-      ...chunk,
-      id,
-      relevanceScore: 1.0, // New chunks start with full relevance
-      isCritical
-    };
-
-    this.contextChunks.push(fullChunk);
-    this.currentTokens += chunk.tokens;
-
-    // Decay relevance of older chunks
-    this.decayRelevance();
-
-    return fullChunk;
-  }
-
-  removeChunk(id: string): boolean {
-    const index = this.contextChunks.findIndex(c => c.id === id);
-    if (index === -1) return false;
-
-    const chunk = this.contextChunks[index]!;
-    this.currentTokens -= chunk.tokens;
-    this.contextChunks.splice(index, 1);
-
-    return true;
-  }
-
-  getChunks(): ContextChunk[] {
-    return [...this.contextChunks];
-  }
-
-  clearChunks(): void {
-    this.contextChunks = [];
-    this.currentTokens = 0;
-  }
-
-  setCurrentTokens(tokens: number): void {
-    this.currentTokens = tokens;
-  }
-
-  getCurrentTokens(): number {
-    return this.currentTokens;
-  }
-
-  getHealth(driftScore: number = 0): ContextHealth {
-    const tokensUsed = this.currentTokens;
-    const utilizationPercent = (tokensUsed / this.tokenLimit) * 100;
-
-    // Calculate relevance score (average of all chunks)
-    const relevanceScore = this.contextChunks.length > 0
-      ? this.contextChunks.reduce((sum, c) => sum + c.relevanceScore, 0) / this.contextChunks.length
-      : 1.0;
-
-    // Determine health status
-    let health: ContextHealth['health'] = 'good';
-    if (utilizationPercent >= UTILIZATION_CRITICAL * 100 || driftScore >= DRIFT_CRITICAL) {
-      health = 'critical';
-    } else if (utilizationPercent >= UTILIZATION_WARNING * 100 || driftScore >= DRIFT_WARNING) {
-      health = 'warning';
-    }
-
-    // Check if compaction is needed
-    const compactionNeeded = health !== 'good';
-    const driftDetected = driftScore >= DRIFT_WARNING;
-
-    // Generate suggestions
-    const suggestions = this.generateSuggestions(health, utilizationPercent, driftScore);
-
-    // Get critical context count
-    const criticalContextCount = this.criticalManager.getCriticalCount();
-
-    const healthResult: ContextHealth = {
-      tokensUsed,
-      tokensLimit: this.tokenLimit,
-      utilizationPercent: Math.round(utilizationPercent * 10) / 10,
-      health,
-      relevanceScore: Math.round(relevanceScore * 100) / 100,
-      driftScore: Math.round(driftScore * 100) / 100,
-      criticalContextCount,
-      driftDetected,
-      compactionNeeded,
-      suggestions
-    };
-
-    // Log to history
-    this.logHealthCheck(healthResult);
-
-    return healthResult;
-  }
-
-  private generateSuggestions(
-    health: ContextHealth['health'],
-    utilization: number,
-    driftScore: number
-  ): string[] {
-    const suggestions: string[] = [];
-
-    if (health === 'good') {
-      suggestions.push('Context is healthy, no action needed');
-      return suggestions;
-    }
-
-    if (utilization >= UTILIZATION_CRITICAL * 100) {
-      suggestions.push('Context nearly full - compaction strongly recommended');
-      suggestions.push('Consider using "aggressive" compaction strategy');
-    } else if (utilization >= UTILIZATION_WARNING * 100) {
-      suggestions.push('Context getting large - consider compaction');
-      suggestions.push('Use "summarize" strategy to compress old context');
-    }
-
-    if (driftScore >= DRIFT_CRITICAL) {
-      suggestions.push('Significant drift detected - AI may be ignoring earlier instructions');
-      suggestions.push('Review critical context and add reminders if needed');
-    } else if (driftScore >= DRIFT_WARNING) {
-      suggestions.push('Some drift detected - consider marking critical items');
-    }
-
-    const criticalCount = this.criticalManager.getCriticalCount();
-    if (criticalCount === 0) {
-      suggestions.push('No critical context marked - consider marking important decisions/requirements');
-    }
-
-    return suggestions;
-  }
-
-  private decayRelevance(): void {
-    // Decay relevance of chunks based on position (older = lower relevance)
-    const totalChunks = this.contextChunks.length;
-
-    for (let i = 0; i < totalChunks; i++) {
-      const chunk = this.contextChunks[i]!;
-
-      // Critical chunks decay slower
-      const decayRate = chunk.isCritical ? 0.98 : 0.95;
-
-      // Position-based decay (older chunks have lower position)
-      const positionFactor = (i + 1) / totalChunks;
-
-      // New relevance is combination of decay and position
-      chunk.relevanceScore = Math.max(0.1, chunk.relevanceScore * decayRate * (0.5 + 0.5 * positionFactor));
-    }
-  }
-
-  private logHealthCheck(health: ContextHealth): void {
-    try {
-      const stmt = this.db.prepare(`
-        INSERT INTO context_health_history
-        (tokens_used, tokens_limit, utilization_percent, drift_score, relevance_score, health, compaction_triggered)
-        VALUES (?, ?, ?, ?, ?, ?, ?)
-      `);
-
-      stmt.run(
-        health.tokensUsed,
-        health.tokensLimit,
-        health.utilizationPercent,
-        health.driftScore,
-        health.relevanceScore,
-        health.health,
-        0
-      );
-    } catch {
-      // Ignore logging errors
-    }
-  }
-
-  getHealthHistory(limit: number = 20): Array<{
-    timestamp: Date;
-    health: ContextHealth['health'];
-    utilizationPercent: number;
-    driftScore: number;
-  }> {
-    try {
-      const stmt = this.db.prepare(`
-        SELECT timestamp, health, utilization_percent, drift_score
-        FROM context_health_history
-        ORDER BY timestamp DESC
-        LIMIT ?
-      `);
-
-      const rows = stmt.all(limit) as Array<{
-        timestamp: number;
-        health: string;
-        utilization_percent: number;
-        drift_score: number;
-      }>;
-
-      return rows.map(row => ({
-        timestamp: new Date(row.timestamp * 1000),
-        health: row.health as ContextHealth['health'],
-        utilizationPercent: row.utilization_percent,
-        driftScore: row.drift_score
-      }));
-    } catch {
-      return [];
-    }
-  }
-
-  estimateTokens(text: string): number {
-    // Rough estimation: ~4 characters per token for English
-    // This is a simple heuristic; real implementation would use a proper tokenizer
-    return Math.ceil(text.length / 4);
-  }
-}