neuronlayer 0.1.0

package/src/core/context-rot/context-rot-prevention.ts

@@ -0,0 +1,213 @@
+import type Database from 'better-sqlite3';
+import type {
+  ContextHealth,
+  CompactionResult,
+  CompactionOptions,
+  CompactionSuggestion,
+  CriticalContext,
+  DriftResult
+} from '../../types/documentation.js';
+import { ContextHealthMonitor } from './context-health.js';
+import { DriftDetector } from './drift-detector.js';
+import { CompactionEngine } from './compaction.js';
+import { CriticalContextManager } from './critical-context.js';
+
+interface Message {
+  role: 'user' | 'assistant' | 'system';
+  content: string;
+  timestamp?: Date;
+}
+
+export class ContextRotPrevention {
+  private healthMonitor: ContextHealthMonitor;
+  private driftDetector: DriftDetector;
+  private compactionEngine: CompactionEngine;
+  private criticalManager: CriticalContextManager;
+  private db: Database.Database;
+
+  constructor(db: Database.Database, tokenLimit?: number) {
+    this.db = db;
+    this.criticalManager = new CriticalContextManager(db);
+    this.healthMonitor = new ContextHealthMonitor(db, this.criticalManager, tokenLimit);
+    this.driftDetector = new DriftDetector(this.criticalManager);
+    this.compactionEngine = new CompactionEngine(this.healthMonitor, this.criticalManager);
+  }
+
+  // ========== Context Health ==========
+
+  getContextHealth(): ContextHealth {
+    const driftResult = this.driftDetector.detectDrift();
+    return this.healthMonitor.getHealth(driftResult.driftScore);
+  }
+
+  setTokenLimit(limit: number): void {
+    this.healthMonitor.setTokenLimit(limit);
+  }
+
+  setCurrentTokens(tokens: number): void {
+    this.healthMonitor.setCurrentTokens(tokens);
+  }
+
+  addContextChunk(content: string, tokens: number, type: 'message' | 'decision' | 'requirement' | 'instruction' | 'code' = 'message'): void {
+    this.healthMonitor.addChunk({
+      content,
+      tokens,
+      timestamp: new Date(),
+      type
+    });
+  }
+
+  // ========== Message Tracking ==========
+
+  addMessage(message: Message): void {
+    this.driftDetector.addMessage(message);
+
+    // Also add as context chunk
+    const tokens = this.healthMonitor.estimateTokens(message.content);
+    this.addContextChunk(message.content, tokens, 'message');
+  }
+
+  clearConversation(): void {
+    this.driftDetector.clearHistory();
+    this.healthMonitor.clearChunks();
+  }
+
+  // ========== Drift Detection ==========
+
+  detectDrift(): DriftResult {
+    return this.driftDetector.detectDrift();
+  }
+
+  addRequirement(requirement: string): void {
+    this.driftDetector.addRequirement(requirement);
+  }
+
+  getRequirements(): string[] {
+    return this.driftDetector.getInitialRequirements();
+  }
+
+  // ========== Critical Context ==========
+
+  markCritical(
+    content: string,
+    options?: {
+      type?: CriticalContext['type'];
+      reason?: string;
+      source?: string;
+    }
+  ): CriticalContext {
+    return this.criticalManager.markCritical(content, options);
+  }
+
+  getCriticalContext(type?: CriticalContext['type']): CriticalContext[] {
+    return this.criticalManager.getCriticalContext(type);
+  }
+
+  removeCritical(id: string): boolean {
+    return this.criticalManager.removeCritical(id);
+  }
+
+  getAllCriticalContent(): string {
+    return this.criticalManager.getAllCriticalContent();
+  }
+
+  // ========== Compaction ==========
+
+  suggestCompaction(): CompactionSuggestion {
+    return this.compactionEngine.suggestCompaction();
+  }
+
+  triggerCompaction(options: CompactionOptions): CompactionResult {
+    const result = this.compactionEngine.compact(options);
+
+    // Log compaction event
+    this.logCompactionEvent(result);
+
+    return result;
+  }
+
+  autoCompact(): CompactionResult {
+    const result = this.compactionEngine.autoCompact();
+    this.logCompactionEvent(result);
+    return result;
+  }
+
+  private logCompactionEvent(result: CompactionResult): void {
+    try {
+      const stmt = this.db.prepare(`
+        UPDATE context_health_history
+        SET compaction_triggered = 1
+        WHERE id = (SELECT MAX(id) FROM context_health_history)
+      `);
+      stmt.run();
+    } catch {
+      // Ignore logging errors
+    }
+  }
+
+  // ========== Health History ==========
+
+  getHealthHistory(limit: number = 20): Array<{
+    timestamp: Date;
+    health: ContextHealth['health'];
+    utilizationPercent: number;
+    driftScore: number;
+  }> {
+    return this.healthMonitor.getHealthHistory(limit);
+  }
+
+  // ========== Utilities ==========
+
+  estimateTokens(text: string): number {
+    return this.healthMonitor.estimateTokens(text);
+  }
+
+  isCritical(content: string): boolean {
+    return this.criticalManager.isCritical(content);
+  }
+
+  extractCriticalFromText(text: string): Array<{ content: string; type: CriticalContext['type'] }> {
+    return this.criticalManager.extractCriticalFromText(text);
+  }
+
+  // ========== Summary for AI ==========
+
+  getContextSummaryForAI(): string {
+    const health = this.getContextHealth();
+    const critical = this.getAllCriticalContent();
+    const drift = this.detectDrift();
+
+    const parts: string[] = [];
+
+    // Health status
+    parts.push(`Context Health: ${health.health.toUpperCase()} (${health.utilizationPercent}% used)`);
+
+    if (health.driftDetected) {
+      parts.push(`\nWARNING: Drift detected (score: ${health.driftScore})`);
+
+      if (drift.missingRequirements.length > 0) {
+        parts.push('\nMissing requirements:');
+        for (const req of drift.missingRequirements.slice(0, 3)) {
+          parts.push(`- ${req}`);
+        }
+      }
+
+      if (drift.suggestedReminders.length > 0) {
+        parts.push('\nReminders:');
+        for (const reminder of drift.suggestedReminders.slice(0, 3)) {
+          parts.push(`- ${reminder}`);
+        }
+      }
+    }
+
+    if (critical) {
+      parts.push(`\n${critical}`);
+    }
+
+    if (health.compactionNeeded) {
+      parts.push(`\nSuggestion: ${health.suggestions[0] || 'Consider compacting context'}`);
+    }
+
+    return parts.join('\n');
+  }
+}

package/src/core/context-rot/critical-context.ts

@@ -0,0 +1,221 @@
+import { randomUUID } from 'crypto';
+import type Database from 'better-sqlite3';
+import type { CriticalContext } from '../../types/documentation.js';
+
+// Patterns for automatically detecting critical content
+const CRITICAL_PATTERNS = [
+  // Explicit instructions
+  { pattern: /\b(always|never|must|required|mandatory)\b/i, type: 'instruction' as const },
+
+  // Decisions
+  { pattern: /\b(we decided|the decision|chose to|decided to|will use)\b/i, type: 'decision' as const },
+
+  // Requirements
+  { pattern: /\b(requirement|constraint|rule|spec|specification)\b/i, type: 'requirement' as const },
+
+  // User preferences
+  { pattern: /\b(i prefer|i want|don't want|please don't|make sure)\b/i, type: 'instruction' as const },
+
+  // Technical constraints
+  { pattern: /\b(cannot|must not|impossible|not allowed|forbidden)\b/i, type: 'requirement' as const },
+
+  // Important markers
+  { pattern: /\b(important|critical|essential|crucial|key point)\b/i, type: 'instruction' as const }
+];
+
+export class CriticalContextManager {
+  private db: Database.Database;
+
+  constructor(db: Database.Database) {
+    this.db = db;
+  }
+
+  markCritical(
+    content: string,
+    options?: {
+      type?: CriticalContext['type'];
+      reason?: string;
+      source?: string;
+      neverCompress?: boolean;
+    }
+  ): CriticalContext {
+    const id = randomUUID();
+    const type = options?.type || this.inferType(content);
+    const neverCompress = options?.neverCompress ?? true;
+
+    const critical: CriticalContext = {
+      id,
+      type,
+      content,
+      reason: options?.reason,
+      source: options?.source,
+      createdAt: new Date(),
+      neverCompress
+    };
+
+    const stmt = this.db.prepare(`
+      INSERT INTO critical_context (id, type, content, reason, source, never_compress, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    stmt.run(
+      id,
+      type,
+      content,
+      options?.reason || null,
+      options?.source || null,
+      neverCompress ? 1 : 0,
+      Math.floor(Date.now() / 1000)
+    );
+
+    return critical;
+  }
+
+  getCriticalContext(type?: CriticalContext['type']): CriticalContext[] {
+    let query = `
+      SELECT id, type, content, reason, source, never_compress, created_at
+      FROM critical_context
+    `;
+    const params: string[] = [];
+
+    if (type) {
+      query += ' WHERE type = ?';
+      params.push(type);
+    }
+
+    query += ' ORDER BY created_at DESC';
+
+    const stmt = this.db.prepare(query);
+    const rows = (params.length > 0 ? stmt.all(params[0]) : stmt.all()) as Array<{
+      id: string;
+      type: string;
+      content: string;
+      reason: string | null;
+      source: string | null;
+      never_compress: number;
+      created_at: number;
+    }>;
+
+    return rows.map(row => ({
+      id: row.id,
+      type: row.type as CriticalContext['type'],
+      content: row.content,
+      reason: row.reason || undefined,
+      source: row.source || undefined,
+      createdAt: new Date(row.created_at * 1000),
+      neverCompress: row.never_compress === 1
+    }));
+  }
+
+  getCriticalById(id: string): CriticalContext | null {
+    const stmt = this.db.prepare(`
+      SELECT id, type, content, reason, source, never_compress, created_at
+      FROM critical_context
+      WHERE id = ?
+    `);
+
+    const row = stmt.get(id) as {
+      id: string;
+      type: string;
+      content: string;
+      reason: string | null;
+      source: string | null;
+      never_compress: number;
+      created_at: number;
+    } | undefined;
+
+    if (!row) return null;
+
+    return {
+      id: row.id,
+      type: row.type as CriticalContext['type'],
+      content: row.content,
+      reason: row.reason || undefined,
+      source: row.source || undefined,
+      createdAt: new Date(row.created_at * 1000),
+      neverCompress: row.never_compress === 1
+    };
+  }
+
+  removeCritical(id: string): boolean {
+    const stmt = this.db.prepare('DELETE FROM critical_context WHERE id = ?');
+    const result = stmt.run(id);
+    return result.changes > 0;
+  }
+
+  getCriticalCount(): number {
+    const stmt = this.db.prepare('SELECT COUNT(*) as count FROM critical_context');
+    const result = stmt.get() as { count: number };
+    return result.count;
+  }
+
+  isCritical(content: string): boolean {
+    return CRITICAL_PATTERNS.some(p => p.pattern.test(content));
+  }
+
+  inferType(content: string): CriticalContext['type'] {
+    for (const { pattern, type } of CRITICAL_PATTERNS) {
+      if (pattern.test(content)) {
+        return type;
+      }
+    }
+    return 'custom';
+  }
+
+  extractCriticalFromText(text: string): Array<{ content: string; type: CriticalContext['type'] }> {
+    const results: Array<{ content: string; type: CriticalContext['type'] }> = [];
+
+    // Split into sentences
+    const sentences = text.split(/[.!?]+/).map(s => s.trim()).filter(Boolean);
+
+    for (const sentence of sentences) {
+      if (this.isCritical(sentence)) {
+        results.push({
+          content: sentence,
+          type: this.inferType(sentence)
+        });
+      }
+    }
+
+    return results;
+  }
+
+  getAllCriticalContent(): string {
+    const critical = this.getCriticalContext();
+
+    if (critical.length === 0) {
+      return '';
+    }
+
+    const grouped = {
+      decision: [] as string[],
+      requirement: [] as string[],
+      instruction: [] as string[],
+      custom: [] as string[]
+    };
+
+    for (const item of critical) {
+      const arr = grouped[item.type as keyof typeof grouped];
+      if (arr) {
+        arr.push(item.content);
+      }
+    }
+
+    const parts: string[] = [];
+
+    if (grouped.decision.length > 0) {
+      parts.push('DECISIONS:\n' + grouped.decision.map(d => `- ${d}`).join('\n'));
+    }
+    if (grouped.requirement.length > 0) {
+      parts.push('REQUIREMENTS:\n' + grouped.requirement.map(r => `- ${r}`).join('\n'));
+    }
+    if (grouped.instruction.length > 0) {
+      parts.push('INSTRUCTIONS:\n' + grouped.instruction.map(i => `- ${i}`).join('\n'));
+    }
+    if (grouped.custom.length > 0) {
+      parts.push('OTHER CRITICAL:\n' + grouped.custom.map(c => `- ${c}`).join('\n'));
+    }
+
+    return parts.join('\n\n');
+  }
+}

package/src/core/context-rot/drift-detector.ts

@@ -0,0 +1,255 @@
+import type { DriftResult, Contradiction, CriticalContext } from '../../types/documentation.js';
+import { CriticalContextManager } from './critical-context.js';
+
+interface Message {
+  role: 'user' | 'assistant' | 'system';
+  content: string;
+  timestamp?: Date;
+}
+
+// Keywords that indicate topic areas
+const TOPIC_KEYWORDS: Record<string, string[]> = {
+  authentication: ['auth', 'login', 'jwt', 'session', 'token', 'oauth', 'password'],
+  database: ['database', 'db', 'sql', 'query', 'table', 'schema', 'migration'],
+  api: ['api', 'endpoint', 'rest', 'graphql', 'route', 'request', 'response'],
+  frontend: ['react', 'vue', 'component', 'ui', 'css', 'html', 'dom'],
+  testing: ['test', 'spec', 'mock', 'assert', 'coverage', 'jest', 'vitest'],
+  deployment: ['deploy', 'docker', 'kubernetes', 'ci', 'cd', 'pipeline'],
+  security: ['security', 'encrypt', 'hash', 'vulnerability', 'xss', 'csrf'],
+  performance: ['performance', 'optimize', 'cache', 'speed', 'memory', 'latency']
+};
+
+// Contradiction patterns (earlier statement vs later statement)
+const CONTRADICTION_PATTERNS = [
+  { earlier: /will use (\w+)/i, later: /use (\w+) instead/i },
+  { earlier: /decided on (\w+)/i, later: /switch(?:ed|ing)? to (\w+)/i },
+  { earlier: /must (\w+)/i, later: /don't need to (\w+)/i },
+  { earlier: /always (\w+)/i, later: /never (\w+)/i }
+];
+
+export class DriftDetector {
+  private criticalManager: CriticalContextManager;
+  private conversationHistory: Message[] = [];
+  private initialRequirements: string[] = [];
+
+  constructor(criticalManager: CriticalContextManager) {
+    this.criticalManager = criticalManager;
+  }
+
+  addMessage(message: Message): void {
+    this.conversationHistory.push({
+      ...message,
+      timestamp: message.timestamp || new Date()
+    });
+
+    // Extract requirements from early user messages
+    if (message.role === 'user' && this.conversationHistory.length <= 5) {
+      const extracted = this.extractRequirements(message.content);
+      this.initialRequirements.push(...extracted);
+    }
+  }
+
+  clearHistory(): void {
+    this.conversationHistory = [];
+    this.initialRequirements = [];
+  }
+
+  getHistory(): Message[] {
+    return [...this.conversationHistory];
+  }
+
+  detectDrift(): DriftResult {
+    const criticalContext = this.criticalManager.getCriticalContext();
+    const recentMessages = this.conversationHistory.slice(-10);
+    const earlyMessages = this.conversationHistory.slice(0, 10);
+
+    // Calculate drift score based on multiple factors
+    const requirementAdherence = this.checkRequirementAdherence(recentMessages);
+    const contradictions = this.findContradictions();
+    const topicShift = this.calculateTopicShift(earlyMessages, recentMessages);
+
+    // Weighted drift score
+    const driftScore = Math.min(1, (
+      (1 - requirementAdherence.score) * 0.4 +
+      (contradictions.length > 0 ? Math.min(contradictions.length * 0.15, 0.3) : 0) +
+      topicShift * 0.3
+    ));
+
+    // Generate suggested reminders
+    const suggestedReminders = this.generateReminders(
+      requirementAdherence.missing,
+      criticalContext
+    );
+
+    return {
+      driftScore: Math.round(driftScore * 100) / 100,
+      driftDetected: driftScore >= 0.3,
+      missingRequirements: requirementAdherence.missing,
+      contradictions,
+      suggestedReminders,
+      topicShift: Math.round(topicShift * 100) / 100
+    };
+  }
+
+  private extractRequirements(text: string): string[] {
+    const requirements: string[] = [];
+    const patterns = [
+      /(?:must|should|need to|have to|required to)\s+(.+?)(?:[.!?]|$)/gi,
+      /(?:make sure|ensure|always)\s+(.+?)(?:[.!?]|$)/gi,
+      /(?:don't|never|avoid)\s+(.+?)(?:[.!?]|$)/gi
+    ];
+
+    for (const pattern of patterns) {
+      let match;
+      while ((match = pattern.exec(text)) !== null) {
+        if (match[1] && match[1].length > 5 && match[1].length < 200) {
+          requirements.push(match[1].trim());
+        }
+      }
+    }
+
+    return requirements;
+  }
+
+  private checkRequirementAdherence(recentMessages: Message[]): {
+    score: number;
+    missing: string[];
+  } {
+    if (this.initialRequirements.length === 0) {
+      return { score: 1, missing: [] };
+    }
+
+    const recentText = recentMessages
+      .filter(m => m.role === 'assistant')
+      .map(m => m.content.toLowerCase())
+      .join(' ');
+
+    const missing: string[] = [];
+    let found = 0;
+
+    for (const req of this.initialRequirements) {
+      // Check if requirement is being addressed in recent responses
+      const keywords = req.toLowerCase().split(/\s+/).filter(w => w.length > 3);
+      const matchCount = keywords.filter(kw => recentText.includes(kw)).length;
+
+      if (matchCount >= keywords.length * 0.5) {
+        found++;
+      } else {
+        missing.push(req);
+      }
+    }
+
+    const score = this.initialRequirements.length > 0
+      ? found / this.initialRequirements.length
+      : 1;
+
+    return { score, missing };
+  }
+
+  private findContradictions(): Contradiction[] {
+    const contradictions: Contradiction[] = [];
+    const messages = this.conversationHistory;
+
+    // Compare each message with later messages
+    for (let i = 0; i < messages.length - 1; i++) {
+      const earlier = messages[i]!;
+
+      for (let j = i + 1; j < messages.length; j++) {
+        const later = messages[j]!;
+
+        // Only check assistant responses for contradictions
+        if (earlier.role !== 'assistant' || later.role !== 'assistant') continue;
+
+        for (const pattern of CONTRADICTION_PATTERNS) {
+          const earlierMatch = earlier.content.match(pattern.earlier);
+          const laterMatch = later.content.match(pattern.later);
+
+          if (earlierMatch && laterMatch) {
+            // Check if it's talking about the same thing
+            const earlierSubject = earlierMatch[1]?.toLowerCase();
+            const laterSubject = laterMatch[1]?.toLowerCase();
+
+            if (earlierSubject && laterSubject && earlierSubject !== laterSubject) {
+              contradictions.push({
+                earlier: earlier.content.slice(0, 100),
+                later: later.content.slice(0, 100),
+                severity: j - i > 10 ? 'high' : j - i > 5 ? 'medium' : 'low'
+              });
+            }
+          }
+        }
+      }
+    }
+
+    // Limit to most recent contradictions
+    return contradictions.slice(-5);
+  }
+
+  private calculateTopicShift(
+    earlyMessages: Message[],
+    recentMessages: Message[]
+  ): number {
+    const earlyTopics = this.extractTopics(earlyMessages);
+    const recentTopics = this.extractTopics(recentMessages);
+
+    if (earlyTopics.size === 0 || recentTopics.size === 0) {
+      return 0;
+    }
+
+    // Calculate Jaccard similarity
+    const intersection = new Set([...earlyTopics].filter(t => recentTopics.has(t)));
+    const union = new Set([...earlyTopics, ...recentTopics]);
+
+    const similarity = intersection.size / union.size;
+
+    // Topic shift is inverse of similarity
+    return 1 - similarity;
+  }
+
+  private extractTopics(messages: Message[]): Set<string> {
+    const topics = new Set<string>();
+    const text = messages.map(m => m.content.toLowerCase()).join(' ');
+
+    for (const [topic, keywords] of Object.entries(TOPIC_KEYWORDS)) {
+      for (const keyword of keywords) {
+        if (text.includes(keyword)) {
+          topics.add(topic);
+          break;
+        }
+      }
+    }
+
+    return topics;
+  }
+
+  private generateReminders(
+    missingRequirements: string[],
+    criticalContext: CriticalContext[]
+  ): string[] {
+    const reminders: string[] = [];
+
+    // Add reminders for missing requirements
+    for (const req of missingRequirements.slice(0, 3)) {
+      reminders.push(`Remember: ${req}`);
+    }
+
+    // Add reminders for critical context
+    for (const critical of criticalContext.slice(0, 3)) {
+      if (critical.type === 'decision') {
+        reminders.push(`Decision: ${critical.content}`);
+      } else if (critical.type === 'requirement') {
+        reminders.push(`Requirement: ${critical.content}`);
+      }
+    }
+
+    return reminders;
+  }
+
+  getInitialRequirements(): string[] {
+    return [...this.initialRequirements];
+  }
+
+  addRequirement(requirement: string): void {
+    this.initialRequirements.push(requirement);
+  }
+}

package/src/core/context-rot/index.ts

@@ -0,0 +1,7 @@
+// Context Rot Prevention Module - Barrel Export
+
+export { ContextRotPrevention } from './context-rot-prevention.js';
+export { ContextHealthMonitor } from './context-health.js';
+export { DriftDetector } from './drift-detector.js';
+export { CompactionEngine } from './compaction.js';
+export { CriticalContextManager } from './critical-context.js';