neuronlayer 0.1.9 → 0.2.0

package/src/core/context-rot/context-rot-prevention.ts

@@ -1,213 +0,0 @@
-import type Database from 'better-sqlite3';
-import type {
-  ContextHealth,
-  CompactionResult,
-  CompactionOptions,
-  CompactionSuggestion,
-  CriticalContext,
-  DriftResult
-} from '../../types/documentation.js';
-import { ContextHealthMonitor } from './context-health.js';
-import { DriftDetector } from './drift-detector.js';
-import { CompactionEngine } from './compaction.js';
-import { CriticalContextManager } from './critical-context.js';
-
-interface Message {
-  role: 'user' | 'assistant' | 'system';
-  content: string;
-  timestamp?: Date;
-}
-
-export class ContextRotPrevention {
-  private healthMonitor: ContextHealthMonitor;
-  private driftDetector: DriftDetector;
-  private compactionEngine: CompactionEngine;
-  private criticalManager: CriticalContextManager;
-  private db: Database.Database;
-
-  constructor(db: Database.Database, tokenLimit?: number) {
-    this.db = db;
-    this.criticalManager = new CriticalContextManager(db);
-    this.healthMonitor = new ContextHealthMonitor(db, this.criticalManager, tokenLimit);
-    this.driftDetector = new DriftDetector(this.criticalManager);
-    this.compactionEngine = new CompactionEngine(this.healthMonitor, this.criticalManager);
-  }
-
-  // ========== Context Health ==========
-
-  getContextHealth(): ContextHealth {
-    const driftResult = this.driftDetector.detectDrift();
-    return this.healthMonitor.getHealth(driftResult.driftScore);
-  }
-
-  setTokenLimit(limit: number): void {
-    this.healthMonitor.setTokenLimit(limit);
-  }
-
-  setCurrentTokens(tokens: number): void {
-    this.healthMonitor.setCurrentTokens(tokens);
-  }
-
-  addContextChunk(content: string, tokens: number, type: 'message' | 'decision' | 'requirement' | 'instruction' | 'code' = 'message'): void {
-    this.healthMonitor.addChunk({
-      content,
-      tokens,
-      timestamp: new Date(),
-      type
-    });
-  }
-
-  // ========== Message Tracking ==========
-
-  addMessage(message: Message): void {
-    this.driftDetector.addMessage(message);
-
-    // Also add as context chunk
-    const tokens = this.healthMonitor.estimateTokens(message.content);
-    this.addContextChunk(message.content, tokens, 'message');
-  }
-
-  clearConversation(): void {
-    this.driftDetector.clearHistory();
-    this.healthMonitor.clearChunks();
-  }
-
-  // ========== Drift Detection ==========
-
-  detectDrift(): DriftResult {
-    return this.driftDetector.detectDrift();
-  }
-
-  addRequirement(requirement: string): void {
-    this.driftDetector.addRequirement(requirement);
-  }
-
-  getRequirements(): string[] {
-    return this.driftDetector.getInitialRequirements();
-  }
-
-  // ========== Critical Context ==========
-
-  markCritical(
-    content: string,
-    options?: {
-      type?: CriticalContext['type'];
-      reason?: string;
-      source?: string;
-    }
-  ): CriticalContext {
-    return this.criticalManager.markCritical(content, options);
-  }
-
-  getCriticalContext(type?: CriticalContext['type']): CriticalContext[] {
-    return this.criticalManager.getCriticalContext(type);
-  }
-
-  removeCritical(id: string): boolean {
-    return this.criticalManager.removeCritical(id);
-  }
-
-  getAllCriticalContent(): string {
-    return this.criticalManager.getAllCriticalContent();
-  }
-
-  // ========== Compaction ==========
-
-  suggestCompaction(): CompactionSuggestion {
-    return this.compactionEngine.suggestCompaction();
-  }
-
-  triggerCompaction(options: CompactionOptions): CompactionResult {
-    const result = this.compactionEngine.compact(options);
-
-    // Log compaction event
-    this.logCompactionEvent(result);
-
-    return result;
-  }
-
-  autoCompact(): CompactionResult {
-    const result = this.compactionEngine.autoCompact();
-    this.logCompactionEvent(result);
-    return result;
-  }
-
-  private logCompactionEvent(result: CompactionResult): void {
-    try {
-      const stmt = this.db.prepare(`
-        UPDATE context_health_history
-        SET compaction_triggered = 1
-        WHERE id = (SELECT MAX(id) FROM context_health_history)
-      `);
-      stmt.run();
-    } catch {
-      // Ignore logging errors
-    }
-  }
-
-  // ========== Health History ==========
-
-  getHealthHistory(limit: number = 20): Array<{
-    timestamp: Date;
-    health: ContextHealth['health'];
-    utilizationPercent: number;
-    driftScore: number;
-  }> {
-    return this.healthMonitor.getHealthHistory(limit);
-  }
-
-  // ========== Utilities ==========
-
-  estimateTokens(text: string): number {
-    return this.healthMonitor.estimateTokens(text);
-  }
-
-  isCritical(content: string): boolean {
-    return this.criticalManager.isCritical(content);
-  }
-
-  extractCriticalFromText(text: string): Array<{ content: string; type: CriticalContext['type'] }> {
-    return this.criticalManager.extractCriticalFromText(text);
-  }
-
-  // ========== Summary for AI ==========
-
-  getContextSummaryForAI(): string {
-    const health = this.getContextHealth();
-    const critical = this.getAllCriticalContent();
-    const drift = this.detectDrift();
-
-    const parts: string[] = [];
-
-    // Health status
-    parts.push(`Context Health: ${health.health.toUpperCase()} (${health.utilizationPercent}% used)`);
-
-    if (health.driftDetected) {
-      parts.push(`\nWARNING: Drift detected (score: ${health.driftScore})`);
-
-      if (drift.missingRequirements.length > 0) {
-        parts.push('\nMissing requirements:');
-        for (const req of drift.missingRequirements.slice(0, 3)) {
-          parts.push(`- ${req}`);
-        }
-      }
-
-      if (drift.suggestedReminders.length > 0) {
-        parts.push('\nReminders:');
-        for (const reminder of drift.suggestedReminders.slice(0, 3)) {
-          parts.push(`- ${reminder}`);
-        }
-      }
-    }
-
-    if (critical) {
-      parts.push(`\n${critical}`);
-    }
-
-    if (health.compactionNeeded) {
-      parts.push(`\nSuggestion: ${health.suggestions[0] || 'Consider compacting context'}`);
-    }
-
-    return parts.join('\n');
-  }
-}

package/src/core/context-rot/critical-context.ts

@@ -1,221 +0,0 @@
-import { randomUUID } from 'crypto';
-import type Database from 'better-sqlite3';
-import type { CriticalContext } from '../../types/documentation.js';
-
-// Patterns for automatically detecting critical content
-const CRITICAL_PATTERNS = [
-  // Explicit instructions
-  { pattern: /\b(always|never|must|required|mandatory)\b/i, type: 'instruction' as const },
-
-  // Decisions
-  { pattern: /\b(we decided|the decision|chose to|decided to|will use)\b/i, type: 'decision' as const },
-
-  // Requirements
-  { pattern: /\b(requirement|constraint|rule|spec|specification)\b/i, type: 'requirement' as const },
-
-  // User preferences
-  { pattern: /\b(i prefer|i want|don't want|please don't|make sure)\b/i, type: 'instruction' as const },
-
-  // Technical constraints
-  { pattern: /\b(cannot|must not|impossible|not allowed|forbidden)\b/i, type: 'requirement' as const },
-
-  // Important markers
-  { pattern: /\b(important|critical|essential|crucial|key point)\b/i, type: 'instruction' as const }
-];
-
-export class CriticalContextManager {
-  private db: Database.Database;
-
-  constructor(db: Database.Database) {
-    this.db = db;
-  }
-
-  markCritical(
-    content: string,
-    options?: {
-      type?: CriticalContext['type'];
-      reason?: string;
-      source?: string;
-      neverCompress?: boolean;
-    }
-  ): CriticalContext {
-    const id = randomUUID();
-    const type = options?.type || this.inferType(content);
-    const neverCompress = options?.neverCompress ?? true;
-
-    const critical: CriticalContext = {
-      id,
-      type,
-      content,
-      reason: options?.reason,
-      source: options?.source,
-      createdAt: new Date(),
-      neverCompress
-    };
-
-    const stmt = this.db.prepare(`
-      INSERT INTO critical_context (id, type, content, reason, source, never_compress, created_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?)
-    `);
-
-    stmt.run(
-      id,
-      type,
-      content,
-      options?.reason || null,
-      options?.source || null,
-      neverCompress ? 1 : 0,
-      Math.floor(Date.now() / 1000)
-    );
-
-    return critical;
-  }
-
-  getCriticalContext(type?: CriticalContext['type']): CriticalContext[] {
-    let query = `
-      SELECT id, type, content, reason, source, never_compress, created_at
-      FROM critical_context
-    `;
-    const params: string[] = [];
-
-    if (type) {
-      query += ' WHERE type = ?';
-      params.push(type);
-    }
-
-    query += ' ORDER BY created_at DESC';
-
-    const stmt = this.db.prepare(query);
-    const rows = (params.length > 0 ? stmt.all(params[0]) : stmt.all()) as Array<{
-      id: string;
-      type: string;
-      content: string;
-      reason: string | null;
-      source: string | null;
-      never_compress: number;
-      created_at: number;
-    }>;
-
-    return rows.map(row => ({
-      id: row.id,
-      type: row.type as CriticalContext['type'],
-      content: row.content,
-      reason: row.reason || undefined,
-      source: row.source || undefined,
-      createdAt: new Date(row.created_at * 1000),
-      neverCompress: row.never_compress === 1
-    }));
-  }
-
-  getCriticalById(id: string): CriticalContext | null {
-    const stmt = this.db.prepare(`
-      SELECT id, type, content, reason, source, never_compress, created_at
-      FROM critical_context
-      WHERE id = ?
-    `);
-
-    const row = stmt.get(id) as {
-      id: string;
-      type: string;
-      content: string;
-      reason: string | null;
-      source: string | null;
-      never_compress: number;
-      created_at: number;
-    } | undefined;
-
-    if (!row) return null;
-
-    return {
-      id: row.id,
-      type: row.type as CriticalContext['type'],
-      content: row.content,
-      reason: row.reason || undefined,
-      source: row.source || undefined,
-      createdAt: new Date(row.created_at * 1000),
-      neverCompress: row.never_compress === 1
-    };
-  }
-
-  removeCritical(id: string): boolean {
-    const stmt = this.db.prepare('DELETE FROM critical_context WHERE id = ?');
-    const result = stmt.run(id);
-    return result.changes > 0;
-  }
-
-  getCriticalCount(): number {
-    const stmt = this.db.prepare('SELECT COUNT(*) as count FROM critical_context');
-    const result = stmt.get() as { count: number };
-    return result.count;
-  }
-
-  isCritical(content: string): boolean {
-    return CRITICAL_PATTERNS.some(p => p.pattern.test(content));
-  }
-
-  inferType(content: string): CriticalContext['type'] {
-    for (const { pattern, type } of CRITICAL_PATTERNS) {
-      if (pattern.test(content)) {
-        return type;
-      }
-    }
-    return 'custom';
-  }
-
-  extractCriticalFromText(text: string): Array<{ content: string; type: CriticalContext['type'] }> {
-    const results: Array<{ content: string; type: CriticalContext['type'] }> = [];
-
-    // Split into sentences
-    const sentences = text.split(/[.!?]+/).map(s => s.trim()).filter(Boolean);
-
-    for (const sentence of sentences) {
-      if (this.isCritical(sentence)) {
-        results.push({
-          content: sentence,
-          type: this.inferType(sentence)
-        });
-      }
-    }
-
-    return results;
-  }
-
-  getAllCriticalContent(): string {
-    const critical = this.getCriticalContext();
-
-    if (critical.length === 0) {
-      return '';
-    }
-
-    const grouped = {
-      decision: [] as string[],
-      requirement: [] as string[],
-      instruction: [] as string[],
-      custom: [] as string[]
-    };
-
-    for (const item of critical) {
-      const arr = grouped[item.type as keyof typeof grouped];
-      if (arr) {
-        arr.push(item.content);
-      }
-    }
-
-    const parts: string[] = [];
-
-    if (grouped.decision.length > 0) {
-      parts.push('DECISIONS:\n' + grouped.decision.map(d => `- ${d}`).join('\n'));
-    }
-    if (grouped.requirement.length > 0) {
-      parts.push('REQUIREMENTS:\n' + grouped.requirement.map(r => `- ${r}`).join('\n'));
-    }
-    if (grouped.instruction.length > 0) {
-      parts.push('INSTRUCTIONS:\n' + grouped.instruction.map(i => `- ${i}`).join('\n'));
-    }
-    if (grouped.custom.length > 0) {
-      parts.push('OTHER CRITICAL:\n' + grouped.custom.map(c => `- ${c}`).join('\n'));
-    }
-
-    return parts.join('\n\n');
-  }
-}

package/src/core/context-rot/drift-detector.ts

@@ -1,255 +0,0 @@
-import type { DriftResult, Contradiction, CriticalContext } from '../../types/documentation.js';
-import { CriticalContextManager } from './critical-context.js';
-
-interface Message {
-  role: 'user' | 'assistant' | 'system';
-  content: string;
-  timestamp?: Date;
-}
-
-// Keywords that indicate topic areas
-const TOPIC_KEYWORDS: Record<string, string[]> = {
-  authentication: ['auth', 'login', 'jwt', 'session', 'token', 'oauth', 'password'],
-  database: ['database', 'db', 'sql', 'query', 'table', 'schema', 'migration'],
-  api: ['api', 'endpoint', 'rest', 'graphql', 'route', 'request', 'response'],
-  frontend: ['react', 'vue', 'component', 'ui', 'css', 'html', 'dom'],
-  testing: ['test', 'spec', 'mock', 'assert', 'coverage', 'jest', 'vitest'],
-  deployment: ['deploy', 'docker', 'kubernetes', 'ci', 'cd', 'pipeline'],
-  security: ['security', 'encrypt', 'hash', 'vulnerability', 'xss', 'csrf'],
-  performance: ['performance', 'optimize', 'cache', 'speed', 'memory', 'latency']
-};
-
-// Contradiction patterns (earlier statement vs later statement)
-const CONTRADICTION_PATTERNS = [
-  { earlier: /will use (\w+)/i, later: /use (\w+) instead/i },
-  { earlier: /decided on (\w+)/i, later: /switch(?:ed|ing)? to (\w+)/i },
-  { earlier: /must (\w+)/i, later: /don't need to (\w+)/i },
-  { earlier: /always (\w+)/i, later: /never (\w+)/i }
-];
-
-export class DriftDetector {
-  private criticalManager: CriticalContextManager;
-  private conversationHistory: Message[] = [];
-  private initialRequirements: string[] = [];
-
-  constructor(criticalManager: CriticalContextManager) {
-    this.criticalManager = criticalManager;
-  }
-
-  addMessage(message: Message): void {
-    this.conversationHistory.push({
-      ...message,
-      timestamp: message.timestamp || new Date()
-    });
-
-    // Extract requirements from early user messages
-    if (message.role === 'user' && this.conversationHistory.length <= 5) {
-      const extracted = this.extractRequirements(message.content);
-      this.initialRequirements.push(...extracted);
-    }
-  }
-
-  clearHistory(): void {
-    this.conversationHistory = [];
-    this.initialRequirements = [];
-  }
-
-  getHistory(): Message[] {
-    return [...this.conversationHistory];
-  }
-
-  detectDrift(): DriftResult {
-    const criticalContext = this.criticalManager.getCriticalContext();
-    const recentMessages = this.conversationHistory.slice(-10);
-    const earlyMessages = this.conversationHistory.slice(0, 10);
-
-    // Calculate drift score based on multiple factors
-    const requirementAdherence = this.checkRequirementAdherence(recentMessages);
-    const contradictions = this.findContradictions();
-    const topicShift = this.calculateTopicShift(earlyMessages, recentMessages);
-
-    // Weighted drift score
-    const driftScore = Math.min(1, (
-      (1 - requirementAdherence.score) * 0.4 +
-      (contradictions.length > 0 ? Math.min(contradictions.length * 0.15, 0.3) : 0) +
-      topicShift * 0.3
-    ));
-
-    // Generate suggested reminders
-    const suggestedReminders = this.generateReminders(
-      requirementAdherence.missing,
-      criticalContext
-    );
-
-    return {
-      driftScore: Math.round(driftScore * 100) / 100,
-      driftDetected: driftScore >= 0.3,
-      missingRequirements: requirementAdherence.missing,
-      contradictions,
-      suggestedReminders,
-      topicShift: Math.round(topicShift * 100) / 100
-    };
-  }
-
-  private extractRequirements(text: string): string[] {
-    const requirements: string[] = [];
-    const patterns = [
-      /(?:must|should|need to|have to|required to)\s+(.+?)(?:[.!?]|$)/gi,
-      /(?:make sure|ensure|always)\s+(.+?)(?:[.!?]|$)/gi,
-      /(?:don't|never|avoid)\s+(.+?)(?:[.!?]|$)/gi
-    ];
-
-    for (const pattern of patterns) {
-      let match;
-      while ((match = pattern.exec(text)) !== null) {
-        if (match[1] && match[1].length > 5 && match[1].length < 200) {
-          requirements.push(match[1].trim());
-        }
-      }
-    }
-
-    return requirements;
-  }
-
-  private checkRequirementAdherence(recentMessages: Message[]): {
-    score: number;
-    missing: string[];
-  } {
-    if (this.initialRequirements.length === 0) {
-      return { score: 1, missing: [] };
-    }
-
-    const recentText = recentMessages
-      .filter(m => m.role === 'assistant')
-      .map(m => m.content.toLowerCase())
-      .join(' ');
-
-    const missing: string[] = [];
-    let found = 0;
-
-    for (const req of this.initialRequirements) {
-      // Check if requirement is being addressed in recent responses
-      const keywords = req.toLowerCase().split(/\s+/).filter(w => w.length > 3);
-      const matchCount = keywords.filter(kw => recentText.includes(kw)).length;
-
-      if (matchCount >= keywords.length * 0.5) {
-        found++;
-      } else {
-        missing.push(req);
-      }
-    }
-
-    const score = this.initialRequirements.length > 0
-      ? found / this.initialRequirements.length
-      : 1;
-
-    return { score, missing };
-  }
-
-  private findContradictions(): Contradiction[] {
-    const contradictions: Contradiction[] = [];
-    const messages = this.conversationHistory;
-
-    // Compare each message with later messages
-    for (let i = 0; i < messages.length - 1; i++) {
-      const earlier = messages[i]!;
-
-      for (let j = i + 1; j < messages.length; j++) {
-        const later = messages[j]!;
-
-        // Only check assistant responses for contradictions
-        if (earlier.role !== 'assistant' || later.role !== 'assistant') continue;
-
-        for (const pattern of CONTRADICTION_PATTERNS) {
-          const earlierMatch = earlier.content.match(pattern.earlier);
-          const laterMatch = later.content.match(pattern.later);
-
-          if (earlierMatch && laterMatch) {
-            // Check if it's talking about the same thing
-            const earlierSubject = earlierMatch[1]?.toLowerCase();
-            const laterSubject = laterMatch[1]?.toLowerCase();
-
-            if (earlierSubject && laterSubject && earlierSubject !== laterSubject) {
-              contradictions.push({
-                earlier: earlier.content.slice(0, 100),
-                later: later.content.slice(0, 100),
-                severity: j - i > 10 ? 'high' : j - i > 5 ? 'medium' : 'low'
-              });
-            }
-          }
-        }
-      }
-    }
-
-    // Limit to most recent contradictions
-    return contradictions.slice(-5);
-  }
-
-  private calculateTopicShift(
-    earlyMessages: Message[],
-    recentMessages: Message[]
-  ): number {
-    const earlyTopics = this.extractTopics(earlyMessages);
-    const recentTopics = this.extractTopics(recentMessages);
-
-    if (earlyTopics.size === 0 || recentTopics.size === 0) {
-      return 0;
-    }
-
-    // Calculate Jaccard similarity
-    const intersection = new Set([...earlyTopics].filter(t => recentTopics.has(t)));
-    const union = new Set([...earlyTopics, ...recentTopics]);
-
-    const similarity = intersection.size / union.size;
-
-    // Topic shift is inverse of similarity
-    return 1 - similarity;
-  }
-
-  private extractTopics(messages: Message[]): Set<string> {
-    const topics = new Set<string>();
-    const text = messages.map(m => m.content.toLowerCase()).join(' ');
-
-    for (const [topic, keywords] of Object.entries(TOPIC_KEYWORDS)) {
-      for (const keyword of keywords) {
-        if (text.includes(keyword)) {
-          topics.add(topic);
-          break;
-        }
-      }
-    }
-
-    return topics;
-  }
-
-  private generateReminders(
-    missingRequirements: string[],
-    criticalContext: CriticalContext[]
-  ): string[] {
-    const reminders: string[] = [];
-
-    // Add reminders for missing requirements
-    for (const req of missingRequirements.slice(0, 3)) {
-      reminders.push(`Remember: ${req}`);
-    }
-
-    // Add reminders for critical context
-    for (const critical of criticalContext.slice(0, 3)) {
-      if (critical.type === 'decision') {
-        reminders.push(`Decision: ${critical.content}`);
-      } else if (critical.type === 'requirement') {
-        reminders.push(`Requirement: ${critical.content}`);
-      }
-    }
-
-    return reminders;
-  }
-
-  getInitialRequirements(): string[] {
-    return [...this.initialRequirements];
-  }
-
-  addRequirement(requirement: string): void {
-    this.initialRequirements.push(requirement);
-  }
-}

package/src/core/context-rot/index.ts

@@ -1,7 +0,0 @@
-// Context Rot Prevention Module - Barrel Export
-
-export { ContextRotPrevention } from './context-rot-prevention.js';
-export { ContextHealthMonitor } from './context-health.js';
-export { DriftDetector } from './drift-detector.js';
-export { CompactionEngine } from './compaction.js';
-export { CriticalContextManager } from './critical-context.js';