neuronlayer 0.1.0

package/src/core/context.ts

@@ -0,0 +1,263 @@
+import { dirname, relative, basename } from 'path';
+import { Tier1Storage } from '../storage/tier1.js';
+import { Tier2Storage } from '../storage/tier2.js';
+import { Tier3Storage } from '../storage/tier3.js';
+import { EmbeddingGenerator } from '../indexing/embeddings.js';
+import { TokenBudget, estimateTokens } from '../utils/tokens.js';
+import type { FeatureContextManager } from './feature-context.js';
+import type { AssembledContext, AssemblyOptions, SearchResult, Decision, ContextParts, HotContext } from '../types/index.js';
+
+export class ContextAssembler {
+  private tier1: Tier1Storage;
+  private tier2: Tier2Storage;
+  private tier3: Tier3Storage;
+  private embeddingGenerator: EmbeddingGenerator;
+  private featureContextManager: FeatureContextManager | null = null;
+
+  constructor(
+    tier1: Tier1Storage,
+    tier2: Tier2Storage,
+    tier3: Tier3Storage,
+    embeddingGenerator: EmbeddingGenerator
+  ) {
+    this.tier1 = tier1;
+    this.tier2 = tier2;
+    this.tier3 = tier3;
+    this.embeddingGenerator = embeddingGenerator;
+  }
+
+  // Set feature context manager (injected after construction to avoid circular deps)
+  setFeatureContextManager(manager: FeatureContextManager): void {
+    this.featureContextManager = manager;
+  }
+
+  async assemble(query: string, options: AssemblyOptions = {}): Promise<AssembledContext> {
+    const budget = new TokenBudget(options.maxTokens || 6000);
+    const queryEmbedding = await this.embeddingGenerator.embed(query);
+
+    // Step 0: HOT CONTEXT (HIGHEST PRIORITY) - Active Feature Context
+    let hotContext: HotContext | null = null;
+    if (this.featureContextManager) {
+      hotContext = this.featureContextManager.getHotContext();
+      if (hotContext.files.length > 0 || hotContext.changes.length > 0) {
+        const hotText = this.formatHotContext(hotContext);
+        if (budget.canFit(hotText)) {
+          budget.allocate(hotText, 'hot-context');
+        }
+      }
+    }
+
+    // Step 1: Load and format Tier 1 (working context)
+    const working = this.tier1.getContext();
+    const workingText = this.formatTier1(working);
+
+    if (workingText && budget.canFit(workingText)) {
+      budget.allocate(workingText, 'tier1');
+    }
+
+    // Step 2: Semantic search in Tier 2
+    const searchResults = this.tier2.search(queryEmbedding, 20);
+    const rankedResults = this.rankResults(searchResults, options.currentFile);
+
+    const tier2Content: SearchResult[] = [];
+    for (const result of rankedResults) {
+      const formatted = this.formatSearchResult(result);
+      if (budget.canFit(formatted)) {
+        budget.allocate(formatted, 'tier2');
+        tier2Content.push(result);
+      } else {
+        break;
+      }
+    }
+
+    // Step 3: Query Tier 3 if budget remains
+    const tier3Content: string[] = [];
+    if (budget.remaining() > 200) {
+      const archives = this.tier3.searchRelevant(query, 3);
+      for (const archive of archives) {
+        if (archive.summary && budget.canFit(archive.summary)) {
+          budget.allocate(archive.summary, 'tier3');
+          tier3Content.push(archive.summary);
+        }
+      }
+    }
+
+    // Step 4: Get relevant decisions
+    let decisions: Decision[] = [];
+    try {
+      decisions = this.tier2.searchDecisions(queryEmbedding, 5);
+    } catch {
+      decisions = this.tier2.getRecentDecisions(5);
+    }
+
+    const decisionsText = this.formatDecisions(decisions);
+    if (decisionsText && budget.canFit(decisionsText)) {
+      budget.allocate(decisionsText, 'decisions');
+    }
+
+    // Step 5: Assemble final context
+    const context = this.formatFinalContext({
+      working: { activeFile: working.activeFile },
+      relevant: tier2Content,
+      archive: tier3Content,
+      decisions
+    }, hotContext);
+
+    return {
+      context,
+      sources: tier2Content.map(r => r.file),
+      tokenCount: budget.used(),
+      decisions
+    };
+  }
+
+  private formatHotContext(hot: HotContext): string {
+    let output = `## Active Feature Context\n\n`;
+
+    if (hot.summary) {
+      output += `**${hot.summary}**\n\n`;
+    }
+
+    if (hot.files.length > 0) {
+      output += `### Files You're Working On\n`;
+      for (const f of hot.files.slice(0, 8)) {
+        output += `- \`${f.path}\` (touched ${f.touchCount}x)\n`;
+      }
+      output += `\n`;
+    }
+
+    if (hot.changes.length > 0) {
+      output += `### Recent Changes\n`;
+      for (const c of hot.changes.slice(0, 5)) {
+        const fileName = basename(c.file);
+        output += `- \`${fileName}\`: ${c.diff}\n`;
+      }
+      output += `\n`;
+    }
+
+    if (hot.queries.length > 0) {
+      output += `### Recent Questions\n`;
+      for (const q of hot.queries.slice(0, 3)) {
+        output += `- "${q.query}"\n`;
+      }
+      output += `\n`;
+    }
+
+    return output;
+  }
+
+  private formatTier1(context: { activeFile: { path: string; content: string; language: string } | null }): string {
+    if (!context.activeFile) {
+      return '';
+    }
+
+    return `### Currently Active File
+File: ${context.activeFile.path}
+\`\`\`${context.activeFile.language}
+${context.activeFile.content}
+\`\`\`
+`;
+  }
+
+  private formatSearchResult(result: SearchResult): string {
+    const score = result.score !== undefined ? result.score : result.similarity;
+    return `#### ${result.file} (relevance: ${(score * 100).toFixed(0)}%)
+\`\`\`
+${result.preview}
+\`\`\`
+`;
+  }
+
+  private formatDecisions(decisions: Decision[]): string {
+    if (decisions.length === 0) {
+      return '';
+    }
+
+    return decisions.map(d => {
+      const date = d.createdAt.toLocaleDateString();
+      return `- **${d.title}** (${date})
+  ${d.description}`;
+    }).join('\n\n');
+  }
+
+  private rankResults(results: SearchResult[], currentFile?: string): SearchResult[] {
+    const filesViewed = this.tier1.getFilesViewed();
+
+    return results
+      .map(r => {
+        let score = r.similarity;
+
+        // Boost: Same directory as current file
+        if (currentFile && dirname(r.file) === dirname(currentFile)) {
+          score *= 1.5;
+        }
+
+        // Boost: Recently modified (within 24 hours)
+        const hoursSinceModified = (Date.now() - r.lastModified) / 3600000;
+        if (hoursSinceModified < 24) {
+          score *= 1 + (0.3 * (24 - hoursSinceModified) / 24);
+        }
+
+        // Boost: Recently viewed in session
+        if (filesViewed.includes(r.file)) {
+          score *= 1.3;
+        }
+
+        return { ...r, score };
+      })
+      .sort((a, b) => (b.score || 0) - (a.score || 0));
+  }
+
+  private formatFinalContext(parts: ContextParts, hotContext?: HotContext | null): string {
+    const sections: string[] = [];
+
+    // Hot context first (highest priority)
+    if (hotContext && (hotContext.files.length > 0 || hotContext.changes.length > 0)) {
+      sections.push(this.formatHotContext(hotContext));
+    }
+
+    sections.push('## Codebase Context\n');
+
+    // Working file
+    if (parts.working.activeFile) {
+      sections.push(`### Working File
+File: ${parts.working.activeFile.path}
+\`\`\`${parts.working.activeFile.language}
+${parts.working.activeFile.content}
+\`\`\`
+`);
+    }
+
+    // Relevant code
+    if (parts.relevant.length > 0) {
+      sections.push('### Relevant Code\n');
+      for (const r of parts.relevant) {
+        const score = r.score !== undefined ? r.score : r.similarity;
+        sections.push(`#### ${r.file} (relevance: ${(score * 100).toFixed(0)}%)
+\`\`\`
+${r.preview}
+\`\`\`
+`);
+      }
+    }
+
+    // Decisions
+    if (parts.decisions.length > 0) {
+      sections.push('### Architecture Decisions\n');
+      for (const d of parts.decisions) {
+        const date = d.createdAt.toLocaleDateString();
+        sections.push(`- **${d.title}** (${date})
+  ${d.description}
+`);
+      }
+    }
+
+    // Archive
+    if (parts.archive.length > 0) {
+      sections.push('### Historical Context\n');
+      sections.push(parts.archive.join('\n\n'));
+    }
+
+    return sections.join('\n').trim();
+  }
+}

package/src/core/decision-extractor.ts

@@ -0,0 +1,339 @@
+import { execSync } from 'child_process';
+import { readFileSync, existsSync } from 'fs';
+import { join } from 'path';
+import { glob } from 'glob';
+import type { Decision } from '../types/index.js';
+import { randomUUID } from 'crypto';
+
+interface ExtractedDecision {
+  title: string;
+  description: string;
+  source: 'git' | 'comment' | 'adr';
+  file?: string;
+  line?: number;
+  tags: string[];
+}
+
+// Patterns for detecting decisions in commit messages
+const COMMIT_PATTERNS = [
+  // Conventional commits with architectural significance
+  /^(?:feat|refactor|perf|build|ci)\(([^)]+)\):\s*(.+)/i,
+  // Explicit decision markers
+  /^(?:DECISION|ARCHITECTURE|ADR):\s*(.+)/i,
+  // "Use X instead of Y" pattern
+  /use\s+(\w+)\s+(?:instead of|over|rather than)\s+(\w+)/i,
+  // "Switch to X" pattern
+  /switch(?:ed|ing)?\s+to\s+(\w+)/i,
+  // "Implement X pattern"
+  /implement(?:ed|ing)?\s+(\w+)\s+pattern/i,
+];
+
+// Patterns for detecting decisions in code comments
+const COMMENT_PATTERNS = [
+  // Explicit decision markers
+  { pattern: /(?:\/\/|#|\/\*)\s*DECISION:\s*(.+?)(?:\*\/)?$/gm, tag: 'decision' },
+  { pattern: /(?:\/\/|#|\/\*)\s*ARCHITECTURE:\s*(.+?)(?:\*\/)?$/gm, tag: 'architecture' },
+  { pattern: /(?:\/\/|#|\/\*)\s*ADR:\s*(.+?)(?:\*\/)?$/gm, tag: 'adr' },
+  { pattern: /(?:\/\/|#|\/\*)\s*WHY:\s*(.+?)(?:\*\/)?$/gm, tag: 'rationale' },
+  { pattern: /(?:\/\/|#|\/\*)\s*NOTE:\s*(.+?)(?:\*\/)?$/gm, tag: 'note' },
+  { pattern: /(?:\/\/|#|\/\*)\s*IMPORTANT:\s*(.+?)(?:\*\/)?$/gm, tag: 'important' },
+];
+
+// Tags to assign based on content
+const TAG_KEYWORDS: Record<string, string[]> = {
+  database: ['database', 'db', 'sql', 'postgres', 'mysql', 'mongo', 'redis', 'sqlite', 'orm', 'prisma'],
+  authentication: ['auth', 'login', 'jwt', 'oauth', 'session', 'password', 'token'],
+  api: ['api', 'rest', 'graphql', 'endpoint', 'route', 'http'],
+  performance: ['performance', 'cache', 'optimize', 'speed', 'fast', 'slow', 'memory'],
+  security: ['security', 'encrypt', 'hash', 'ssl', 'https', 'csrf', 'xss', 'injection'],
+  testing: ['test', 'jest', 'vitest', 'mocha', 'cypress', 'e2e', 'unit'],
+  infrastructure: ['docker', 'kubernetes', 'aws', 'gcp', 'azure', 'deploy', 'ci', 'cd'],
+  frontend: ['react', 'vue', 'angular', 'svelte', 'css', 'ui', 'component'],
+  backend: ['server', 'node', 'express', 'fastify', 'middleware'],
+};
+
+export class DecisionExtractor {
+  private projectPath: string;
+  private isGitRepo: boolean;
+
+  constructor(projectPath: string) {
+    this.projectPath = projectPath;
+    this.isGitRepo = existsSync(join(projectPath, '.git'));
+  }
+
+  async extractAll(): Promise<ExtractedDecision[]> {
+    const decisions: ExtractedDecision[] = [];
+
+    // Extract from git commits
+    if (this.isGitRepo) {
+      try {
+        const gitDecisions = await this.extractFromGitCommits();
+        decisions.push(...gitDecisions);
+      } catch (error) {
+        console.error('Error extracting from git:', error);
+      }
+    }
+
+    // Extract from code comments
+    try {
+      const commentDecisions = await this.extractFromComments();
+      decisions.push(...commentDecisions);
+    } catch (error) {
+      console.error('Error extracting from comments:', error);
+    }
+
+    // Extract from ADR files
+    try {
+      const adrDecisions = await this.extractFromADRFiles();
+      decisions.push(...adrDecisions);
+    } catch (error) {
+      console.error('Error extracting from ADR files:', error);
+    }
+
+    return decisions;
+  }
+
+  private async extractFromGitCommits(): Promise<ExtractedDecision[]> {
+    const decisions: ExtractedDecision[] = [];
+
+    try {
+      // Get recent commits with architectural significance
+      const output = execSync(
+        'git log --oneline -100 --format="%H|%s|%b"',
+        { cwd: this.projectPath, encoding: 'utf-8', maxBuffer: 1024 * 1024 }
+      );
+
+      const commits = output.split('\n').filter(Boolean);
+
+      for (const commit of commits) {
+        const [hash, subject, ...bodyParts] = commit.split('|');
+        const body = bodyParts.join('|');
+        const fullMessage = `${subject}\n${body}`.trim();
+
+        // Check for decision patterns
+        for (const pattern of COMMIT_PATTERNS) {
+          const match = fullMessage.match(pattern);
+          if (match) {
+            const title = this.extractTitle(subject || '');
+            const description = body || subject || '';
+
+            if (title && this.isArchitecturallySignificant(fullMessage)) {
+              decisions.push({
+                title,
+                description: description.slice(0, 500),
+                source: 'git',
+                tags: this.extractTags(fullMessage)
+              });
+              break;
+            }
+          }
+        }
+      }
+    } catch (error) {
+      // Git command failed, probably not a git repo or no commits
+    }
+
+    return decisions.slice(0, 20); // Limit to 20 most recent
+  }
+
+  private async extractFromComments(): Promise<ExtractedDecision[]> {
+    const decisions: ExtractedDecision[] = [];
+
+    // Find all code files
+    const patterns = ['**/*.ts', '**/*.js', '**/*.py', '**/*.go', '**/*.java', '**/*.rs'];
+    const ignorePatterns = ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'];
+
+    for (const pattern of patterns) {
+      try {
+        const files = await glob(pattern, {
+          cwd: this.projectPath,
+          ignore: ignorePatterns,
+          absolute: true,
+          nodir: true
+        });
+
+        for (const file of files.slice(0, 100)) { // Limit files to scan
+          try {
+            const content = readFileSync(file, 'utf-8');
+            const lines = content.split('\n');
+
+            for (let i = 0; i < lines.length; i++) {
+              const line = lines[i] || '';
+
+              for (const { pattern, tag } of COMMENT_PATTERNS) {
+                pattern.lastIndex = 0; // Reset regex state
+                const match = pattern.exec(line);
+                if (match && match[1]) {
+                  const relativePath = file.replace(this.projectPath, '').replace(/^[/\\]/, '');
+
+                  decisions.push({
+                    title: this.extractTitle(match[1]),
+                    description: match[1],
+                    source: 'comment',
+                    file: relativePath,
+                    line: i + 1,
+                    tags: [tag, ...this.extractTags(match[1])]
+                  });
+                }
+              }
+            }
+          } catch {
+            // Skip files that can't be read
+          }
+        }
+      } catch {
+        // Glob failed
+      }
+    }
+
+    return decisions;
+  }
+
+  private async extractFromADRFiles(): Promise<ExtractedDecision[]> {
+    const decisions: ExtractedDecision[] = [];
+
+    // Look for ADR files in common locations
+    const adrPatterns = [
+      '**/docs/decisions/*.md',
+      '**/docs/adr/*.md',
+      '**/adr/*.md',
+      '**/decisions/*.md'
+    ];
+
+    for (const pattern of adrPatterns) {
+      try {
+        const files = await glob(pattern, {
+          cwd: this.projectPath,
+          ignore: ['**/node_modules/**'],
+          absolute: true,
+          nodir: true
+        });
+
+        for (const file of files) {
+          try {
+            const content = readFileSync(file, 'utf-8');
+            const title = this.extractADRTitle(content);
+            const status = this.extractADRStatus(content);
+
+            if (title && status !== 'superseded' && status !== 'deprecated') {
+              const relativePath = file.replace(this.projectPath, '').replace(/^[/\\]/, '');
+
+              decisions.push({
+                title,
+                description: this.extractADRDescription(content),
+                source: 'adr',
+                file: relativePath,
+                tags: ['adr', ...this.extractTags(content)]
+              });
+            }
+          } catch {
+            // Skip files that can't be read
+          }
+        }
+      } catch {
+        // Glob failed
+      }
+    }
+
+    return decisions;
+  }
+
+  private extractTitle(text: string): string {
+    // Clean up the text to make a good title
+    let title = text
+      .replace(/^(?:feat|fix|refactor|perf|build|ci|docs|style|test|chore)\([^)]*\):\s*/i, '')
+      .replace(/^(?:DECISION|ARCHITECTURE|ADR|WHY|NOTE|IMPORTANT):\s*/i, '')
+      .trim();
+
+    // Capitalize first letter
+    if (title.length > 0) {
+      title = title.charAt(0).toUpperCase() + title.slice(1);
+    }
+
+    // Truncate if too long
+    if (title.length > 100) {
+      title = title.slice(0, 97) + '...';
+    }
+
+    return title;
+  }
+
+  private extractTags(text: string): string[] {
+    const tags: string[] = [];
+    const lowerText = text.toLowerCase();
+
+    for (const [tag, keywords] of Object.entries(TAG_KEYWORDS)) {
+      for (const keyword of keywords) {
+        if (lowerText.includes(keyword)) {
+          tags.push(tag);
+          break;
+        }
+      }
+    }
+
+    return [...new Set(tags)]; // Deduplicate
+  }
+
+  private isArchitecturallySignificant(text: string): boolean {
+    const significantKeywords = [
+      'architecture', 'design', 'pattern', 'implement', 'refactor',
+      'migrate', 'switch', 'replace', 'instead', 'because', 'why',
+      'decision', 'chose', 'choose', 'use', 'adopt', 'introduce'
+    ];
+
+    const lowerText = text.toLowerCase();
+    return significantKeywords.some(keyword => lowerText.includes(keyword));
+  }
+
+  private extractADRTitle(content: string): string {
+    // Look for # Title or title: in YAML frontmatter
+    const titleMatch = content.match(/^#\s+(.+)$/m) ||
+      content.match(/^title:\s*["']?([^"'\n]+)["']?$/m);
+
+    return titleMatch ? titleMatch[1]!.trim() : '';
+  }
+
+  private extractADRStatus(content: string): string {
+    const statusMatch = content.match(/^(?:##\s*)?status:\s*["']?(\w+)["']?$/im);
+    return statusMatch ? statusMatch[1]!.toLowerCase() : 'accepted';
+  }
+
+  private extractADRDescription(content: string): string {
+    // Try to find context or decision section
+    const contextMatch = content.match(/##\s*Context\s*\n([\s\S]*?)(?=##|$)/i);
+    const decisionMatch = content.match(/##\s*Decision\s*\n([\s\S]*?)(?=##|$)/i);
+
+    let description = '';
+    if (contextMatch) {
+      description += contextMatch[1]!.trim();
+    }
+    if (decisionMatch) {
+      description += (description ? '\n\n' : '') + decisionMatch[1]!.trim();
+    }
+
+    if (!description) {
+      // Fallback: use first paragraph after title
+      const lines = content.split('\n');
+      const startIndex = lines.findIndex(l => l.startsWith('#'));
+      if (startIndex >= 0) {
+        const remaining = lines.slice(startIndex + 1).join('\n').trim();
+        const firstPara = remaining.split(/\n\n/)[0];
+        description = firstPara || '';
+      }
+    }
+
+    return description.slice(0, 1000);
+  }
+
+  // Convert extracted decisions to proper Decision objects
+  toDecisions(extracted: ExtractedDecision[]): Decision[] {
+    return extracted.map(e => ({
+      id: randomUUID(),
+      title: e.title,
+      description: `${e.description}\n\n[Source: ${e.source}${e.file ? `, ${e.file}` : ''}${e.line ? `:${e.line}` : ''}]`,
+      files: e.file ? [e.file] : [],
+      tags: e.tags,
+      createdAt: new Date()
+    }));
+  }
+}

package/src/core/decisions.ts

@@ -0,0 +1,69 @@
+import { randomUUID } from 'crypto';
+import { Tier1Storage } from '../storage/tier1.js';
+import { Tier2Storage } from '../storage/tier2.js';
+import { EmbeddingGenerator } from '../indexing/embeddings.js';
+import type { Decision } from '../types/index.js';
+
+export class DecisionTracker {
+  private tier1: Tier1Storage;
+  private tier2: Tier2Storage;
+  private embeddingGenerator: EmbeddingGenerator;
+
+  constructor(
+    tier1: Tier1Storage,
+    tier2: Tier2Storage,
+    embeddingGenerator: EmbeddingGenerator
+  ) {
+    this.tier1 = tier1;
+    this.tier2 = tier2;
+    this.embeddingGenerator = embeddingGenerator;
+  }
+
+  async recordDecision(
+    title: string,
+    description: string,
+    files: string[] = [],
+    tags: string[] = []
+  ): Promise<Decision> {
+    const decision: Decision = {
+      id: randomUUID(),
+      title,
+      description,
+      files,
+      tags,
+      createdAt: new Date()
+    };
+
+    // Store in Tier 1 for immediate access
+    this.tier1.addDecision(decision);
+
+    // Generate embedding for semantic search
+    const textToEmbed = `${title}\n${description}\n${tags.join(' ')}`;
+    const embedding = await this.embeddingGenerator.embed(textToEmbed);
+
+    // Store in Tier 2 for persistence
+    this.tier2.upsertDecision(decision, embedding);
+
+    return decision;
+  }
+
+  getRecentDecisions(limit: number = 10): Decision[] {
+    // First try Tier 1 (faster)
+    const tier1Decisions = this.tier1.getRecentDecisions(limit);
+    if (tier1Decisions.length >= limit) {
+      return tier1Decisions;
+    }
+
+    // Fall back to Tier 2 for more
+    return this.tier2.getRecentDecisions(limit);
+  }
+
+  getDecision(id: string): Decision | null {
+    return this.tier2.getDecision(id);
+  }
+
+  async searchDecisions(query: string, limit: number = 5): Promise<Decision[]> {
+    const queryEmbedding = await this.embeddingGenerator.embed(query);
+    return this.tier2.searchDecisions(queryEmbedding, limit);
+  }
+}