@ddse/acm-aicoder 0.5.0

package/src/context/bm25.ts

@@ -0,0 +1,173 @@
+// Simple BM25 search implementation for ACM examples
+
+/**
+ * Document interface for BM25 search
+ */
+export interface Document {
+  id: string;
+  title?: string;
+  content: string;
+  [key: string]: any;
+}
+
+/**
+ * Search result with score
+ */
+export interface SearchResult {
+  document: Document;
+  score: number;
+}
+
+/**
+ * BM25 parameters
+ */
+export interface BM25Params {
+  k1?: number; // Term frequency saturation (default: 1.5)
+  b?: number;  // Length normalization (default: 0.75)
+}
+
+/**
+ * Simple tokenizer
+ */
+function tokenize(text?: string | null): string[] {
+  if (!text) return [];
+
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s]/g, ' ')
+    .split(/\s+/)
+    .filter((token) => token.length > 0);
+}
+
+/**
+ * BM25 Search Engine
+ * 
+ * Implements the BM25 ranking function for full-text search.
+ * BM25 is a probabilistic ranking function used by search engines
+ * to estimate the relevance of documents to a query.
+ */
+export class BM25Search {
+  private documents: Document[] = [];
+  private documentTokens: Map<string, string[]> = new Map();
+  private documentFrequency: Map<string, number> = new Map();
+  private averageDocumentLength: number = 0;
+  private k1: number;
+  private b: number;
+
+  constructor(params: BM25Params = {}) {
+    this.k1 = params.k1 ?? 1.5;
+    this.b = params.b ?? 0.75;
+  }
+
+  /**
+   * Index documents for search
+   */
+  index(documents: Document[]): void {
+    this.documents = documents;
+    this.documentTokens.clear();
+    this.documentFrequency.clear();
+
+    // Tokenize documents
+    let totalLength = 0;
+    for (const doc of documents) {
+      const text = this.extractText(doc);
+      const tokens = tokenize(text);
+      this.documentTokens.set(doc.id, tokens);
+      totalLength += tokens.length;
+
+      // Update document frequency
+      const uniqueTokens = new Set(tokens);
+      for (const token of uniqueTokens) {
+        this.documentFrequency.set(token, (this.documentFrequency.get(token) || 0) + 1);
+      }
+    }
+
+    this.averageDocumentLength = totalLength / documents.length;
+  }
+
+  /**
+   * Extract searchable text from document
+   */
+  private extractText(doc: Document): string {
+    const parts: string[] = [];
+    if (doc.title) parts.push(doc.title);
+    if (doc.content) parts.push(doc.content);
+    return parts.join(' ');
+  }
+
+  /**
+   * Calculate IDF (Inverse Document Frequency)
+   */
+  private idf(term: string): number {
+    const df = this.documentFrequency.get(term) || 0;
+    if (df === 0) return 0;
+
+    const n = this.documents.length;
+    return Math.log((n - df + 0.5) / (df + 0.5) + 1);
+  }
+
+  /**
+   * Calculate BM25 score for a document
+   */
+  private score(docId: string, queryTokens: string[]): number {
+    const tokens = this.documentTokens.get(docId);
+    if (!tokens) return 0;
+
+    const docLength = tokens.length;
+    let score = 0;
+
+    for (const queryToken of queryTokens) {
+      const termFreq = tokens.filter((t) => t === queryToken).length;
+      if (termFreq === 0) continue;
+
+      const idf = this.idf(queryToken);
+      const numerator = termFreq * (this.k1 + 1);
+      const denominator = termFreq + this.k1 * (1 - this.b + this.b * (docLength / this.averageDocumentLength));
+
+      score += idf * (numerator / denominator);
+    }
+
+    return score;
+  }
+
+  /**
+   * Search for documents matching the query
+   */
+  search(query: string, limit: number = 10): SearchResult[] {
+    if (this.documents.length === 0) {
+      return [];
+    }
+
+    const queryTokens = tokenize(query);
+    if (queryTokens.length === 0) {
+      return [];
+    }
+
+    // Score all documents
+    const results: SearchResult[] = [];
+    for (const doc of this.documents) {
+      const score = this.score(doc.id, queryTokens);
+      if (score > 0) {
+        results.push({ document: doc, score });
+      }
+    }
+
+    // Sort by score (descending) and limit
+    results.sort((a, b) => b.score - a.score);
+    return results.slice(0, limit);
+  }
+
+  /**
+   * Get all indexed documents
+   */
+  getDocuments(): Document[] {
+    return this.documents;
+  }
+
+  /**
+   * Get document count
+   */
+  getDocumentCount(): number {
+    return this.documents.length;
+  }
+}

package/src/context/code-search.ts

@@ -0,0 +1,188 @@
+// Code Search - BM25 based search for code files
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { BM25Search, type Document } from './bm25.js';
+import type { WorkspaceIndex, FileMetadata, SearchResult } from './types.js';
+
+export interface SearchOptions {
+  k?: number; // Number of results
+  preferTypes?: string[]; // Prefer certain file types
+  includeContext?: boolean; // Include surrounding lines
+  contextLines?: number; // Number of context lines
+}
+
+export class CodeSearch {
+  private bm25: BM25Search;
+  private rootPath: string;
+  private documents: Map<string, Document> = new Map();
+
+  constructor(rootPath: string) {
+    this.rootPath = rootPath;
+    this.bm25 = new BM25Search({ k1: 1.5, b: 0.75 });
+  }
+
+  /**
+   * Index files for search
+   */
+  async indexFiles(index: WorkspaceIndex): Promise<void> {
+    const documents: Document[] = [];
+    this.documents.clear();
+
+    // Filter to text files only
+    const textFiles = index.files.filter(f => 
+      !f.isBinary && 
+      f.size < 500_000 && // Only index files < 500KB
+      (f.language === 'typescript' || 
+       f.language === 'javascript' || 
+       f.language === 'markdown' ||
+       f.language === 'json')
+    );
+
+    // Read and index each file
+    for (const file of textFiles) {
+      try {
+        const fullPath = path.join(this.rootPath, file.path);
+        const content = await fs.readFile(fullPath, 'utf-8');
+        
+        const doc: Document = {
+          id: file.path,
+          title: path.basename(file.path),
+          content,
+          language: file.language,
+          size: file.size,
+          path: file.path,
+        };
+
+        documents.push(doc);
+        this.documents.set(file.path, doc);
+      } catch {
+        // Skip files we can't read
+      }
+    }
+
+    // Build BM25 index
+    this.bm25.index(documents);
+  }
+
+  /**
+   * Search for code
+   */
+  async search(query: string, options: SearchOptions = {}): Promise<SearchResult[]> {
+    const k = options.k ?? 10;
+    const preferTypes = new Set(options.preferTypes || []);
+
+    // Search using BM25
+    const bm25Results = this.bm25.search(query, k * 2); // Get more initially
+
+    // Apply preferences and convert to SearchResult
+    const results: SearchResult[] = [];
+
+    for (const result of bm25Results) {
+      const doc = result.document;
+      let score = result.score;
+
+      // Boost score for preferred file types
+      if (preferTypes.size > 0) {
+        const ext = path.extname(doc.path);
+        const lang = doc.language as string;
+        if (preferTypes.has(ext) || preferTypes.has(lang)) {
+          score *= 1.5;
+        }
+      }
+
+      // Find best matching line
+      const snippet = this.findBestSnippet(doc.content, query, options);
+
+      results.push({
+        path: doc.path,
+        score,
+        snippet: snippet.text,
+        line: snippet.line,
+        column: snippet.column,
+      });
+    }
+
+    // Re-sort by adjusted scores and limit
+    return results
+      .sort((a, b) => b.score - a.score)
+      .slice(0, k);
+  }
+
+  /**
+   * Find the best snippet matching the query
+   */
+  private findBestSnippet(
+    content: string, 
+    query: string,
+    options: SearchOptions
+  ): { text: string; line: number; column: number } {
+    const lines = content.split('\n');
+    const queryLower = query.toLowerCase();
+    const queryTokens = queryLower.split(/\s+/);
+
+    let bestLine = 0;
+    let bestScore = 0;
+
+    // Find line with most query tokens
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      const lineLower = line.toLowerCase();
+      
+      let score = 0;
+      for (const token of queryTokens) {
+        if (lineLower.includes(token)) {
+          score += 1;
+        }
+      }
+
+      if (score > bestScore) {
+        bestScore = score;
+        bestLine = i;
+      }
+    }
+
+    // Extract snippet with context
+    const contextLines = options.includeContext ? (options.contextLines ?? 2) : 0;
+    const startLine = Math.max(0, bestLine - contextLines);
+    const endLine = Math.min(lines.length - 1, bestLine + contextLines);
+
+    const snippetLines = lines.slice(startLine, endLine + 1);
+    const snippet = snippetLines
+      .map((line, idx) => {
+        const lineNum = startLine + idx + 1;
+        const marker = lineNum === bestLine + 1 ? '>' : ' ';
+        return `${marker} ${lineNum.toString().padStart(4, ' ')} ${line}`;
+      })
+      .join('\n');
+
+    // Find column position (first occurrence of any query token)
+    let column = 0;
+    const bestLineLower = lines[bestLine].toLowerCase();
+    for (const token of queryTokens) {
+      const pos = bestLineLower.indexOf(token);
+      if (pos >= 0) {
+        column = pos;
+        break;
+      }
+    }
+
+    return {
+      text: snippet,
+      line: bestLine + 1,
+      column,
+    };
+  }
+
+  /**
+   * Search for symbol by name (exact or partial match)
+   */
+  async searchSymbol(symbolName: string): Promise<SearchResult[]> {
+    // Search for exact symbol name with high weight
+    return this.search(`${symbolName} function class interface`, {
+      k: 5,
+      preferTypes: ['.ts', '.tsx', '.js', '.jsx'],
+      includeContext: true,
+      contextLines: 3,
+    });
+  }
+}

package/src/context/context-pack.ts

@@ -0,0 +1,133 @@
+// Context Pack Generator - Create rich context for LLM planning
+import type { 
+  ContextPack, 
+  WorkspaceIndex, 
+  SymbolInfo, 
+  DependencyInfo, 
+  TestMapping 
+} from './types.js';
+import { CodeSearch } from './code-search.js';
+
+export interface ContextPackOptions {
+  maxFiles?: number;
+  maxSymbols?: number;
+  includeTests?: boolean;
+  includeDependencies?: boolean;
+}
+
+export class ContextPackGenerator {
+  private search: CodeSearch;
+
+  constructor(search: CodeSearch) {
+    this.search = search;
+  }
+
+  /**
+   * Generate context pack for a goal
+   */
+  async generate(
+    goal: string,
+    index: WorkspaceIndex,
+    symbols: SymbolInfo[],
+    dependencies: DependencyInfo[],
+    testMappings: TestMapping[],
+    options: ContextPackOptions = {}
+  ): Promise<ContextPack> {
+    const maxFiles = options.maxFiles ?? 8;
+    const maxSymbols = options.maxSymbols ?? 20;
+    const normalizedGoal = (goal || '').toString();
+    const searchQuery = normalizedGoal.trim().length > 0
+      ? normalizedGoal
+      : 'workspace overview';
+
+    // Search for relevant files
+    const searchResults = await this.search.search(searchQuery, {
+      k: maxFiles,
+      includeContext: true,
+      contextLines: 2,
+    });
+
+    // Extract relevant symbols (top matching symbols)
+    const relevantSymbols = this.findRelevantSymbols(normalizedGoal, symbols, maxSymbols);
+
+    // Filter dependencies if requested
+    const relevantDeps = options.includeDependencies 
+      ? dependencies.slice(0, 20) // Top 20 deps
+      : [];
+
+    // Extract test files if requested
+    const testFiles = options.includeTests
+      ? testMappings.map(m => m.testFile)
+      : [];
+
+    // Build context pack
+    const pack: ContextPack = {
+      goal: normalizedGoal,
+      files: searchResults.map(r => ({
+        path: r.path,
+        snippet: r.snippet,
+        relevance: r.score,
+      })),
+      symbols: relevantSymbols,
+      dependencies: relevantDeps,
+      testFiles,
+      summary: this.generateSummary(index, searchResults.length, relevantSymbols.length),
+      generatedAt: new Date().toISOString(),
+    };
+
+    return pack;
+  }
+
+  /**
+   * Find symbols relevant to the goal
+   */
+  private findRelevantSymbols(goal: string, symbols: SymbolInfo[], limit: number): SymbolInfo[] {
+    const goalLower = (goal || '').toLowerCase();
+    const tokens = goalLower.split(/\s+/).filter(Boolean);
+
+    if (tokens.length === 0) {
+      return [];
+    }
+
+    // Score symbols by relevance
+    const scored = symbols.map(symbol => {
+      let score = 0;
+      const nameLower = symbol.name.toLowerCase();
+
+      // Exact match
+      if (tokens.some(t => nameLower === t)) {
+        score += 10;
+      }
+
+      // Partial match
+      if (tokens.some(t => nameLower.includes(t) || t.includes(nameLower))) {
+        score += 5;
+      }
+
+      // Boost exported symbols
+      if (symbol.kind === 'function' || symbol.kind === 'class') {
+        score += 2;
+      }
+
+      return { symbol, score };
+    });
+
+    // Sort by score and return top symbols
+    return scored
+      .filter(s => s.score > 0)
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit)
+      .map(s => s.symbol);
+  }
+
+  /**
+   * Generate a summary of the context
+   */
+  private generateSummary(index: WorkspaceIndex, fileCount: number, symbolCount: number): string {
+    const totalFiles = index.totalFiles;
+    const totalSize = (index.totalSize / 1024 / 1024).toFixed(2);
+
+    return `Workspace: ${totalFiles} files, ${totalSize}MB. ` +
+           `Context: ${fileCount} relevant files, ${symbolCount} symbols.`;
+  }
+}

package/src/context/dependency-mapper.ts

@@ -0,0 +1,72 @@
+// Dependency Mapper - Parse and analyze dependencies
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import type { DependencyInfo, WorkspaceIndex } from './types.js';
+
+export class DependencyMapper {
+  private rootPath: string;
+
+  constructor(rootPath: string) {
+    this.rootPath = rootPath;
+  }
+
+  /**
+   * Extract dependencies from package.json files
+   */
+  async extractDependencies(index: WorkspaceIndex): Promise<DependencyInfo[]> {
+    const dependencies: DependencyInfo[] = [];
+
+    // Find all package.json files
+    const packageFiles = index.files.filter(f => 
+      path.basename(f.path) === 'package.json'
+    );
+
+    for (const file of packageFiles) {
+      try {
+        const fullPath = path.join(this.rootPath, file.path);
+        const content = await fs.readFile(fullPath, 'utf-8');
+        const pkg = JSON.parse(content);
+
+        // Extract dependencies
+        if (pkg.dependencies) {
+          for (const [name, version] of Object.entries(pkg.dependencies)) {
+            dependencies.push({
+              name,
+              version: version as string,
+              type: 'dependency',
+              packageJsonPath: file.path,
+            });
+          }
+        }
+
+        // Extract devDependencies
+        if (pkg.devDependencies) {
+          for (const [name, version] of Object.entries(pkg.devDependencies)) {
+            dependencies.push({
+              name,
+              version: version as string,
+              type: 'devDependency',
+              packageJsonPath: file.path,
+            });
+          }
+        }
+
+        // Extract peerDependencies
+        if (pkg.peerDependencies) {
+          for (const [name, version] of Object.entries(pkg.peerDependencies)) {
+            dependencies.push({
+              name,
+              version: version as string,
+              type: 'peerDependency',
+              packageJsonPath: file.path,
+            });
+          }
+        }
+      } catch {
+        // Skip invalid package.json files
+      }
+    }
+
+    return dependencies;
+  }
+}

package/src/context/index.ts

@@ -0,0 +1,8 @@
+// Context Engine - Intelligent repository understanding
+export * from './workspace-indexer.js';
+export * from './symbol-extractor.js';
+export * from './dependency-mapper.js';
+export * from './test-mapper.js';
+export * from './code-search.js';
+export * from './context-pack.js';
+export * from './types.js';