minovative-mind-cli 2.14.1 → 2.14.3

package/dist/utils/contextRanker.js

@@ -0,0 +1,603 @@
+/**
+ * @file src/utils/contextRanker.ts
+ * @description Classical BM25 / TF-IDF information retrieval engine with dependency graph
+ * centrality scoring and deterministic 3-tier token budgeting.
+ *
+ * Partitions candidate workspace files into:
+ * - Tier 1: Full raw source code for high-relevance edit targets and primary files.
+ * - Tier 2: Compact AST declaration skeletons for peripheral hubs and dependent modules.
+ * - Tier 3: Omitted overflow files to strictly respect token budget boundaries.
+ */
+import path from 'node:path';
+import { extractDeclarationsOutline } from './symbolExtractor.js';
+import { CONTEXT_BUDGET_CONFIG } from './contextPrompts.js';
+// ─── Stop Words ──────────────────────────────────────────────────────
+const COMMON_STOP_WORDS = new Set([
+    'a',
+    'about',
+    'above',
+    'after',
+    'again',
+    'against',
+    'all',
+    'am',
+    'an',
+    'and',
+    'any',
+    'are',
+    'as',
+    'at',
+    'be',
+    'because',
+    'been',
+    'before',
+    'being',
+    'below',
+    'between',
+    'both',
+    'but',
+    'by',
+    'can',
+    'could',
+    'did',
+    'do',
+    'does',
+    'doing',
+    'down',
+    'during',
+    'each',
+    'few',
+    'for',
+    'from',
+    'further',
+    'had',
+    'has',
+    'have',
+    'having',
+    'he',
+    'her',
+    'here',
+    'hers',
+    'herself',
+    'him',
+    'himself',
+    'his',
+    'how',
+    'i',
+    'if',
+    'in',
+    'into',
+    'is',
+    'it',
+    'its',
+    'itself',
+    'just',
+    'me',
+    'more',
+    'most',
+    'my',
+    'myself',
+    'no',
+    'nor',
+    'not',
+    'now',
+    'of',
+    'off',
+    'on',
+    'once',
+    'only',
+    'or',
+    'other',
+    'our',
+    'ours',
+    'ourselves',
+    'out',
+    'over',
+    'own',
+    'same',
+    'should',
+    'so',
+    'some',
+    'such',
+    'than',
+    'that',
+    'the',
+    'their',
+    'theirs',
+    'them',
+    'themselves',
+    'then',
+    'there',
+    'these',
+    'they',
+    'this',
+    'those',
+    'through',
+    'to',
+    'too',
+    'under',
+    'until',
+    'up',
+    'very',
+    'was',
+    'we',
+    'were',
+    'what',
+    'when',
+    'where',
+    'which',
+    'while',
+    'who',
+    'whom',
+    'why',
+    'will',
+    'with',
+    'would',
+    'you',
+    'your',
+    'yours',
+    'yourself',
+    'yourselves',
+]);
+// ─── Tokenizer & Query Preprocessor ──────────────────────────────────
+/**
+ * Tokenizes code and text into normalized keywords.
+ * Handles camelCase, PascalCase, snake_case, kebab-case, and dot/slash delimiters.
+ *
+ * @param text The input string to tokenize
+ * @param filterStopWords Whether to filter common English stop words (default: true)
+ * @returns Array of lowercase alphanumeric tokens
+ */
+export function tokenize(text, filterStopWords = true) {
+    if (!text)
+        return [];
+    // Split on camelCase boundaries (e.g. "contextRanker" -> "context Ranker")
+    const splitCamel = text.replace(/([a-z0-9])([A-Z])/g, '$1 $2').replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2');
+    // Split by non-alphanumeric characters (punctuation, symbols, whitespace)
+    const rawTokens = splitCamel.toLowerCase().split(/[^a-z0-9_]+/);
+    const tokens = [];
+    for (const raw of rawTokens) {
+        // Also split any remaining snake_case tokens
+        const subTokens = raw.split('_');
+        for (const sub of subTokens) {
+            const trimmed = sub.trim();
+            if (trimmed.length < 2)
+                continue;
+            if (filterStopWords && COMMON_STOP_WORDS.has(trimmed))
+                continue;
+            tokens.push(trimmed);
+        }
+    }
+    return tokens;
+}
+/**
+ * Extracts unique search terms from a user prompt or query.
+ */
+export function extractQueryTerms(prompt) {
+    const tokens = tokenize(prompt, true);
+    return Array.from(new Set(tokens));
+}
+/**
+ * In-memory BM25 index and scoring implementation.
+ * Delivers deterministic <5ms search ranking across candidate files.
+ */
+export class BM25Engine {
+    avgDocLength = 0;
+    b;
+    docFrequency = new Map();
+    documents = [];
+    k1;
+    totalDocs = 0;
+    constructor(options) {
+        this.k1 = options?.k1 ?? 1.2;
+        this.b = options?.b ?? 0.75;
+    }
+    /**
+     * Adds a document to the BM25 index.
+     * Path tokens can be repeated to apply a structural path boost.
+     */
+    addDocument(id, content, filePath, pathBoostFactor = 3) {
+        const contentTokens = tokenize(content, true);
+        const pathTokens = tokenize(filePath, true);
+        // Apply path boost factor by repeating path tokens
+        const boostedTokens = [...contentTokens];
+        for (let i = 0; i < pathBoostFactor; i++) {
+            boostedTokens.push(...pathTokens);
+        }
+        const termFreqs = new Map();
+        const uniqueTermsInDoc = new Set();
+        for (const token of boostedTokens) {
+            termFreqs.set(token, (termFreqs.get(token) || 0) + 1);
+            uniqueTermsInDoc.add(token);
+        }
+        for (const term of uniqueTermsInDoc) {
+            this.docFrequency.set(term, (this.docFrequency.get(term) || 0) + 1);
+        }
+        this.documents.push({
+            id,
+            tokens: boostedTokens,
+            termFreqs,
+            length: boostedTokens.length,
+        });
+    }
+    /**
+     * Finalizes document statistics (average document length).
+     */
+    build() {
+        this.totalDocs = this.documents.length;
+        if (this.totalDocs === 0) {
+            this.avgDocLength = 0;
+            return;
+        }
+        const totalLength = this.documents.reduce((acc, doc) => acc + doc.length, 0);
+        this.avgDocLength = totalLength / this.totalDocs;
+    }
+    /**
+     * Scores all indexed documents against query terms using standard BM25 formula with Robertson-Spärck Jones IDF.
+     *
+     * @param queryTerms Array of query terms
+     * @returns Map of document ID to raw BM25 score
+     */
+    score(queryTerms) {
+        const scores = new Map();
+        if (this.totalDocs === 0 || queryTerms.length === 0) {
+            for (const doc of this.documents) {
+                scores.set(doc.id, 0);
+            }
+            return scores;
+        }
+        for (const doc of this.documents) {
+            let docScore = 0;
+            for (const term of queryTerms) {
+                const tf = doc.termFreqs.get(term) || 0;
+                if (tf === 0)
+                    continue;
+                const df = this.docFrequency.get(term) || 0;
+                // Smoothed BM25 IDF: ln((N - n + 0.5) / (n + 0.5) + 1)
+                const idf = Math.log((this.totalDocs - df + 0.5) / (df + 0.5) + 1);
+                // BM25 term frequency saturation
+                const numerator = tf * (this.k1 + 1);
+                const denominator = this.avgDocLength > 0
+                    ? tf + this.k1 * (1 - this.b + this.b * (doc.length / this.avgDocLength))
+                    : tf + this.k1;
+                docScore += idf * (numerator / denominator);
+            }
+            scores.set(doc.id, docScore);
+        }
+        return scores;
+    }
+}
+/**
+ * Computes BM25 scores for a collection of file candidates.
+ *
+ * @param candidates List of file candidates
+ * @param queryTerms Query terms from prompt
+ * @param options BM25 parameters
+ * @returns Map of filePath to normalized BM25 score in [0.0, 1.0]
+ */
+export function computeBM25Scores(candidates, queryTerms, options) {
+    const engine = new BM25Engine(options);
+    for (const candidate of candidates) {
+        engine.addDocument(candidate.filePath, candidate.content, candidate.filePath);
+    }
+    engine.build();
+    const rawScores = engine.score(queryTerms);
+    let maxScore = 0;
+    for (const score of rawScores.values()) {
+        if (score > maxScore)
+            maxScore = score;
+    }
+    const result = new Map();
+    for (const [filePath, raw] of rawScores.entries()) {
+        const normalized = maxScore > 0 ? raw / maxScore : 0;
+        result.set(filePath, { rawScore: raw, normalizedScore: normalized });
+    }
+    return result;
+}
+/**
+ * Computes TF-IDF scores for a collection of candidates as a lightweight alternative.
+ */
+export function computeTfIdfScores(candidates, queryTerms) {
+    const totalDocs = candidates.length;
+    const docTokensMap = new Map();
+    const docFreq = new Map();
+    for (const candidate of candidates) {
+        const tokens = tokenize(`${candidate.filePath} ${candidate.content}`, true);
+        docTokensMap.set(candidate.filePath, tokens);
+        const unique = new Set(tokens);
+        for (const term of unique) {
+            docFreq.set(term, (docFreq.get(term) || 0) + 1);
+        }
+    }
+    const rawScores = new Map();
+    let maxScore = 0;
+    for (const candidate of candidates) {
+        const tokens = docTokensMap.get(candidate.filePath) || [];
+        const docLen = tokens.length || 1;
+        const tfMap = new Map();
+        for (const t of tokens) {
+            tfMap.set(t, (tfMap.get(t) || 0) + 1);
+        }
+        let score = 0;
+        for (const query of queryTerms) {
+            const tf = (tfMap.get(query) || 0) / docLen;
+            const df = docFreq.get(query) || 0;
+            if (df > 0) {
+                const idf = Math.log(1 + totalDocs / df);
+                score += tf * idf;
+            }
+        }
+        rawScores.set(candidate.filePath, score);
+        if (score > maxScore)
+            maxScore = score;
+    }
+    const result = new Map();
+    for (const [filePath, raw] of rawScores.entries()) {
+        const normalized = maxScore > 0 ? raw / maxScore : 0;
+        result.set(filePath, { rawScore: raw, normalizedScore: normalized });
+    }
+    return result;
+}
+// ─── Path & Filename Match Scoring ───────────────────────────────────
+/**
+ * Calculates a relevance score for a file path based on matching query terms against
+ * the file's basename, directory segments, and file extension.
+ *
+ * @param filePath File path
+ * @param queryTerms Extracted query terms
+ * @returns Path match score between 0.0 and 1.0
+ */
+export function calculatePathMatchScore(filePath, queryTerms) {
+    if (!filePath || queryTerms.length === 0)
+        return 0;
+    const basename = path.basename(filePath);
+    const basenameNoExt = path.parse(filePath).name;
+    const baseTokens = new Set(tokenize(basenameNoExt, false));
+    const fullPathTokens = new Set(tokenize(filePath, false));
+    let matchedBaseTerms = 0;
+    let matchedPathTerms = 0;
+    for (const term of queryTerms) {
+        const termLower = term.toLowerCase();
+        // Exact basename match (highest boost)
+        if (basenameNoExt.toLowerCase() === termLower || basename.toLowerCase() === termLower) {
+            return 1.0;
+        }
+        if (baseTokens.has(termLower)) {
+            matchedBaseTerms++;
+        }
+        else if (fullPathTokens.has(termLower)) {
+            matchedPathTerms++;
+        }
+    }
+    const totalTerms = queryTerms.length;
+    const baseScore = (matchedBaseTerms / totalTerms) * 0.8;
+    const pathScore = (matchedPathTerms / totalTerms) * 0.4;
+    return Math.min(1.0, baseScore + pathScore);
+}
+// ─── Centrality Extraction & Normalization ───────────────────────────
+/**
+ * Extracts and normalizes centrality metrics for a list of candidate files.
+ *
+ * @param candidates List of candidate files
+ * @param graph Optional dependency graph
+ * @param customCentrality Optional pre-computed centrality map
+ * @returns Map of filePath to normalized centrality score in [0.0, 1.0]
+ */
+export function computeCandidateCentralityScores(candidates, graph, customCentrality) {
+    const result = new Map();
+    let maxRawScore = 0;
+    for (const candidate of candidates) {
+        let metrics = { inDegree: 0, outDegree: 0, totalDegree: 0, score: 0 };
+        if (customCentrality?.has(candidate.filePath)) {
+            metrics = customCentrality.get(candidate.filePath);
+        }
+        else if (graph) {
+            metrics = graph.getCentrality(candidate.filePath);
+        }
+        if (metrics.score > maxRawScore) {
+            maxRawScore = metrics.score;
+        }
+        result.set(candidate.filePath, { metrics, normalizedScore: 0 });
+    }
+    for (const [filePath, entry] of result.entries()) {
+        const normalized = maxRawScore > 0 ? entry.metrics.score / maxRawScore : 0;
+        result.set(filePath, { metrics: entry.metrics, normalizedScore: normalized });
+    }
+    return result;
+}
+// ─── Token Estimator ─────────────────────────────────────────────────
+/**
+ * Estimates token count from string length using a characters-per-token heuristic.
+ */
+export function estimateTokens(text, charsPerToken = CONTEXT_BUDGET_CONFIG.ESTIMATED_CHARS_PER_TOKEN) {
+    if (!text)
+        return 0;
+    return Math.ceil(text.length / charsPerToken);
+}
+// ─── Main Ranking & 3-Tier Budget Allocation Engine ───────────────────
+/**
+ * Core Context Ranker:
+ * Ranks candidate files via hybrid BM25 + Graph Centrality + Path Match scoring,
+ * then partitions them into Tier 1 (full source), Tier 2 (AST outline skeleton),
+ * and Tier 3 (omitted) within strict token budgets.
+ *
+ * @param files Array of candidate file objects or map of filePath -> content
+ * @param prompt User prompt or instruction
+ * @param options Ranking and budgeting options
+ * @returns Partitioned 3-Tier result with complete metadata and statistics
+ */
+export function rankFilesForBudget(files, prompt, options) {
+    // 1. Normalize input candidates
+    let candidates = [];
+    if (Array.isArray(files)) {
+        candidates = files;
+    }
+    else if (files instanceof Map) {
+        for (const [filePath, content] of files.entries()) {
+            candidates.push({ filePath, content });
+        }
+    }
+    else if (typeof files === 'object' && files !== null) {
+        for (const [filePath, content] of Object.entries(files)) {
+            candidates.push({ filePath, content });
+        }
+    }
+    const charsPerToken = options?.charsPerToken ?? CONTEXT_BUDGET_CONFIG.ESTIMATED_CHARS_PER_TOKEN;
+    // Determine budgets
+    const defaultMaxChars = CONTEXT_BUDGET_CONFIG.DEFAULT_CONTEXT_MAX_CHARS;
+    const charBudget = typeof options?.charBudget === 'number' && options.charBudget > 0
+        ? options.charBudget
+        : typeof options?.tokenBudget === 'number' && options.tokenBudget > 0
+            ? options.tokenBudget * charsPerToken
+            : defaultMaxChars;
+    const tokenBudget = Math.floor(charBudget / charsPerToken);
+    const tier1Ratio = options?.tier1BudgetRatio ?? 0.7;
+    const tier2Ratio = options?.tier2BudgetRatio ?? 0.3;
+    const tier1CharBudget = Math.floor(charBudget * tier1Ratio);
+    const tier2CharBudget = Math.floor(charBudget * tier2Ratio);
+    const weights = {
+        bm25: options?.weights?.bm25 ?? 0.6,
+        centrality: options?.weights?.centrality ?? 0.25,
+        pathMatch: options?.weights?.pathMatch ?? 0.15,
+    };
+    // Normalize weight sum
+    const weightSum = weights.bm25 + weights.centrality + weights.pathMatch;
+    const normWeights = {
+        bm25: weightSum > 0 ? weights.bm25 / weightSum : 0.6,
+        centrality: weightSum > 0 ? weights.centrality / weightSum : 0.25,
+        pathMatch: weightSum > 0 ? weights.pathMatch / weightSum : 0.15,
+    };
+    if (candidates.length === 0) {
+        return {
+            tier1: [],
+            tier2: [],
+            tier3: [],
+            rankedFiles: [],
+            totalTokensUsed: 0,
+            totalCharsUsed: 0,
+            stats: {
+                totalCandidates: 0,
+                tier1Count: 0,
+                tier2Count: 0,
+                tier3Count: 0,
+                tokenBudget,
+                tokensUsed: 0,
+                charBudget,
+                charsUsed: 0,
+                budgetUtilization: 0,
+            },
+        };
+    }
+    // 2. Extract query terms from user prompt
+    const queryTerms = extractQueryTerms(prompt);
+    // 3. Compute BM25 scores
+    const bm25Map = computeBM25Scores(candidates, queryTerms, {
+        k1: options?.bm25K1,
+        b: options?.bm25B,
+    });
+    // 4. Compute Centrality scores
+    const centralityMap = computeCandidateCentralityScores(candidates, options?.graph, options?.centralityMetrics);
+    // 5. Precompute AST outlines & composite scores
+    const scoredFiles = candidates.map((candidate) => {
+        const bm25Info = bm25Map.get(candidate.filePath) || { rawScore: 0, normalizedScore: 0 };
+        const centralityInfo = centralityMap.get(candidate.filePath) || {
+            metrics: { inDegree: 0, outDegree: 0, totalDegree: 0, score: 0 },
+            normalizedScore: 0,
+        };
+        const pathMatchScore = calculatePathMatchScore(candidate.filePath, queryTerms);
+        // Compute AST outline skeleton
+        const outline = candidate.outline || extractDeclarationsOutline(candidate.content, candidate.filePath) || '';
+        let compositeScore = normWeights.bm25 * bm25Info.normalizedScore +
+            normWeights.centrality * centralityInfo.normalizedScore +
+            normWeights.pathMatch * pathMatchScore;
+        // Explicit targets receive top priority boost
+        if (candidate.isExplicitTarget) {
+            compositeScore += 10.0;
+        }
+        return {
+            filePath: candidate.filePath,
+            content: candidate.content,
+            outline,
+            tier: 3, // Default to tier 3, will be assigned during budget partitioning
+            score: compositeScore,
+            bm25Score: bm25Info.normalizedScore,
+            centralityScore: centralityInfo.normalizedScore,
+            pathMatchScore,
+            estimatedTokens: estimateTokens(candidate.content, charsPerToken),
+            estimatedChars: candidate.content.length,
+            isExplicitTarget: candidate.isExplicitTarget,
+            metadata: candidate.metadata,
+        };
+    });
+    // 6. Sort candidate files descending by composite relevance score
+    scoredFiles.sort((a, b) => b.score - a.score);
+    // 7. Partition into Tier 1 (Full Content), Tier 2 (AST Skeleton), and Tier 3 (Omitted)
+    const tier1 = [];
+    const tier2 = [];
+    const tier3 = [];
+    let currentTotalChars = 0;
+    let currentTier1Chars = 0;
+    let currentTier2Chars = 0;
+    for (const file of scoredFiles) {
+        const fullContentChars = file.content.length;
+        const outlineChars = file.outline.length || Math.min(fullContentChars, 200);
+        // Check if score is below minimal cutoff threshold (unless explicit target)
+        if (!file.isExplicitTarget && options?.minScoreThreshold && file.score < options.minScoreThreshold) {
+            file.tier = 3;
+            file.estimatedChars = 0;
+            file.estimatedTokens = 0;
+            tier3.push(file);
+            continue;
+        }
+        // Attempt Tier 1: Full content
+        // Criteria: Explicit targets or fits within remaining Tier 1 budget and overall budget
+        const fitsTier1 = file.isExplicitTarget ||
+            (currentTier1Chars + fullContentChars <= tier1CharBudget && currentTotalChars + fullContentChars <= charBudget);
+        if (fitsTier1 && currentTotalChars + fullContentChars <= charBudget) {
+            file.tier = 1;
+            file.estimatedChars = fullContentChars;
+            file.estimatedTokens = estimateTokens(file.content, charsPerToken);
+            tier1.push(file);
+            currentTier1Chars += fullContentChars;
+            currentTotalChars += fullContentChars;
+            continue;
+        }
+        // Attempt Tier 2: AST Outline Skeleton
+        // Criteria: Fits within Tier 2 budget and overall budget
+        const fitsTier2 = currentTier2Chars + outlineChars <= tier2CharBudget && currentTotalChars + outlineChars <= charBudget;
+        if (fitsTier2) {
+            file.tier = 2;
+            file.estimatedChars = outlineChars;
+            file.estimatedTokens = estimateTokens(file.outline, charsPerToken);
+            tier2.push(file);
+            currentTier2Chars += outlineChars;
+            currentTotalChars += outlineChars;
+            continue;
+        }
+        // If it doesn't fit in Tier 1 or Tier 2, it is assigned to Tier 3 (Omitted)
+        file.tier = 3;
+        file.estimatedChars = 0;
+        file.estimatedTokens = 0;
+        tier3.push(file);
+    }
+    const totalTokensUsed = Math.ceil(currentTotalChars / charsPerToken);
+    return {
+        tier1,
+        tier2,
+        tier3,
+        rankedFiles: scoredFiles,
+        totalTokensUsed,
+        totalCharsUsed: currentTotalChars,
+        stats: {
+            totalCandidates: scoredFiles.length,
+            tier1Count: tier1.length,
+            tier2Count: tier2.length,
+            tier3Count: tier3.length,
+            tokenBudget,
+            tokensUsed: totalTokensUsed,
+            charBudget,
+            charsUsed: currentTotalChars,
+            budgetUtilization: charBudget > 0 ? Math.min(1.0, currentTotalChars / charBudget) : 0,
+        },
+    };
+}

package/dist/utils/dependencyTracer/modules/graph.d.ts

@@ -1,9 +1,12 @@
-import { DependencyNode } from './types.js';
+import { DependencyNode, CentralityMetrics } from './types.js';
 export interface DependencyGraph {
     getImports(filePath: string): string[];
     getImportedBy(filePath: string): string[];
     getReverseDependencyTree(filePath: string, maxDepth?: number): string[];
     getForwardDependencyTree(filePath: string, maxDepth?: number): string[];
+    getCentrality(filePath: string): CentralityMetrics;
+    getAllCentrality(): Map<string, CentralityMetrics>;
     readonly nodes: ReadonlyMap<string, DependencyNode>;
 }
+export declare function computeGraphCentrality(nodes: ReadonlyMap<string, DependencyNode> | Map<string, DependencyNode>): Map<string, CentralityMetrics>;
 export declare function bfsTraverse(nodes: Map<string, DependencyNode>, startFile: string, direction: 'imports' | 'importedBy', maxDepth: number): string[];

package/dist/utils/dependencyTracer/modules/graph.js

@@ -1,3 +1,14 @@
+export function computeGraphCentrality(nodes) {
+    const result = new Map();
+    for (const [file, node] of nodes.entries()) {
+        const inDegree = node.importedBy.size;
+        const outDegree = node.imports.size;
+        const totalDegree = inDegree + outDegree;
+        const score = inDegree * 1.5 + outDegree * 0.5;
+        result.set(file, { inDegree, outDegree, totalDegree, score });
+    }
+    return result;
+}
 export function bfsTraverse(nodes, startFile, direction, maxDepth) {
     const visited = new Set();
     visited.add(startFile);

package/dist/utils/dependencyTracer/modules/types.d.ts

@@ -2,3 +2,13 @@ export interface DependencyNode {
     imports: Set<string>;
     importedBy: Set<string>;
 }
+export interface CentralityMetrics {
+    /** Number of files that directly import this file (hub dependency) */
+    inDegree: number;
+    /** Number of files this file directly imports */
+    outDegree: number;
+    /** Total direct connections (inDegree + outDegree) */
+    totalDegree: number;
+    /** Centrality score (weighted: inDegree * 1.5 + outDegree * 0.5) */
+    score: number;
+}