minovative-mind-cli 2.14.1 → 2.14.3

package/dist/utils/contextPrompts.js

@@ -5,6 +5,7 @@
  */
 import { changeLogger } from '../services/changeLogger.js';
 import { extractDeclarationsOutline } from './symbolExtractor.js';
+import { rankFilesForBudget } from './contextRanker.js';
 /**
  * Standard character and token budgeting constants for context injections.
  */
@@ -15,6 +16,10 @@ export const CONTEXT_BUDGET_CONFIG = {
     DEFAULT_SCOPED_MAX_CHARS: 120_000,
     /** Estimated characters per token heuristic */
     ESTIMATED_CHARS_PER_TOKEN: 4,
+    /** Maximum fraction of budget allocated to Tier 1 full files in tiered context (70%) */
+    TIER1_BUDGET_RATIO: 0.7,
+    /** Maximum fraction of budget allocated to Tier 2 AST skeletons in tiered context (30%) */
+    TIER2_BUDGET_RATIO: 0.3,
 };
 /**
  * Estimates token count from a given text string.
@@ -88,7 +93,7 @@ ${context.webSearchSummary}
  * @param scoped Whether this content represents an AST-scoped outline.
  * @returns Formatted workspace_file block.
  */
-function formatWorkspaceFileBlock(filePath, contentText, scoped = false) {
+export function formatWorkspaceFileBlock(filePath, contentText, scoped = false) {
     let aliasAttr = '';
     if (filePath.startsWith('@')) {
         const slashIndex = filePath.indexOf('/');
@@ -140,12 +145,17 @@ function assembleContextWithFiles(baseContext, relevantFiles, maxChars, useScope
     const safetyAllowance = 1; // for newline before closing tag
     for (let i = 0; i < fileEntries.length; i++) {
         const [filePath, contentObj] = fileEntries[i];
+        // Check if explicitly omitted via Tier 3
+        if (contentObj.tier === 3) {
+            continue;
+        }
+        let isScoped = useScopedOutline || Boolean(contentObj.scoped) || contentObj.tier === 2;
         let contentText = contentObj.text;
-        if (useScopedOutline) {
-            contentText = extractDeclarationsOutline(contentText, filePath) || contentText;
+        if (isScoped) {
+            contentText = contentObj.outline || extractDeclarationsOutline(contentText, filePath) || contentText;
         }
         if (!hasBudget) {
-            injection += formatWorkspaceFileBlock(filePath, contentText, useScopedOutline);
+            injection += formatWorkspaceFileBlock(filePath, contentText, isScoped);
             continue;
         }
         const remainingFilesCount = fileEntries.length - i - 1;
@@ -153,14 +163,14 @@ function assembleContextWithFiles(baseContext, relevantFiles, maxChars, useScope
             ? `<!-- ${remainingFilesCount} additional relevant file(s) omitted to stay within token budget -->\n`.length
             : 0;
         const remainingBudgetForThisFile = maxChars - injection.length - closingTag.length - omissionNoticeEstimate - safetyAllowance;
-        const fullBlock = formatWorkspaceFileBlock(filePath, contentText, useScopedOutline);
+        const fullBlock = formatWorkspaceFileBlock(filePath, contentText, isScoped);
         if (fullBlock.length <= remainingBudgetForThisFile) {
             injection += fullBlock;
             continue;
         }
         // Token-efficiency optimization: If full file does not fit and wasn't scoped, attempt AST outline first
-        if (!useScopedOutline) {
-            const outlinedText = extractDeclarationsOutline(contentText, filePath);
+        if (!isScoped) {
+            const outlinedText = contentObj.outline || extractDeclarationsOutline(contentText, filePath);
             if (outlinedText) {
                 const outlinedBlock = formatWorkspaceFileBlock(filePath, outlinedText, true);
                 if (outlinedBlock.length <= remainingBudgetForThisFile) {
@@ -170,13 +180,13 @@ function assembleContextWithFiles(baseContext, relevantFiles, maxChars, useScope
             }
         }
         // Try partial/truncated block
-        const emptyBlock = formatWorkspaceFileBlock(filePath, '', useScopedOutline);
+        const emptyBlock = formatWorkspaceFileBlock(filePath, '', isScoped);
         const wrapperOverhead = emptyBlock.length;
         const availableContentChars = remainingBudgetForThisFile - wrapperOverhead - truncationNotice.length;
         let includedTruncated = false;
         if (availableContentChars > 50) {
             const truncatedContent = contentText.slice(0, availableContentChars) + truncationNotice;
-            injection += formatWorkspaceFileBlock(filePath, truncatedContent, useScopedOutline);
+            injection += formatWorkspaceFileBlock(filePath, truncatedContent, isScoped);
             includedTruncated = true;
         }
         const remainingFiles = fileEntries.length - (includedTruncated ? i + 1 : i);
@@ -227,6 +237,68 @@ export function buildScopedContextInjection(context, maxChars) {
     const baseContext = buildBaseContext(context);
     return assembleContextWithFiles(baseContext, context.relevantFiles, maxChars, true);
 }
+/**
+ * Constructs a tiered, asymmetric XML/Markdown-like project context string.
+ *
+ * Automatically partitions files into:
+ * - Tier 1: Primary edit targets & high-relevance files with 100% full raw source code.
+ * - Tier 2: Peripheral hubs & dependent modules with compact AST declaration skeletons (scoped="outline").
+ * - Tier 3: Budget overflow files (omitted with token budget omission notice).
+ *
+ * @param context The collected context data from ContextAgent.
+ * @param options Tiered context options or maximum character budget.
+ * @returns A formatted string containing project profile, structure, investigation summaries, and tiered file blocks.
+ */
+export function buildTieredContextInjection(context, options) {
+    const maxChars = typeof options === 'number'
+        ? options
+        : typeof options?.maxChars === 'number'
+            ? options.maxChars
+            : CONTEXT_BUDGET_CONFIG.DEFAULT_CONTEXT_MAX_CHARS;
+    const baseContext = buildBaseContext(context);
+    // If pre-computed tierPartition exists on context or options, use it directly
+    const tierPartition = typeof options === 'object' && options?.tierPartition ? options.tierPartition : context.tierPartition;
+    if (tierPartition) {
+        const fileMap = new Map();
+        for (const f of tierPartition.tier1 || []) {
+            fileMap.set(f.filePath, { text: f.content, tier: 1, scoped: false });
+        }
+        for (const f of tierPartition.tier2 || []) {
+            fileMap.set(f.filePath, { text: f.outline || f.content, outline: f.outline, tier: 2, scoped: true });
+        }
+        return assembleContextWithFiles(baseContext, fileMap, maxChars, false);
+    }
+    // If a prompt is provided in options, dynamically rank and partition using contextRanker
+    const prompt = typeof options === 'object' ? options?.prompt : undefined;
+    if (prompt && context.relevantFiles && context.relevantFiles.size > 0) {
+        try {
+            const candidates = Array.from(context.relevantFiles.entries()).map(([filePath, contentObj]) => ({
+                filePath,
+                content: contentObj.text,
+                isExplicitTarget: Boolean(contentObj.tier === 1),
+                outline: contentObj.outline,
+            }));
+            const partition = rankFilesForBudget(candidates, prompt, {
+                charBudget: maxChars,
+                graph: typeof options === 'object' ? options?.graph : undefined,
+            });
+            const fileMap = new Map();
+            for (const f of partition.tier1) {
+                const orig = context.relevantFiles.get(f.filePath);
+                fileMap.set(f.filePath, { text: f.content, inlineData: orig?.inlineData, tier: 1, scoped: false });
+            }
+            for (const f of partition.tier2) {
+                const orig = context.relevantFiles.get(f.filePath);
+                fileMap.set(f.filePath, { text: f.outline || f.content, outline: f.outline, inlineData: orig?.inlineData, tier: 2, scoped: true });
+            }
+            return assembleContextWithFiles(baseContext, fileMap, maxChars, false);
+        }
+        catch {
+            // Fallback to standard assembly if ranking fails
+        }
+    }
+    return assembleContextWithFiles(baseContext, context.relevantFiles, maxChars, false);
+}
 /**
  * Formats an array of resolved context mention blocks into a single structured
  * XML `<context_mentions>` block adhering to character/token budget constraints.

package/dist/utils/contextRanker.d.ts

@@ -0,0 +1,216 @@
+/**
+ * @file src/utils/contextRanker.ts
+ * @description Classical BM25 / TF-IDF information retrieval engine with dependency graph
+ * centrality scoring and deterministic 3-tier token budgeting.
+ *
+ * Partitions candidate workspace files into:
+ * - Tier 1: Full raw source code for high-relevance edit targets and primary files.
+ * - Tier 2: Compact AST declaration skeletons for peripheral hubs and dependent modules.
+ * - Tier 3: Omitted overflow files to strictly respect token budget boundaries.
+ */
+import { type DependencyGraph, type CentralityMetrics } from './dependencyTracer.js';
+export type ContextTier = 1 | 2 | 3;
+export interface FileCandidate {
+    /** Relative workspace file path or alias (e.g. "src/services/auth.ts") */
+    filePath: string;
+    /** Raw full text content of the file */
+    content: string;
+    /** If true, this file is an explicitly targeted or user-mentioned file */
+    isExplicitTarget?: boolean;
+    /** Optional pre-computed AST declarations outline skeleton */
+    outline?: string;
+    /** Optional metadata associated with the file candidate */
+    metadata?: Record<string, unknown>;
+}
+export interface RankedFile {
+    /** File path */
+    filePath: string;
+    /** Full raw file content */
+    content: string;
+    /** AST declarations outline skeleton */
+    outline: string;
+    /** Assigned context tier (1 = full, 2 = outline skeleton, 3 = omitted) */
+    tier: ContextTier;
+    /** Final composite relevance score (0.0 to 1.0+, higher is more relevant) */
+    score: number;
+    /** BM25 / TF-IDF query match score (normalized 0.0 to 1.0) */
+    bm25Score: number;
+    /** Dependency graph hub centrality score (normalized 0.0 to 1.0) */
+    centralityScore: number;
+    /** File path keyword match score (0.0 to 1.0) */
+    pathMatchScore: number;
+    /** Estimated token cost for this file based on its assigned tier */
+    estimatedTokens: number;
+    /** Estimated character cost for this file based on its assigned tier */
+    estimatedChars: number;
+    /** Whether the file was flagged as an explicit target */
+    isExplicitTarget?: boolean;
+    /** Metadata preserved from candidate */
+    metadata?: Record<string, unknown>;
+}
+export interface ContextRankerWeights {
+    /** Weight for BM25 content and query matching (default: 0.60) */
+    bm25?: number;
+    /** Weight for dependency graph centrality / hub score (default: 0.25) */
+    centrality?: number;
+    /** Weight for file path keyword match (default: 0.15) */
+    pathMatch?: number;
+}
+export interface RankFilesOptions {
+    /** Dependency graph to derive structural centrality from */
+    graph?: DependencyGraph;
+    /** Pre-calculated centrality metrics map (optional alternative to graph) */
+    centralityMetrics?: Map<string, CentralityMetrics>;
+    /** Total token budget available (default derived from CONTEXT_BUDGET_CONFIG: 60,000 tokens) */
+    tokenBudget?: number;
+    /** Total character budget (if provided, overrides tokenBudget * charsPerToken) */
+    charBudget?: number;
+    /** Estimated characters per token ratio (default: 4) */
+    charsPerToken?: number;
+    /** Maximum fraction of budget allocated to Tier 1 full files (default: 0.70) */
+    tier1BudgetRatio?: number;
+    /** Maximum fraction of budget allocated to Tier 2 AST skeletons (default: 0.30) */
+    tier2BudgetRatio?: number;
+    /** Minimum composite score required to be considered for Tier 1 / Tier 2 (default: 0.0) */
+    minScoreThreshold?: number;
+    /** Custom scoring weights */
+    weights?: ContextRankerWeights;
+    /** BM25 parameter k1 (term frequency saturation, default: 1.2) */
+    bm25K1?: number;
+    /** BM25 parameter b (document length normalization, default: 0.75) */
+    bm25B?: number;
+}
+export interface TierPartitionResult {
+    /** Tier 1: Primary edit targets & high-relevance files with 100% full content */
+    tier1: RankedFile[];
+    /** Tier 2: Peripheral hubs & dependent modules with compact AST skeletons */
+    tier2: RankedFile[];
+    /** Tier 3: Omitted files that exceed budget constraints */
+    tier3: RankedFile[];
+    /** All evaluated candidate files sorted descending by composite score */
+    rankedFiles: RankedFile[];
+    /** Total estimated tokens consumed by Tier 1 + Tier 2 files */
+    totalTokensUsed: number;
+    /** Total characters consumed by Tier 1 + Tier 2 files */
+    totalCharsUsed: number;
+    /** Budget statistics */
+    stats: {
+        totalCandidates: number;
+        tier1Count: number;
+        tier2Count: number;
+        tier3Count: number;
+        tokenBudget: number;
+        tokensUsed: number;
+        charBudget: number;
+        charsUsed: number;
+        budgetUtilization: number;
+    };
+}
+/**
+ * Tokenizes code and text into normalized keywords.
+ * Handles camelCase, PascalCase, snake_case, kebab-case, and dot/slash delimiters.
+ *
+ * @param text The input string to tokenize
+ * @param filterStopWords Whether to filter common English stop words (default: true)
+ * @returns Array of lowercase alphanumeric tokens
+ */
+export declare function tokenize(text: string, filterStopWords?: boolean): string[];
+/**
+ * Extracts unique search terms from a user prompt or query.
+ */
+export declare function extractQueryTerms(prompt: string): string[];
+export interface BM25Document {
+    id: string;
+    tokens: string[];
+    termFreqs: Map<string, number>;
+    length: number;
+}
+export interface BM25Options {
+    k1?: number;
+    b?: number;
+}
+/**
+ * In-memory BM25 index and scoring implementation.
+ * Delivers deterministic <5ms search ranking across candidate files.
+ */
+export declare class BM25Engine {
+    private avgDocLength;
+    private b;
+    private docFrequency;
+    private documents;
+    private k1;
+    private totalDocs;
+    constructor(options?: BM25Options);
+    /**
+     * Adds a document to the BM25 index.
+     * Path tokens can be repeated to apply a structural path boost.
+     */
+    addDocument(id: string, content: string, filePath: string, pathBoostFactor?: number): void;
+    /**
+     * Finalizes document statistics (average document length).
+     */
+    build(): void;
+    /**
+     * Scores all indexed documents against query terms using standard BM25 formula with Robertson-Spärck Jones IDF.
+     *
+     * @param queryTerms Array of query terms
+     * @returns Map of document ID to raw BM25 score
+     */
+    score(queryTerms: string[]): Map<string, number>;
+}
+/**
+ * Computes BM25 scores for a collection of file candidates.
+ *
+ * @param candidates List of file candidates
+ * @param queryTerms Query terms from prompt
+ * @param options BM25 parameters
+ * @returns Map of filePath to normalized BM25 score in [0.0, 1.0]
+ */
+export declare function computeBM25Scores(candidates: FileCandidate[], queryTerms: string[], options?: BM25Options): Map<string, {
+    rawScore: number;
+    normalizedScore: number;
+}>;
+/**
+ * Computes TF-IDF scores for a collection of candidates as a lightweight alternative.
+ */
+export declare function computeTfIdfScores(candidates: FileCandidate[], queryTerms: string[]): Map<string, {
+    rawScore: number;
+    normalizedScore: number;
+}>;
+/**
+ * Calculates a relevance score for a file path based on matching query terms against
+ * the file's basename, directory segments, and file extension.
+ *
+ * @param filePath File path
+ * @param queryTerms Extracted query terms
+ * @returns Path match score between 0.0 and 1.0
+ */
+export declare function calculatePathMatchScore(filePath: string, queryTerms: string[]): number;
+/**
+ * Extracts and normalizes centrality metrics for a list of candidate files.
+ *
+ * @param candidates List of candidate files
+ * @param graph Optional dependency graph
+ * @param customCentrality Optional pre-computed centrality map
+ * @returns Map of filePath to normalized centrality score in [0.0, 1.0]
+ */
+export declare function computeCandidateCentralityScores(candidates: FileCandidate[], graph?: DependencyGraph, customCentrality?: Map<string, CentralityMetrics>): Map<string, {
+    metrics: CentralityMetrics;
+    normalizedScore: number;
+}>;
+/**
+ * Estimates token count from string length using a characters-per-token heuristic.
+ */
+export declare function estimateTokens(text: string, charsPerToken?: number): number;
+/**
+ * Core Context Ranker:
+ * Ranks candidate files via hybrid BM25 + Graph Centrality + Path Match scoring,
+ * then partitions them into Tier 1 (full source), Tier 2 (AST outline skeleton),
+ * and Tier 3 (omitted) within strict token budgets.
+ *
+ * @param files Array of candidate file objects or map of filePath -> content
+ * @param prompt User prompt or instruction
+ * @param options Ranking and budgeting options
+ * @returns Partitioned 3-Tier result with complete metadata and statistics
+ */
+export declare function rankFilesForBudget(files: FileCandidate[] | Map<string, string> | Record<string, string>, prompt: string, options?: RankFilesOptions): TierPartitionResult;