distill-mcp 0.6.0-beta

package/dist/tools/token-budget.test.js

@@ -0,0 +1,275 @@
+/**
+ * Token Budget Tests
+ *
+ * Ensures tool definitions stay within token budgets to prevent
+ * context window bloat from MCP tool descriptions.
+ *
+ * These tests guard against regression - any change that increases
+ * token consumption will fail the test.
+ */
+import { describe, it, expect } from "vitest";
+import { countTokens } from "../utils/token-counter.js";
+// Import all tool definitions
+import { autoOptimizeTool } from "./auto-optimize.js";
+import { smartFileReadTool } from "./smart-file-read.js";
+import { discoverToolsTool } from "./discover-tools.js";
+import { analyzeBuildOutputTool } from "./analyze-build-output.js";
+import { compressContextTool } from "./compress-context.js";
+import { semanticCompressTool } from "./semantic-compress.js";
+import { diffCompressTool } from "./diff-compress.js";
+import { summarizeLogsTool } from "./summarize-logs.js";
+import { codeSkeletonTool } from "./code-skeleton.js";
+import { contextBudgetTool } from "./context-budget.js";
+import { conversationCompressTool } from "./conversation-compress.js";
+import { deduplicateErrorsTool } from "./deduplicate-errors.js";
+import { smartCacheTool } from "./smart-cache-tool.js";
+import { smartPipelineTool } from "./smart-pipeline.js";
+// ============================================================================
+// Token Budgets (in tokens)
+// ============================================================================
+/**
+ * Maximum tokens allowed per tool definition.
+ * These are intentionally tight to catch any bloat early.
+ *
+ * 2024-12: Tightened budgets after schema optimization
+ */
+const TOKEN_BUDGETS = {
+    // Core tools (always loaded) - ultra-minimal
+    auto_optimize: 90,
+    smart_file_read: 120,
+    discover_tools: 80, // +5 for TOON format option
+    // Compress category - aggressively optimized
+    compress_context: 95,
+    semantic_compress: 60,
+    diff_compress: 75,
+    conversation_compress: 110,
+    // Analyze category
+    analyze_build_output: 95,
+    context_budget: 105,
+    // Logs category
+    summarize_logs: 115,
+    deduplicate_errors: 65,
+    // Code category
+    code_skeleton: 75,
+    smart_cache: 90,
+    // Pipeline category
+    smart_pipeline: 80,
+};
+/**
+ * Maximum tokens for the entire ListTools response (core tools only).
+ * Currently: auto_optimize + smart_file_read + discover_tools
+ * 2024-12: Reduced from 500 after schema optimization
+ */
+const CORE_TOOLS_BUDGET = 300;
+/**
+ * Maximum tokens for all tools combined.
+ * 2024-12: Reduced from 1500 after aggressive schema optimization
+ */
+const ALL_TOOLS_BUDGET = 1200;
+// ============================================================================
+// Helper Functions
+// ============================================================================
+/**
+ * Serialize a tool definition as it would appear in ListTools response
+ */
+function serializeToolForMCP(tool) {
+    return JSON.stringify({
+        name: tool.name,
+        description: tool.description,
+        inputSchema: tool.inputSchema,
+    });
+}
+/**
+ * Count tokens in a tool definition
+ */
+function countToolTokens(tool) {
+    const serialized = serializeToolForMCP(tool);
+    return countTokens(serialized);
+}
+// ============================================================================
+// All Tools
+// ============================================================================
+const ALL_TOOLS = [
+    autoOptimizeTool,
+    smartFileReadTool,
+    discoverToolsTool,
+    analyzeBuildOutputTool,
+    compressContextTool,
+    semanticCompressTool,
+    diffCompressTool,
+    summarizeLogsTool,
+    codeSkeletonTool,
+    contextBudgetTool,
+    conversationCompressTool,
+    deduplicateErrorsTool,
+    smartCacheTool,
+    smartPipelineTool,
+];
+const CORE_TOOLS = [
+    autoOptimizeTool,
+    smartFileReadTool,
+    discoverToolsTool,
+];
+// ============================================================================
+// Tests
+// ============================================================================
+describe("Tool Token Budgets", () => {
+    describe("Individual tool budgets", () => {
+        it.each(ALL_TOOLS.map((t) => [t.name, t]))("%s should be under budget", (name, tool) => {
+            const tokens = countToolTokens(tool);
+            const budget = TOKEN_BUDGETS[name];
+            expect(tokens).toBeLessThanOrEqual(budget);
+            // Log for visibility
+            const usage = Math.round((tokens / budget) * 100);
+            console.log(`  ${name}: ${tokens}/${budget} tokens (${usage}%)`);
+        });
+    });
+    describe("Aggregate budgets", () => {
+        it("core tools should be under combined budget", () => {
+            const totalTokens = CORE_TOOLS.reduce((sum, tool) => sum + countToolTokens(tool), 0);
+            expect(totalTokens).toBeLessThanOrEqual(CORE_TOOLS_BUDGET);
+            const usage = Math.round((totalTokens / CORE_TOOLS_BUDGET) * 100);
+            console.log(`  Core tools total: ${totalTokens}/${CORE_TOOLS_BUDGET} tokens (${usage}%)`);
+        });
+        it("all tools should be under combined budget", () => {
+            const totalTokens = ALL_TOOLS.reduce((sum, tool) => sum + countToolTokens(tool), 0);
+            expect(totalTokens).toBeLessThanOrEqual(ALL_TOOLS_BUDGET);
+            const usage = Math.round((totalTokens / ALL_TOOLS_BUDGET) * 100);
+            console.log(`  All tools total: ${totalTokens}/${ALL_TOOLS_BUDGET} tokens (${usage}%)`);
+        });
+    });
+    describe("Token distribution", () => {
+        it("should have balanced token distribution (no tool > 20% of total)", () => {
+            const totalTokens = ALL_TOOLS.reduce((sum, tool) => sum + countToolTokens(tool), 0);
+            for (const tool of ALL_TOOLS) {
+                const tokens = countToolTokens(tool);
+                const percentage = (tokens / totalTokens) * 100;
+                expect(percentage).toBeLessThan(20);
+            }
+        });
+    });
+});
+describe("Tool Schema Constraints", () => {
+    describe("Description length", () => {
+        it.each(ALL_TOOLS.map((t) => [t.name, t]))("%s description should be concise (< 150 chars)", (name, tool) => {
+            const description = tool.description;
+            expect(description.length).toBeLessThan(150);
+        });
+    });
+    describe("Schema structure", () => {
+        it.each(ALL_TOOLS.map((t) => [t.name, t]))("%s should not have deeply nested descriptions", (name, tool) => {
+            const schema = tool.inputSchema;
+            const serialized = JSON.stringify(schema);
+            // Count "description" occurrences - should be minimal
+            const descriptionCount = (serialized.match(/"description"/g) || [])
+                .length;
+            // Allow max 3 descriptions per schema (for complex tools)
+            expect(descriptionCount).toBeLessThanOrEqual(3);
+        });
+    });
+});
+describe("ListTools Response Size", () => {
+    it("should generate compact ListTools response for core tools", () => {
+        const response = {
+            tools: CORE_TOOLS.map((tool) => ({
+                name: tool.name,
+                description: tool.description,
+                inputSchema: tool.inputSchema,
+            })),
+        };
+        const serialized = JSON.stringify(response);
+        const tokens = countTokens(serialized);
+        // ListTools response should be under 600 tokens for core tools
+        expect(tokens).toBeLessThan(600);
+        console.log(`  ListTools (core): ${serialized.length} chars, ${tokens} tokens`);
+    });
+    it("should generate compact ListTools response for all tools", () => {
+        const response = {
+            tools: ALL_TOOLS.map((tool) => ({
+                name: tool.name,
+                description: tool.description,
+                inputSchema: tool.inputSchema,
+            })),
+        };
+        const serialized = JSON.stringify(response);
+        const tokens = countTokens(serialized);
+        // Full ListTools response should be under 1800 tokens
+        expect(tokens).toBeLessThan(1800);
+        console.log(`  ListTools (all): ${serialized.length} chars, ${tokens} tokens`);
+    });
+});
+describe("Token Reduction Verification", () => {
+    /**
+     * Baseline values from BEFORE optimization (commit d4cdb98).
+     * These are used to verify we actually reduced tokens.
+     */
+    const BASELINE_TOKENS = {
+        auto_optimize: 287,
+        smart_file_read: 342,
+        discover_tools: 153,
+        core_total: 782,
+    };
+    it("auto_optimize should be reduced from baseline", () => {
+        const current = countToolTokens(autoOptimizeTool);
+        const baseline = BASELINE_TOKENS.auto_optimize;
+        const reduction = Math.round((1 - current / baseline) * 100);
+        expect(current).toBeLessThan(baseline);
+        expect(reduction).toBeGreaterThan(30); // At least 30% reduction
+        console.log(`  auto_optimize: ${baseline} → ${current} (${reduction}% reduction)`);
+    });
+    it("smart_file_read should be reduced from baseline", () => {
+        const current = countToolTokens(smartFileReadTool);
+        const baseline = BASELINE_TOKENS.smart_file_read;
+        const reduction = Math.round((1 - current / baseline) * 100);
+        expect(current).toBeLessThan(baseline);
+        expect(reduction).toBeGreaterThan(20); // At least 20% reduction
+        console.log(`  smart_file_read: ${baseline} → ${current} (${reduction}% reduction)`);
+    });
+    it("discover_tools should be reduced from baseline", () => {
+        const current = countToolTokens(discoverToolsTool);
+        const baseline = BASELINE_TOKENS.discover_tools;
+        const reduction = Math.round((1 - current / baseline) * 100);
+        expect(current).toBeLessThan(baseline);
+        expect(reduction).toBeGreaterThan(20); // At least 20% reduction
+        console.log(`  discover_tools: ${baseline} → ${current} (${reduction}% reduction)`);
+    });
+    it("core tools total should be at least 40% reduced from baseline", () => {
+        const currentTotal = CORE_TOOLS.reduce((sum, tool) => sum + countToolTokens(tool), 0);
+        const baseline = BASELINE_TOKENS.core_total;
+        const reduction = Math.round((1 - currentTotal / baseline) * 100);
+        expect(currentTotal).toBeLessThan(baseline);
+        expect(reduction).toBeGreaterThan(40); // At least 40% total reduction
+        console.log(`  Core total: ${baseline} → ${currentTotal} (${reduction}% reduction)`);
+    });
+});
+describe("Regression Prevention", () => {
+    /**
+     * Snapshot of current token counts.
+     * Update these when intentionally adding features.
+     * Any unexpected change will fail the test.
+     *
+     * 2024-12: Optimized schemas to reduce token overhead
+     * - Removed property descriptions (moved to tool description)
+     * - Removed rarely-used properties from public schema
+     * - Simplified nested object type declarations
+     * - Added TOON format output option to discover_tools
+     */
+    const CURRENT_SNAPSHOT = {
+        auto_optimize: 80,
+        smart_file_read: 106,
+        discover_tools: 78, // +15 for TOON format option (list|toon|toon-tabular)
+    };
+    // Tolerance: ±5 tokens for minor changes
+    const TOLERANCE = 5;
+    it.each(Object.entries(CURRENT_SNAPSHOT))("%s should match snapshot (±5 tokens)", (name, expected) => {
+        const tool = ALL_TOOLS.find((t) => t.name === name);
+        if (!tool)
+            throw new Error(`Tool ${name} not found`);
+        const actual = countToolTokens(tool);
+        const diff = Math.abs(actual - expected);
+        expect(diff).toBeLessThanOrEqual(TOLERANCE);
+        if (diff > 0) {
+            console.log(`  ${name}: expected ${expected}, got ${actual} (diff: ${diff})`);
+        }
+    });
+});

package/dist/utils/bm25.d.ts

@@ -0,0 +1,86 @@
+/**
+ * BM25 (Best Matching 25) implementation for tool search
+ *
+ * BM25 is a ranking function used by search engines to rank documents
+ * based on query terms appearing in each document.
+ *
+ * Formula: score = IDF * ((tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (dl/avgdl))))
+ *
+ * Where:
+ * - tf = term frequency in document
+ * - k1 = term saturation parameter (default: 1.2)
+ * - b = length normalization parameter (default: 0.75)
+ * - dl = document length (number of terms)
+ * - avgdl = average document length across corpus
+ * - IDF = log((N - df + 0.5) / (df + 0.5) + 1)
+ * - N = total number of documents
+ * - df = document frequency (docs containing term)
+ */
+/**
+ * Configuration options for BM25 algorithm
+ */
+export interface BM25Options {
+    /** Term saturation parameter. Higher = more weight to term frequency. Default: 1.2 */
+    k1?: number;
+    /** Length normalization. 0 = no normalization, 1 = full normalization. Default: 0.75 */
+    b?: number;
+}
+/**
+ * Result item from BM25 search
+ */
+export interface BM25Result<T> {
+    /** Original item from the corpus */
+    item: T;
+    /** BM25 relevance score (higher = more relevant) */
+    score: number;
+    /** Query terms that matched in this document */
+    matchedTerms: string[];
+}
+/**
+ * BM25 search index interface
+ */
+export interface BM25Index<T> {
+    /** Search the index and return ranked results */
+    search: (query: string) => BM25Result<T>[];
+    /** Get corpus statistics */
+    stats: () => {
+        documentCount: number;
+        avgDocLength: number;
+        vocabularySize: number;
+    };
+}
+/**
+ * Tokenize text into searchable terms
+ *
+ * - Lowercases text
+ * - Removes punctuation
+ * - Splits on whitespace
+ * - Filters words shorter than 2 characters
+ */
+export declare function tokenize(text: string): string[];
+/**
+ * Create a BM25 search index from a collection of items
+ *
+ * @param items - Array of items to index
+ * @param getSearchableText - Function to extract searchable text from each item
+ * @param options - BM25 configuration options
+ * @returns BM25 search index
+ *
+ * @example
+ * ```typescript
+ * const tools = [
+ *   { name: "compress", description: "Compress content" },
+ *   { name: "analyze", description: "Analyze build output" }
+ * ];
+ *
+ * const index = createBM25Index(
+ *   tools,
+ *   (tool) => `${tool.name} ${tool.description}`
+ * );
+ *
+ * const results = index.search("compress content");
+ * // [{ item: { name: "compress", ... }, score: 2.5, matchedTerms: ["compress", "content"] }]
+ * ```
+ */
+export declare function createBM25Index<T>(items: T[], getSearchableText: (item: T) => string, options?: BM25Options): BM25Index<T>;
+//# sourceMappingURL=bm25.d.ts.map

package/dist/utils/bm25.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/utils/bm25.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,sFAAsF;IACtF,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,wFAAwF;IACxF,CAAC,CAAC,EAAE,MAAM,CAAC;CACZ;AAED;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,oCAAoC;IACpC,IAAI,EAAE,CAAC,CAAC;IACR,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS,CAAC,CAAC;IAC1B,iDAAiD;IACjD,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,4BAA4B;IAC5B,KAAK,EAAE,MAAM;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE,CAAC;CACtF;AAED;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM/C;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAC/B,KAAK,EAAE,CAAC,EAAE,EACV,iBAAiB,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EACtC,OAAO,CAAC,EAAE,WAAW,GACpB,SAAS,CAAC,CAAC,CAAC,CAkHd"}

package/dist/utils/bm25.js

@@ -0,0 +1,153 @@
+/**
+ * BM25 (Best Matching 25) implementation for tool search
+ *
+ * BM25 is a ranking function used by search engines to rank documents
+ * based on query terms appearing in each document.
+ *
+ * Formula: score = IDF * ((tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (dl/avgdl))))
+ *
+ * Where:
+ * - tf = term frequency in document
+ * - k1 = term saturation parameter (default: 1.2)
+ * - b = length normalization parameter (default: 0.75)
+ * - dl = document length (number of terms)
+ * - avgdl = average document length across corpus
+ * - IDF = log((N - df + 0.5) / (df + 0.5) + 1)
+ * - N = total number of documents
+ * - df = document frequency (docs containing term)
+ */
+/**
+ * Tokenize text into searchable terms
+ *
+ * - Lowercases text
+ * - Removes punctuation
+ * - Splits on whitespace
+ * - Filters words shorter than 2 characters
+ */
+export function tokenize(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\w\s]/g, " ")
+        .split(/\s+/)
+        .filter((word) => word.length >= 2);
+}
+/**
+ * Create a BM25 search index from a collection of items
+ *
+ * @param items - Array of items to index
+ * @param getSearchableText - Function to extract searchable text from each item
+ * @param options - BM25 configuration options
+ * @returns BM25 search index
+ *
+ * @example
+ * ```typescript
+ * const tools = [
+ *   { name: "compress", description: "Compress content" },
+ *   { name: "analyze", description: "Analyze build output" }
+ * ];
+ *
+ * const index = createBM25Index(
+ *   tools,
+ *   (tool) => `${tool.name} ${tool.description}`
+ * );
+ *
+ * const results = index.search("compress content");
+ * // [{ item: { name: "compress", ... }, score: 2.5, matchedTerms: ["compress", "content"] }]
+ * ```
+ */
+export function createBM25Index(items, getSearchableText, options) {
+    const k1 = options?.k1 ?? 1.2;
+    const b = options?.b ?? 0.75;
+    // Handle empty corpus
+    if (items.length === 0) {
+        return {
+            search: () => [],
+            stats: () => ({ documentCount: 0, avgDocLength: 0, vocabularySize: 0 }),
+        };
+    }
+    // Build tokenized corpus
+    const documents = items.map((item) => tokenize(getSearchableText(item)));
+    // Calculate average document length
+    const totalLength = documents.reduce((sum, doc) => sum + doc.length, 0);
+    const avgdl = totalLength / documents.length;
+    // Build document frequency map (how many docs contain each term)
+    const df = new Map();
+    for (const doc of documents) {
+        const uniqueTerms = new Set(doc);
+        for (const term of uniqueTerms) {
+            df.set(term, (df.get(term) ?? 0) + 1);
+        }
+    }
+    const N = documents.length;
+    /**
+     * Calculate IDF (Inverse Document Frequency) for a term
+     * Using the standard BM25 IDF formula with smoothing
+     */
+    function idf(term) {
+        const docFreq = df.get(term) ?? 0;
+        // BM25 IDF formula: log((N - df + 0.5) / (df + 0.5) + 1)
+        return Math.log((N - docFreq + 0.5) / (docFreq + 0.5) + 1);
+    }
+    /**
+     * Score a single document against query terms
+     */
+    function scoreDocument(docIndex, queryTerms) {
+        const doc = documents[docIndex];
+        if (!doc) {
+            return { score: 0, matchedTerms: [] };
+        }
+        const dl = doc.length;
+        // Build term frequency map for this document
+        const termFreq = new Map();
+        for (const term of doc) {
+            termFreq.set(term, (termFreq.get(term) ?? 0) + 1);
+        }
+        let score = 0;
+        const matchedTerms = [];
+        for (const term of queryTerms) {
+            const tf = termFreq.get(term) ?? 0;
+            if (tf > 0) {
+                matchedTerms.push(term);
+                const termIdf = idf(term);
+                // BM25 term score formula
+                const numerator = tf * (k1 + 1);
+                const denominator = tf + k1 * (1 - b + b * (dl / avgdl));
+                score += termIdf * (numerator / denominator);
+            }
+        }
+        return { score, matchedTerms };
+    }
+    return {
+        search(query) {
+            const queryTerms = tokenize(query);
+            // Empty query returns no results
+            if (queryTerms.length === 0) {
+                return [];
+            }
+            const results = [];
+            for (let i = 0; i < items.length; i++) {
+                const item = items[i];
+                if (!item)
+                    continue;
+                const { score, matchedTerms } = scoreDocument(i, queryTerms);
+                // Only include documents with at least one matching term
+                if (score > 0) {
+                    results.push({
+                        item,
+                        score,
+                        matchedTerms,
+                    });
+                }
+            }
+            // Sort by score descending (most relevant first)
+            return results.sort((a, b) => b.score - a.score);
+        },
+        stats() {
+            return {
+                documentCount: N,
+                avgDocLength: avgdl,
+                vocabularySize: df.size,
+            };
+        },
+    };
+}

package/dist/utils/bm25.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * BM25 Search Algorithm Tests
+ */
+export {};
+//# sourceMappingURL=bm25.test.d.ts.map

package/dist/utils/bm25.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.test.d.ts","sourceRoot":"","sources":["../../src/utils/bm25.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/utils/bm25.test.js

@@ -0,0 +1,156 @@
+/**
+ * BM25 Search Algorithm Tests
+ */
+import { describe, it, expect } from "vitest";
+import { tokenize, createBM25Index } from "./bm25.js";
+describe("BM25 utilities", () => {
+    describe("tokenize", () => {
+        it("should lowercase and split text", () => {
+            const result = tokenize("Hello World Test");
+            expect(result).toEqual(["hello", "world", "test"]);
+        });
+        it("should remove punctuation", () => {
+            const result = tokenize("hello, world! how's it going?");
+            expect(result).toEqual(["hello", "world", "how", "it", "going"]);
+        });
+        it("should filter short words (less than 2 chars)", () => {
+            const result = tokenize("I am a test for x y z");
+            expect(result).toEqual(["am", "test", "for"]);
+        });
+        it("should handle empty string", () => {
+            const result = tokenize("");
+            expect(result).toEqual([]);
+        });
+        it("should handle string with only punctuation", () => {
+            const result = tokenize("!@#$%^&*()");
+            expect(result).toEqual([]);
+        });
+    });
+    describe("createBM25Index", () => {
+        const testTools = [
+            { name: "compress", description: "Compress and reduce content size" },
+            { name: "analyze", description: "Analyze build output and errors" },
+            { name: "summarize", description: "Summarize log files" },
+            { name: "optimize", description: "Optimize token usage" },
+        ];
+        const getSearchableText = (tool) => `${tool.name} ${tool.description}`;
+        it("should return empty array for empty query", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const results = index.search("");
+            expect(results).toEqual([]);
+        });
+        it("should return empty array for query with only short words", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const results = index.search("a x");
+            expect(results).toEqual([]);
+        });
+        it("should return empty array for empty corpus", () => {
+            const index = createBM25Index([], getSearchableText);
+            const results = index.search("compress");
+            expect(results).toEqual([]);
+        });
+        it("should find exact name matches", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const results = index.search("compress");
+            expect(results.length).toBeGreaterThan(0);
+            expect(results[0].item.name).toBe("compress");
+        });
+        it("should find matches in description", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const results = index.search("build output");
+            expect(results.length).toBeGreaterThan(0);
+            expect(results[0].item.name).toBe("analyze");
+        });
+        it("should rank exact matches higher", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const results = index.search("compress content");
+            expect(results.length).toBeGreaterThan(0);
+            // "compress" tool should be first as it matches both query terms
+            expect(results[0].item.name).toBe("compress");
+            expect(results[0].matchedTerms).toContain("compress");
+            expect(results[0].matchedTerms).toContain("content");
+        });
+        it("should handle multi-word queries", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const results = index.search("analyze build errors");
+            expect(results.length).toBeGreaterThan(0);
+            expect(results[0].item.name).toBe("analyze");
+            expect(results[0].matchedTerms.length).toBeGreaterThan(1);
+        });
+        it("should return matchedTerms for each result", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const results = index.search("compress");
+            expect(results.length).toBeGreaterThan(0);
+            expect(results[0].matchedTerms).toContain("compress");
+        });
+        it("should return results sorted by score descending", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const results = index.search("optimize token");
+            for (let i = 1; i < results.length; i++) {
+                expect(results[i - 1].score).toBeGreaterThanOrEqual(results[i].score);
+            }
+        });
+        it("should return positive scores for matching documents", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const results = index.search("compress");
+            for (const result of results) {
+                expect(result.score).toBeGreaterThan(0);
+            }
+        });
+        it("should not return documents without matching terms", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const results = index.search("nonexistent");
+            expect(results).toEqual([]);
+        });
+        it("should provide corpus statistics", () => {
+            const index = createBM25Index(testTools, getSearchableText);
+            const stats = index.stats();
+            expect(stats.documentCount).toBe(4);
+            expect(stats.avgDocLength).toBeGreaterThan(0);
+            expect(stats.vocabularySize).toBeGreaterThan(0);
+        });
+        it("should handle custom BM25 parameters", () => {
+            const index = createBM25Index(testTools, getSearchableText, {
+                k1: 2.0,
+                b: 0.5,
+            });
+            const results = index.search("compress");
+            expect(results.length).toBeGreaterThan(0);
+        });
+    });
+    describe("BM25 ranking behavior", () => {
+        it("should rank documents with rare terms higher", () => {
+            const docs = [
+                { id: 1, text: "common common common" },
+                { id: 2, text: "common rare unique" },
+                { id: 3, text: "common common" },
+            ];
+            const index = createBM25Index(docs, (d) => d.text);
+            const results = index.search("unique");
+            expect(results.length).toBe(1);
+            expect(results[0].item.id).toBe(2);
+        });
+        it("should give higher scores to shorter documents (length normalization)", () => {
+            const docs = [
+                { id: 1, text: "compress file" },
+                { id: 2, text: "compress file with additional words that make it longer" },
+            ];
+            const index = createBM25Index(docs, (d) => d.text);
+            const results = index.search("compress file");
+            // Shorter document should have higher score due to length normalization
+            expect(results[0].item.id).toBe(1);
+        });
+        it("should handle repeated terms correctly", () => {
+            const docs = [
+                { id: 1, text: "error error error" },
+                { id: 2, text: "error warning info" },
+            ];
+            const index = createBM25Index(docs, (d) => d.text);
+            const results = index.search("error");
+            // Both should match, with scores reflecting term frequency
+            expect(results.length).toBe(2);
+            // Document with more "error" terms should score higher (BM25 saturation)
+            expect(results[0].item.id).toBe(1);
+        });
+    });
+});