distill-mcp 0.6.0-beta

package/dist/tools/dynamic-loader.js

@@ -0,0 +1,378 @@
+/**
+ * Dynamic Tool Loader
+ *
+ * Manages lazy loading of MCP tools to reduce token consumption.
+ * Only core tools are loaded at startup; others are loaded on-demand.
+ */
+import { createBM25Index } from "../utils/bm25.js";
+import { createHybridSearchIndex, } from "../utils/hybrid-search.js";
+/**
+ * Tool catalog with metadata for discovery
+ * Full tool definitions are loaded lazily via the loader function
+ */
+export const TOOL_CATALOG = [
+    // Core tools (always loaded)
+    {
+        name: "auto_optimize",
+        category: "core",
+        keywords: ["optimize", "auto", "detect", "compress"],
+        description: "Auto-detect content type and apply optimal compression",
+        loader: async () => (await import("./auto-optimize.js")).autoOptimizeTool,
+    },
+    {
+        name: "smart_file_read",
+        category: "core",
+        keywords: ["file", "read", "ast", "code", "extract"],
+        description: "Read files with AST-based extraction",
+        loader: async () => (await import("./smart-file-read.js")).smartFileReadTool,
+    },
+    {
+        name: "code_execute",
+        category: "core",
+        keywords: ["execute", "code", "sdk", "typescript", "script", "sandbox"],
+        description: "Execute TypeScript with ctxopt SDK (98% token savings)",
+        loader: async () => (await import("./code-execute.js")).codeExecuteTool,
+    },
+    // Compress category
+    {
+        name: "compress_context",
+        category: "compress",
+        keywords: ["compress", "context", "reduce", "shrink", "generic"],
+        description: "Compress generic text content (logs, configs)",
+        loader: async () => (await import("./compress-context.js")).compressContextTool,
+    },
+    {
+        name: "semantic_compress",
+        category: "compress",
+        keywords: ["semantic", "compress", "tfidf", "importance"],
+        description: "TF-IDF based semantic compression",
+        loader: async () => (await import("./semantic-compress.js")).semanticCompressTool,
+    },
+    {
+        name: "diff_compress",
+        category: "compress",
+        keywords: ["diff", "git", "compress", "changes"],
+        description: "Compress git diff output",
+        loader: async () => (await import("./diff-compress.js")).diffCompressTool,
+    },
+    {
+        name: "conversation_compress",
+        category: "compress",
+        keywords: ["conversation", "chat", "history", "compress", "messages"],
+        description: "Compress conversation history",
+        loader: async () => (await import("./conversation-compress.js")).conversationCompressTool,
+    },
+    // Analyze category
+    {
+        name: "analyze_context",
+        category: "analyze",
+        keywords: ["analyze", "context", "tokens", "cost", "estimate"],
+        description: "Analyze prompt for token usage and cost",
+        loader: async () => {
+            const mod = await import("./analyze-context.js");
+            return {
+                name: "analyze_context",
+                description: "Analyze a prompt or context for token usage and optimization opportunities.",
+                inputSchema: mod.analyzeContextSchema,
+                execute: async (args) => mod.analyzeContext(args, {}),
+            };
+        },
+    },
+    {
+        name: "analyze_build_output",
+        category: "analyze",
+        keywords: ["build", "error", "typescript", "eslint", "webpack", "compile"],
+        description: "Parse and compress build errors",
+        loader: async () => (await import("./analyze-build-output.js")).analyzeBuildOutputTool,
+    },
+    {
+        name: "context_budget",
+        category: "analyze",
+        keywords: ["budget", "tokens", "limit", "estimate", "cost"],
+        description: "Pre-flight token budget estimation",
+        loader: async () => (await import("./context-budget.js")).contextBudgetTool,
+    },
+    // Logs category
+    {
+        name: "summarize_logs",
+        category: "logs",
+        keywords: ["logs", "summarize", "server", "test", "output"],
+        description: "Summarize verbose log output",
+        loader: async () => (await import("./summarize-logs.js")).summarizeLogsTool,
+    },
+    {
+        name: "deduplicate_errors",
+        category: "logs",
+        keywords: ["deduplicate", "errors", "group", "repeat"],
+        description: "Group and deduplicate repeated errors",
+        loader: async () => (await import("./deduplicate-errors.js")).deduplicateErrorsTool,
+    },
+    // Code category
+    {
+        name: "code_skeleton",
+        category: "code",
+        keywords: ["skeleton", "signatures", "code", "structure", "overview"],
+        description: "Extract function/class signatures only",
+        loader: async () => (await import("./code-skeleton.js")).codeSkeletonTool,
+    },
+    {
+        name: "smart_cache",
+        category: "code",
+        keywords: ["cache", "store", "retrieve", "invalidate"],
+        description: "Manage parsed file cache",
+        loader: async () => (await import("./smart-cache-tool.js")).smartCacheTool,
+    },
+    // Pipeline category
+    {
+        name: "smart_pipeline",
+        category: "pipeline",
+        keywords: ["pipeline", "chain", "auto", "multi-step"],
+        description: "Chain multiple compression tools automatically",
+        loader: async () => (await import("./smart-pipeline.js")).smartPipelineTool,
+    },
+    {
+        name: "optimization_tips",
+        category: "pipeline",
+        keywords: ["tips", "best-practices", "optimize", "help"],
+        description: "Get context optimization best practices",
+        loader: async () => {
+            const mod = await import("./optimization-tips.js");
+            return {
+                name: "optimization_tips",
+                description: "Get context engineering best practices and optimization tips.",
+                inputSchema: mod.optimizationTipsSchema,
+                execute: async (args) => mod.optimizationTips(args, {}),
+            };
+        },
+    },
+    // Intelligence layer
+    {
+        name: "detect_retry_loop",
+        category: "analyze",
+        keywords: ["retry", "loop", "build", "test", "fail", "repeat", "stuck"],
+        description: "Detect retry loops in command history",
+        loader: async () => (await import("./detect-retry-loop.js")).detectRetryLoopTool,
+    },
+    {
+        name: "session_stats",
+        category: "core",
+        keywords: ["session", "stats", "statistics", "tokens", "saved", "analytics"],
+        description: "Get session-wide token savings and usage statistics",
+        loader: async () => (await import("./session-stats.js")).sessionStatsTool,
+    },
+    // Phase 6: Advanced Compression
+    {
+        name: "set_output_config",
+        category: "pipeline",
+        keywords: ["config", "output", "format", "toon", "verbosity", "settings"],
+        description: "Configure global output format and verbosity",
+        loader: async () => (await import("./set-output-config.js")).setOutputConfigTool,
+    },
+    {
+        name: "multifile_compress",
+        category: "compress",
+        keywords: ["multifile", "deduplicate", "shared", "imports", "types", "chunk", "context"],
+        description: "Compress multiple files with cross-file deduplication",
+        loader: async () => (await import("./multifile-compress.js")).multifileCompressTool,
+    },
+    {
+        name: "conversation_memory",
+        category: "compress",
+        keywords: ["conversation", "memory", "decisions", "restore", "context", "history"],
+        description: "Manage long conversation context with decision extraction",
+        loader: async () => (await import("./conversation-memory.js")).conversationMemoryTool,
+    },
+];
+/**
+ * Dynamic Tool Loader
+ *
+ * Manages tool loading and provides discovery capabilities.
+ */
+export class DynamicToolLoader {
+    loadedTools = new Map();
+    onChangeCallbacks = [];
+    bm25Index = null;
+    hybridIndex = null;
+    /**
+     * Get searchable text from tool metadata
+     */
+    static getSearchableText(tool) {
+        return `${tool.name} ${tool.keywords.join(" ")} ${tool.description}`;
+    }
+    /**
+     * Get or create BM25 search index (lazy initialization)
+     */
+    getBM25Index() {
+        if (!this.bm25Index) {
+            this.bm25Index = createBM25Index(TOOL_CATALOG, DynamicToolLoader.getSearchableText, { k1: 1.2, b: 0.75 });
+        }
+        return this.bm25Index;
+    }
+    /**
+     * Get or create hybrid search index (lazy initialization)
+     */
+    getHybridIndex() {
+        if (!this.hybridIndex) {
+            this.hybridIndex = createHybridSearchIndex(TOOL_CATALOG, DynamicToolLoader.getSearchableText, { bm25Weight: 0.4, semanticWeight: 0.6 });
+        }
+        return this.hybridIndex;
+    }
+    /**
+     * Get metadata for all available tools (without loading them)
+     */
+    getAvailableTools() {
+        return TOOL_CATALOG.map((t) => ({
+            name: t.name,
+            category: t.category,
+            description: t.description,
+        }));
+    }
+    /**
+     * Search tools by query string using BM25 ranking
+     * Returns results sorted by relevance score (most relevant first)
+     */
+    searchTools(query) {
+        const index = this.getBM25Index();
+        const results = index.search(query);
+        return results.map((r) => r.item);
+    }
+    /**
+     * Search tools with BM25 scores and matched terms
+     * Useful for debugging or displaying search relevance
+     */
+    searchToolsWithScores(query) {
+        const index = this.getBM25Index();
+        return index.search(query);
+    }
+    /**
+     * Hybrid search combining BM25 (lexical) and semantic similarity
+     *
+     * This method finds tools even when the query uses different words
+     * than the tool description (e.g., "shrink output" → compress).
+     *
+     * @param query - Search query
+     * @returns Ranked results with both BM25 and semantic scores
+     */
+    async searchToolsHybrid(query) {
+        const index = this.getHybridIndex();
+        return index.search(query);
+    }
+    /**
+     * Preload semantic embeddings for faster hybrid search
+     *
+     * Call this during idle time to avoid latency on first hybrid search.
+     * The embedding model (~23MB) is downloaded and cached on first use.
+     */
+    async preloadSemanticSearch() {
+        const index = this.getHybridIndex();
+        await index.precomputeEmbeddings();
+    }
+    /**
+     * Check if semantic search is ready (embeddings loaded)
+     */
+    isSemanticSearchReady() {
+        return this.hybridIndex?.isSemanticReady() ?? false;
+    }
+    /**
+     * Get tools by category
+     */
+    getToolsByCategory(category) {
+        return TOOL_CATALOG.filter((t) => t.category === category);
+    }
+    /**
+     * Load core tools (called at startup)
+     */
+    async loadCoreTools() {
+        const coreTools = TOOL_CATALOG.filter((t) => t.category === "core");
+        return this.loadToolsFromMetadata(coreTools);
+    }
+    /**
+     * Load tools by category
+     */
+    async loadByCategory(category) {
+        const tools = this.getToolsByCategory(category);
+        return this.loadToolsFromMetadata(tools);
+    }
+    /**
+     * Load tools matching a query
+     */
+    async loadByQuery(query) {
+        const tools = this.searchTools(query);
+        return this.loadToolsFromMetadata(tools);
+    }
+    /**
+     * Load specific tools by name
+     */
+    async loadByNames(names) {
+        const tools = TOOL_CATALOG.filter((t) => names.includes(t.name));
+        return this.loadToolsFromMetadata(tools);
+    }
+    /**
+     * Load all tools (fallback for clients that don't support dynamic loading)
+     */
+    async loadAllTools() {
+        return this.loadToolsFromMetadata(TOOL_CATALOG);
+    }
+    /**
+     * Get already loaded tools
+     */
+    getLoadedTools() {
+        return Array.from(this.loadedTools.values());
+    }
+    /**
+     * Check if a tool is loaded
+     */
+    isLoaded(name) {
+        return this.loadedTools.has(name);
+    }
+    /**
+     * Register callback for tool list changes
+     */
+    onToolsChanged(callback) {
+        this.onChangeCallbacks.push(callback);
+    }
+    /**
+     * Internal: Load tools from metadata array
+     */
+    async loadToolsFromMetadata(metadata) {
+        const newlyLoaded = [];
+        for (const meta of metadata) {
+            if (!this.loadedTools.has(meta.name)) {
+                const tool = await meta.loader();
+                this.loadedTools.set(meta.name, tool);
+                newlyLoaded.push(tool);
+            }
+        }
+        // Notify if new tools were loaded
+        if (newlyLoaded.length > 0) {
+            this.emitChange();
+        }
+        return newlyLoaded;
+    }
+    /**
+     * Emit change notification
+     */
+    emitChange() {
+        for (const cb of this.onChangeCallbacks) {
+            try {
+                cb();
+            }
+            catch {
+                // Ignore callback errors
+            }
+        }
+    }
+}
+/**
+ * Singleton instance
+ */
+let loaderInstance = null;
+export function getDynamicLoader() {
+    if (!loaderInstance) {
+        loaderInstance = new DynamicToolLoader();
+    }
+    return loaderInstance;
+}
+export function resetDynamicLoader() {
+    loaderInstance = null;
+}

package/dist/tools/dynamic-loader.test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Dynamic Tool Loader Tests
+ *
+ * Tests for lazy loading functionality to ensure:
+ * 1. Only core tools are loaded at startup
+ * 2. Non-core tools are loaded on-demand
+ * 3. Token savings from lazy loading
+ */
+export {};
+//# sourceMappingURL=dynamic-loader.test.d.ts.map

package/dist/tools/dynamic-loader.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dynamic-loader.test.d.ts","sourceRoot":"","sources":["../../src/tools/dynamic-loader.test.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}

package/dist/tools/dynamic-loader.test.js

@@ -0,0 +1,164 @@
+/**
+ * Dynamic Tool Loader Tests
+ *
+ * Tests for lazy loading functionality to ensure:
+ * 1. Only core tools are loaded at startup
+ * 2. Non-core tools are loaded on-demand
+ * 3. Token savings from lazy loading
+ */
+import { describe, it, expect, beforeEach } from "vitest";
+import { getDynamicLoader, resetDynamicLoader, TOOL_CATALOG, } from "./dynamic-loader.js";
+import { countTokens } from "../utils/token-counter.js";
+describe("DynamicToolLoader", () => {
+    let loader;
+    beforeEach(() => {
+        resetDynamicLoader();
+        loader = getDynamicLoader();
+    });
+    describe("Core tools loading", () => {
+        it("should only load core tools at startup", async () => {
+            const coreTools = await loader.loadCoreTools();
+            // Should only load tools marked as "core"
+            const coreNames = coreTools.map((t) => t.name);
+            expect(coreNames).toContain("auto_optimize");
+            expect(coreNames).toContain("smart_file_read");
+            // Should NOT load non-core tools
+            expect(coreNames).not.toContain("compress_context");
+            expect(coreNames).not.toContain("analyze_build_output");
+            expect(coreNames).not.toContain("summarize_logs");
+        });
+        it("should have fewer loaded tools than total catalog", async () => {
+            await loader.loadCoreTools();
+            const loadedTools = loader.getLoadedTools();
+            expect(loadedTools.length).toBeLessThan(TOOL_CATALOG.length);
+            console.log(`  Core tools: ${loadedTools.length}/${TOOL_CATALOG.length} (${Math.round((loadedTools.length / TOOL_CATALOG.length) * 100)}%)`);
+        });
+    });
+    describe("On-demand loading", () => {
+        it("should load tools by category", async () => {
+            await loader.loadCoreTools();
+            const beforeCount = loader.getLoadedTools().length;
+            const compressTools = await loader.loadByCategory("compress");
+            const afterCount = loader.getLoadedTools().length;
+            expect(afterCount).toBeGreaterThan(beforeCount);
+            // Verify compress tools are loaded
+            const loadedNames = loader.getLoadedTools().map((t) => t.name);
+            expect(loadedNames).toContain("compress_context");
+        });
+        it("should load tools by name", async () => {
+            await loader.loadCoreTools();
+            expect(loader.isLoaded("summarize_logs")).toBe(false);
+            await loader.loadByNames(["summarize_logs"]);
+            expect(loader.isLoaded("summarize_logs")).toBe(true);
+        });
+        it("should load tools by query", async () => {
+            await loader.loadCoreTools();
+            const beforeCount = loader.getLoadedTools().length;
+            await loader.loadByQuery("build");
+            const afterCount = loader.getLoadedTools().length;
+            expect(afterCount).toBeGreaterThan(beforeCount);
+        });
+        it("should not duplicate already loaded tools", async () => {
+            await loader.loadCoreTools();
+            const firstCount = loader.getLoadedTools().length;
+            await loader.loadCoreTools(); // Load again
+            const secondCount = loader.getLoadedTools().length;
+            expect(secondCount).toBe(firstCount);
+        });
+    });
+    describe("Token savings from lazy loading", () => {
+        it("should save tokens by loading only core tools", async () => {
+            // Calculate tokens for core tools only
+            const coreTools = await loader.loadCoreTools();
+            const coreTokens = coreTools.reduce((sum, tool) => {
+                const serialized = JSON.stringify({
+                    name: tool.name,
+                    description: tool.description,
+                    inputSchema: tool.inputSchema,
+                });
+                return sum + countTokens(serialized);
+            }, 0);
+            // Reset and load all tools
+            resetDynamicLoader();
+            const freshLoader = getDynamicLoader();
+            const allTools = await freshLoader.loadAllTools();
+            const allTokens = allTools.reduce((sum, tool) => {
+                const serialized = JSON.stringify({
+                    name: tool.name,
+                    description: tool.description,
+                    inputSchema: tool.inputSchema,
+                });
+                return sum + countTokens(serialized);
+            }, 0);
+            // Core tools should use significantly fewer tokens
+            const savings = Math.round((1 - coreTokens / allTokens) * 100);
+            expect(savings).toBeGreaterThan(50); // At least 50% savings
+            console.log(`  Token savings: ${allTokens} → ${coreTokens} (-${savings}%)`);
+        });
+    });
+    describe("Tool catalog integrity", () => {
+        it("should have all required categories", () => {
+            const categories = new Set(TOOL_CATALOG.map((t) => t.category));
+            expect(categories).toContain("core");
+            expect(categories).toContain("compress");
+            expect(categories).toContain("analyze");
+            expect(categories).toContain("logs");
+            expect(categories).toContain("code");
+            expect(categories).toContain("pipeline");
+        });
+        it("should have unique tool names", () => {
+            const names = TOOL_CATALOG.map((t) => t.name);
+            const uniqueNames = new Set(names);
+            expect(uniqueNames.size).toBe(names.length);
+        });
+        it("should have loaders that resolve", async () => {
+            for (const meta of TOOL_CATALOG) {
+                const tool = await meta.loader();
+                expect(tool).toBeDefined();
+                expect(tool.name).toBe(meta.name);
+                expect(tool.execute).toBeInstanceOf(Function);
+            }
+        });
+    });
+    describe("Change notifications", () => {
+        it("should notify when new tools are loaded", async () => {
+            let notified = false;
+            loader.onToolsChanged(() => {
+                notified = true;
+            });
+            await loader.loadCoreTools();
+            expect(notified).toBe(true);
+        });
+        it("should not notify when no new tools are loaded", async () => {
+            await loader.loadCoreTools();
+            let notified = false;
+            loader.onToolsChanged(() => {
+                notified = true;
+            });
+            await loader.loadCoreTools(); // Load same tools again
+            expect(notified).toBe(false);
+        });
+    });
+});
+describe("Metadata-only discovery", () => {
+    let loader;
+    beforeEach(() => {
+        resetDynamicLoader();
+        loader = getDynamicLoader();
+    });
+    it("should get available tools without loading them", () => {
+        const available = loader.getAvailableTools();
+        expect(available.length).toBe(TOOL_CATALOG.length);
+        expect(loader.getLoadedTools().length).toBe(0);
+    });
+    it("should search tools without loading them", () => {
+        const results = loader.searchTools("compress");
+        expect(results.length).toBeGreaterThan(0);
+        expect(loader.getLoadedTools().length).toBe(0);
+    });
+    it("should get tools by category without loading them", () => {
+        const results = loader.getToolsByCategory("logs");
+        expect(results.length).toBeGreaterThan(0);
+        expect(loader.getLoadedTools().length).toBe(0);
+    });
+});

package/dist/tools/lazy-mcp.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Lazy MCP - Pattern for 95% token reduction
+ *
+ * Implements the lazy-mcp pattern where only 2 meta-tools are exposed:
+ * - browse_tools: Navigate the tool hierarchy
+ * - run_tool: Execute any tool by name
+ *
+ * This reduces initial context from ~15,000 tokens to ~800 tokens.
+ *
+ * @see https://github.com/voicetreelab/lazy-mcp
+ */
+import type { ToolDefinition, ToolExecuteResult } from "./registry.js";
+export declare const browseToolsTool: ToolDefinition;
+/**
+ * Set the registry reference for tool execution
+ * Called by the server during initialization
+ */
+export declare function setLazyMcpRegistry(registry: {
+    execute: (name: string, args: unknown) => Promise<ToolExecuteResult>;
+}): void;
+export declare const runToolTool: ToolDefinition;
+export declare const lazyMcpTools: ToolDefinition[];
+/**
+ * Calculate token savings from lazy loading
+ */
+export declare function calculateLazySavings(): {
+    lazyTokens: number;
+    fullTokens: number;
+    savingsPercent: number;
+};
+//# sourceMappingURL=lazy-mcp.d.ts.map

package/dist/tools/lazy-mcp.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lazy-mcp.d.ts","sourceRoot":"","sources":["../../src/tools/lazy-mcp.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAkEvE,eAAO,MAAM,eAAe,EAAE,cAK7B,CAAC;AAyBF;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE;IAC3C,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;CACtE,GAAG,IAAI,CAEP;AAgDD,eAAO,MAAM,WAAW,EAAE,cAKzB,CAAC;AAMF,eAAO,MAAM,YAAY,EAAE,cAAc,EAAmC,CAAC;AAE7E;;GAEG;AACH,wBAAgB,oBAAoB,IAAI;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;CACxB,CAUA"}