distill-mcp 0.6.0-beta

package/dist/utils/toon-serializer.test.js

@@ -0,0 +1,290 @@
+/**
+ * TOON Serializer Tests
+ *
+ * Tests for converting MCP tool definitions to TOON format.
+ */
+import { describe, it, expect } from "vitest";
+import { serializeToolsToToon, serializeToolsToToonTabular, serializeMetadataToToon, serializeMetadataToToonTabular, compareTokens, } from "./toon-serializer.js";
+// Sample tool definitions for testing
+const sampleTools = [
+    {
+        name: "auto_optimize",
+        description: "Auto-compress verbose output (build errors, logs). 80-95% token reduction.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                content: { type: "string" },
+                hint: { enum: ["build", "logs", "errors", "code", "auto"] },
+                aggressive: { type: "boolean" },
+                format: { enum: ["plain", "markdown"] },
+            },
+            required: ["content"],
+        },
+        execute: async () => ({ content: [{ type: "text", text: "" }] }),
+    },
+    {
+        name: "smart_file_read",
+        description: "Read code with AST extraction. Modes: structure, target, query, lines, skeleton.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                filePath: { type: "string" },
+                target: {
+                    properties: {
+                        type: { enum: ["function", "class", "interface", "type", "variable", "method"] },
+                        name: { type: "string" },
+                    },
+                },
+                query: { type: "string" },
+                skeleton: { type: "boolean" },
+            },
+            required: ["filePath"],
+        },
+        execute: async () => ({ content: [{ type: "text", text: "" }] }),
+    },
+    {
+        name: "discover_tools",
+        description: "Find and load optimization tools. Categories: compress, analyze, logs, code, pipeline.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                query: { type: "string" },
+                category: { enum: ["compress", "analyze", "logs", "code", "pipeline"] },
+                load: { type: "boolean" },
+            },
+        },
+        execute: async () => ({ content: [{ type: "text", text: "" }] }),
+    },
+];
+describe("TOON Serializer", () => {
+    describe("serializeToolsToToon", () => {
+        it("should serialize tools to TOON format", () => {
+            const result = serializeToolsToToon(sampleTools);
+            expect(result).toContain("tools[3]:");
+            expect(result).toContain("auto_optimize");
+            expect(result).toContain("smart_file_read");
+            expect(result).toContain("discover_tools");
+        });
+        it("should include parameter information", () => {
+            const result = serializeToolsToToon(sampleTools);
+            // Check for required vs optional params
+            expect(result).toContain("content");
+            expect(result).toContain("hint?");
+            expect(result).toContain("filePath");
+        });
+        it("should include type information for non-string params", () => {
+            const result = serializeToolsToToon(sampleTools);
+            expect(result).toContain(":bool");
+            // Enum types should show values
+            expect(result).toMatch(/build\|logs\|errors/);
+        });
+        it("should group by category when requested", () => {
+            const categories = new Map([
+                ["auto_optimize", "core"],
+                ["smart_file_read", "core"],
+                ["discover_tools", "meta"],
+            ]);
+            const result = serializeToolsToToon(sampleTools, {
+                groupByCategory: true,
+                categories,
+            });
+            expect(result).toContain("core[2]:");
+            expect(result).toContain("meta[1]:");
+        });
+    });
+    describe("serializeToolsToToonTabular", () => {
+        it("should serialize tools to tabular TOON format", () => {
+            const result = serializeToolsToToonTabular(sampleTools);
+            expect(result).toContain("tools[3]{name,params,desc}:");
+            expect(result).toContain("auto_optimize,");
+            expect(result).toContain("smart_file_read,");
+        });
+        it("should use comma-separated values with proper escaping", () => {
+            const result = serializeToolsToToonTabular(sampleTools);
+            const lines = result.split("\n");
+            // Header should define 3 fields
+            expect(lines[0]).toContain("{name,params,desc}");
+            // Tool lines should exist
+            const toolLines = lines.slice(1).filter((l) => l.trim());
+            expect(toolLines.length).toBe(3);
+            // Values with commas should be quoted
+            for (const line of toolLines) {
+                // Line should start with tool name
+                expect(line.trim()).toMatch(/^(auto_optimize|smart_file_read|discover_tools),/);
+            }
+        });
+    });
+    describe("compareTokens", () => {
+        it("should show TOON uses fewer tokens than JSON", () => {
+            const result = compareTokens(sampleTools);
+            expect(result.json).toBeGreaterThan(0);
+            expect(result.toon).toBeGreaterThan(0);
+            expect(result.toonTabular).toBeGreaterThan(0);
+            expect(result.savings).toBeGreaterThan(0);
+            // TOON should use fewer tokens than JSON
+            expect(result.toon).toBeLessThan(result.json);
+            expect(result.toonTabular).toBeLessThan(result.json);
+        });
+        it("should achieve at least 20% token reduction", () => {
+            const result = compareTokens(sampleTools);
+            expect(result.savings).toBeGreaterThanOrEqual(20);
+            console.log(`  Token savings: ${result.savings}% (JSON: ${result.json}, TOON: ${result.toon}, Tabular: ${result.toonTabular})`);
+        });
+    });
+    describe("Edge cases", () => {
+        it("should handle empty tools array", () => {
+            const result = serializeToolsToToon([]);
+            expect(result).toBe("tools[0]:");
+        });
+        it("should handle tool with no parameters", () => {
+            const noParamTool = {
+                name: "simple_tool",
+                description: "A simple tool with no params",
+                inputSchema: { type: "object", properties: {} },
+                execute: async () => ({ content: [{ type: "text", text: "" }] }),
+            };
+            const result = serializeToolsToToon([noParamTool]);
+            expect(result).toContain("simple_tool()");
+        });
+        it("should handle nested object parameters", () => {
+            const result = serializeToolsToToon(sampleTools);
+            // target parameter has nested properties
+            expect(result).toMatch(/target\?:\{type,name\}/);
+        });
+        it("should truncate long descriptions", () => {
+            const longDescTool = {
+                name: "long_desc",
+                description: "This is a very long description that should be truncated to save tokens in the TOON output format because we want to be efficient.",
+                inputSchema: { type: "object", properties: {} },
+                execute: async () => ({ content: [{ type: "text", text: "" }] }),
+            };
+            const result = serializeToolsToToon([longDescTool]);
+            expect(result.length).toBeLessThan(200);
+            expect(result).toContain("...");
+        });
+        it("should handle enum with many values", () => {
+            const manyEnumTool = {
+                name: "many_enum",
+                description: "Tool with many enum values",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        option: { enum: ["a", "b", "c", "d", "e", "f", "g"] },
+                    },
+                },
+                execute: async () => ({ content: [{ type: "text", text: "" }] }),
+            };
+            const result = serializeToolsToToon([manyEnumTool]);
+            // Should truncate to first 3 values + ...
+            expect(result).toContain("a|b|c|...");
+        });
+    });
+});
+describe("TOON Format Compliance", () => {
+    it("should use TOON array syntax [N]", () => {
+        const result = serializeToolsToToon(sampleTools);
+        expect(result).toMatch(/tools\[\d+\]:/);
+    });
+    it("should use TOON tabular header syntax {fields}", () => {
+        const result = serializeToolsToToonTabular(sampleTools);
+        expect(result).toMatch(/tools\[\d+\]\{name,params,desc\}:/);
+    });
+    it("should use indentation for nested content", () => {
+        const result = serializeToolsToToon(sampleTools);
+        const lines = result.split("\n");
+        // Tool entries should be indented
+        const toolLines = lines.filter((l) => l.includes("auto_optimize") || l.includes("smart_file_read"));
+        for (const line of toolLines) {
+            expect(line.startsWith("  ")).toBe(true);
+        }
+    });
+});
+// Sample metadata for lazy loading tests
+const sampleMetadata = [
+    {
+        name: "auto_optimize",
+        category: "core",
+        description: "Auto-detect content type and apply optimal compression",
+    },
+    {
+        name: "smart_file_read",
+        category: "core",
+        description: "Read files with AST-based extraction",
+    },
+    {
+        name: "compress_context",
+        category: "compress",
+        description: "Compress generic text content (logs, configs)",
+    },
+    {
+        name: "summarize_logs",
+        category: "logs",
+        description: "Summarize verbose log output",
+    },
+];
+describe("Lightweight Metadata TOON Serialization", () => {
+    describe("serializeMetadataToToon", () => {
+        it("should serialize metadata without loading tools", () => {
+            const result = serializeMetadataToToon(sampleMetadata, { groupByCategory: false });
+            expect(result).toContain("tools[4]:");
+            expect(result).toContain("auto_optimize");
+            expect(result).toContain("smart_file_read");
+            expect(result).toContain("compress_context");
+        });
+        it("should group by category when requested", () => {
+            const result = serializeMetadataToToon(sampleMetadata, { groupByCategory: true });
+            expect(result).toContain("core[2]:");
+            expect(result).toContain("compress[1]:");
+            expect(result).toContain("logs[1]:");
+        });
+        it("should use arrow notation for descriptions", () => {
+            const result = serializeMetadataToToon(sampleMetadata);
+            expect(result).toContain("→");
+            expect(result).toContain("auto_optimize → Auto-detect");
+        });
+        it("should truncate long descriptions", () => {
+            const longMeta = [
+                {
+                    name: "long_tool",
+                    category: "core",
+                    description: "This is a very long description that should be truncated to save tokens",
+                },
+            ];
+            const result = serializeMetadataToToon(longMeta);
+            expect(result).toContain("...");
+        });
+    });
+    describe("serializeMetadataToToonTabular", () => {
+        it("should serialize metadata to tabular format", () => {
+            const result = serializeMetadataToToonTabular(sampleMetadata);
+            expect(result).toContain("tools[4]{name,desc}:");
+            expect(result).toContain("auto_optimize,");
+        });
+        it("should handle descriptions with commas", () => {
+            const commasMeta = [
+                {
+                    name: "comma_tool",
+                    category: "core",
+                    description: "First, second, third",
+                },
+            ];
+            const result = serializeMetadataToToonTabular(commasMeta);
+            // Should quote the description
+            expect(result).toContain('"First, second, third"');
+        });
+    });
+    describe("Token comparison", () => {
+        it("should be more compact than full TOON with parameters", () => {
+            const metaResult = serializeMetadataToToon(sampleMetadata);
+            const fullResult = serializeToolsToToon(sampleTools);
+            // Metadata-only should be shorter (no parameter info)
+            expect(metaResult.length).toBeLessThan(fullResult.length);
+        });
+        it("should show significant savings for lazy loading", () => {
+            const metaResult = serializeMetadataToToonTabular(sampleMetadata);
+            // Very compact output
+            const lines = metaResult.split("\n");
+            expect(lines.length).toBe(5); // header + 4 tools
+        });
+    });
+});

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "distill-mcp",
+  "version": "0.6.0-beta",
+  "description": "Distill - MCP Server for LLM token optimization and context compression",
+  "type": "module",
+  "bin": {
+    "distill-mcp": "./bin/cli.js"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "bin",
+    "scripts"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "check-types": "tsc --noEmit",
+    "lint": "eslint .",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage"
+  },
+  "dependencies": {
+    "@ctxopt/shared": "workspace:*",
+    "@huggingface/transformers": "^3.8.1",
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "js-tiktoken": "^1.0.15",
+    "tree-sitter-wasms": "^0.1.13",
+    "web-tree-sitter": "0.22.6",
+    "zod": "^3.24.1"
+  },
+  "devDependencies": {
+    "@repo/eslint-config": "workspace:*",
+    "@repo/typescript-config": "workspace:*",
+    "@types/node": "^22.15.3",
+    "@vitest/coverage-v8": "^4.0.16",
+    "eslint": "^9.39.1",
+    "typescript": "5.9.2",
+    "vitest": "^4.0.16"
+  },
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "claude",
+    "anthropic",
+    "token-optimization",
+    "context-engineering"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ArthurDEV44/ctxopt.git",
+    "directory": "packages/mcp-server"
+  },
+  "license": "MIT"
+}

package/scripts/install.ps1

@@ -0,0 +1,133 @@
+# CtxOpt MCP Server Installation Script for Windows
+# https://ctxopt.dev
+#
+# Usage:
+#   irm https://ctxopt.dev/install.ps1 | iex
+#
+# This script will:
+#   1. Detect your package manager
+#   2. Install @ctxopt/mcp-server globally
+#   3. Auto-configure detected IDEs (Claude Code, Cursor, Windsurf)
+#   4. Verify the installation
+
+$ErrorActionPreference = "Stop"
+
+# Colors and formatting
+function Write-Info { Write-Host "i " -ForegroundColor Blue -NoNewline; Write-Host $args }
+function Write-Success { Write-Host "√ " -ForegroundColor Green -NoNewline; Write-Host $args }
+function Write-Warning { Write-Host "! " -ForegroundColor Yellow -NoNewline; Write-Host $args }
+function Write-Error { Write-Host "x " -ForegroundColor Red -NoNewline; Write-Host $args }
+
+function Get-PackageManager {
+    if (Get-Command bun -ErrorAction SilentlyContinue) { return "bun" }
+    if (Get-Command npm -ErrorAction SilentlyContinue) { return "npm" }
+    if (Get-Command yarn -ErrorAction SilentlyContinue) { return "yarn" }
+    if (Get-Command pnpm -ErrorAction SilentlyContinue) { return "pnpm" }
+    return "none"
+}
+
+function Test-NodeVersion {
+    if (-not (Get-Command node -ErrorAction SilentlyContinue)) {
+        Write-Error "Node.js is not installed."
+        Write-Host ""
+        Write-Host "Please install Node.js 18 or higher:"
+        Write-Host "  https://nodejs.org/"
+        Write-Host ""
+        exit 1
+    }
+
+    $nodeVersion = (node -v).TrimStart('v').Split('.')[0]
+    if ([int]$nodeVersion -lt 18) {
+        Write-Error "Node.js version must be 18 or higher (found v$nodeVersion)."
+        Write-Host ""
+        Write-Host "Please upgrade Node.js:"
+        Write-Host "  https://nodejs.org/"
+        Write-Host ""
+        exit 1
+    }
+}
+
+function Install-Package {
+    param([string]$PackageManager)
+
+    Write-Info "Installing @ctxopt/mcp-server using $PackageManager..."
+
+    switch ($PackageManager) {
+        "bun" { bun install -g @ctxopt/mcp-server }
+        "npm" { npm install -g @ctxopt/mcp-server }
+        "yarn" { yarn global add @ctxopt/mcp-server }
+        "pnpm" { pnpm add -g @ctxopt/mcp-server }
+        default {
+            Write-Error "No supported package manager found."
+            Write-Host ""
+            Write-Host "Please install one of the following:"
+            Write-Host "  - npm (comes with Node.js)"
+            Write-Host "  - bun: https://bun.sh"
+            Write-Host "  - yarn: npm install -g yarn"
+            Write-Host "  - pnpm: npm install -g pnpm"
+            Write-Host ""
+            exit 1
+        }
+    }
+}
+
+function Main {
+    Write-Host ""
+    Write-Host "================================================" -ForegroundColor Cyan
+    Write-Host "     CtxOpt MCP Server Installation" -ForegroundColor Cyan
+    Write-Host "================================================" -ForegroundColor Cyan
+    Write-Host ""
+
+    # Detect environment
+    $pm = Get-PackageManager
+    Write-Info "Detected OS: Windows"
+    Write-Info "Detected package manager: $pm"
+    Write-Host ""
+
+    # Check Node.js
+    Test-NodeVersion
+    $nodeVer = node -v
+    Write-Success "Node.js $nodeVer detected"
+
+    # Install package
+    Write-Host ""
+    Install-Package -PackageManager $pm
+    Write-Success "Package installed successfully"
+
+    # Verify installation
+    Write-Host ""
+    if (Get-Command ctxopt-mcp -ErrorAction SilentlyContinue) {
+        $version = ctxopt-mcp --version 2>$null
+        if (-not $version) { $version = "unknown" }
+        Write-Success "ctxopt-mcp v$version is now available"
+    } else {
+        Write-Warning "ctxopt-mcp not found in PATH. You may need to restart your terminal."
+    }
+
+    # Run setup
+    Write-Host ""
+    Write-Info "Configuring IDEs..."
+    Write-Host ""
+
+    if (Get-Command ctxopt-mcp -ErrorAction SilentlyContinue) {
+        ctxopt-mcp setup
+    } else {
+        # Fallback to npx if global install didn't add to PATH yet
+        npx @ctxopt/mcp-server setup
+    }
+
+    # Final message
+    Write-Host ""
+    Write-Host "Installation complete!" -ForegroundColor Green
+    Write-Host ""
+    Write-Host "Next steps:"
+    Write-Host "  1. Restart your IDE to load the MCP server"
+    Write-Host "  2. Run 'ctxopt-mcp doctor' to verify everything is working"
+    Write-Host ""
+    Write-Host "Documentation: " -NoNewline
+    Write-Host "https://ctxopt.dev/docs" -ForegroundColor Cyan
+    Write-Host ""
+}
+
+# Run main function
+Main

package/scripts/install.sh

@@ -0,0 +1,183 @@
+#!/bin/bash
+
+# CtxOpt MCP Server Installation Script
+# https://ctxopt.dev
+#
+# Usage:
+#   curl -fsSL https://ctxopt.dev/install.sh | bash
+#
+# This script will:
+#   1. Detect your OS and package manager
+#   2. Install @ctxopt/mcp-server globally
+#   3. Auto-configure detected IDEs (Claude Code, Cursor, Windsurf)
+#   4. Verify the installation
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+BOLD='\033[1m'
+NC='\033[0m' # No Color
+
+# Logging functions
+log() {
+    echo -e "${BLUE}ℹ${NC} $1"
+}
+
+success() {
+    echo -e "${GREEN}✓${NC} $1"
+}
+
+warn() {
+    echo -e "${YELLOW}⚠${NC} $1"
+}
+
+error() {
+    echo -e "${RED}✗${NC} $1"
+}
+
+# Detect OS
+detect_os() {
+    case "$(uname -s)" in
+        Linux*)     OS="linux";;
+        Darwin*)    OS="macos";;
+        CYGWIN*|MINGW*|MSYS*) OS="windows";;
+        *)          OS="unknown";;
+    esac
+    echo "$OS"
+}
+
+# Detect package manager
+detect_package_manager() {
+    if command -v bun &> /dev/null; then
+        echo "bun"
+    elif command -v npm &> /dev/null; then
+        echo "npm"
+    elif command -v yarn &> /dev/null; then
+        echo "yarn"
+    elif command -v pnpm &> /dev/null; then
+        echo "pnpm"
+    else
+        echo "none"
+    fi
+}
+
+# Check Node.js version
+check_node() {
+    if ! command -v node &> /dev/null; then
+        error "Node.js is not installed."
+        echo ""
+        echo "Please install Node.js 18 or higher:"
+        echo "  https://nodejs.org/"
+        echo ""
+        exit 1
+    fi
+
+    NODE_VERSION=$(node -v | cut -d'v' -f2 | cut -d'.' -f1)
+    if [ "$NODE_VERSION" -lt 18 ]; then
+        error "Node.js version must be 18 or higher (found v$NODE_VERSION)."
+        echo ""
+        echo "Please upgrade Node.js:"
+        echo "  https://nodejs.org/"
+        echo ""
+        exit 1
+    fi
+}
+
+# Install package
+install_package() {
+    local pm=$1
+
+    log "Installing @ctxopt/mcp-server using $pm..."
+
+    case "$pm" in
+        bun)
+            bun install -g @ctxopt/mcp-server
+            ;;
+        npm)
+            npm install -g @ctxopt/mcp-server
+            ;;
+        yarn)
+            yarn global add @ctxopt/mcp-server
+            ;;
+        pnpm)
+            pnpm add -g @ctxopt/mcp-server
+            ;;
+        *)
+            error "No supported package manager found."
+            echo ""
+            echo "Please install one of the following:"
+            echo "  • npm (comes with Node.js)"
+            echo "  • bun: https://bun.sh"
+            echo "  • yarn: npm install -g yarn"
+            echo "  • pnpm: npm install -g pnpm"
+            echo ""
+            exit 1
+            ;;
+    esac
+}
+
+# Main installation flow
+main() {
+    echo ""
+    echo -e "${BOLD}${CYAN}╔══════════════════════════════════════════╗${NC}"
+    echo -e "${BOLD}${CYAN}║     CtxOpt MCP Server Installation       ║${NC}"
+    echo -e "${BOLD}${CYAN}╚══════════════════════════════════════════╝${NC}"
+    echo ""
+
+    # Detect environment
+    OS=$(detect_os)
+    PM=$(detect_package_manager)
+
+    log "Detected OS: $OS"
+    log "Detected package manager: $PM"
+    echo ""
+
+    # Check Node.js
+    check_node
+    success "Node.js $(node -v) detected"
+
+    # Install package
+    echo ""
+    install_package "$PM"
+    success "Package installed successfully"
+
+    # Verify installation
+    echo ""
+    if command -v ctxopt-mcp &> /dev/null; then
+        VERSION=$(ctxopt-mcp --version 2>/dev/null || echo "unknown")
+        success "ctxopt-mcp v$VERSION is now available"
+    else
+        warn "ctxopt-mcp not found in PATH. You may need to restart your terminal."
+    fi
+
+    # Run setup
+    echo ""
+    log "Configuring IDEs..."
+    echo ""
+
+    if command -v ctxopt-mcp &> /dev/null; then
+        ctxopt-mcp setup
+    else
+        # Fallback to npx if global install didn't add to PATH yet
+        npx @ctxopt/mcp-server setup
+    fi
+
+    # Final message
+    echo ""
+    echo -e "${BOLD}${GREEN}Installation complete!${NC}"
+    echo ""
+    echo "Next steps:"
+    echo "  1. Restart your IDE to load the MCP server"
+    echo "  2. Run 'ctxopt-mcp doctor' to verify everything is working"
+    echo ""
+    echo -e "Documentation: ${CYAN}https://ctxopt.dev/docs${NC}"
+    echo ""
+}
+
+# Run main function
+main "$@"