distill-mcp 0.6.0-beta

package/dist/sandbox/sdk/multifile.js

@@ -0,0 +1,130 @@
+/**
+ * SDK Multi-File Functions
+ *
+ * Multi-file compression and analysis for sandbox use.
+ * Provides cross-file deduplication, skeleton extraction, and chunking.
+ */
+import { detectLanguageFromPath } from "../../utils/language-detector.js";
+import { compressMultiFile, extractSharedElements, createChunks, } from "../../compressors/multifile.js";
+/**
+ * Create multi-file API with host callbacks
+ */
+export function createMultifileAPI(workingDir, callbacks) {
+    /**
+     * Load files from patterns into FileContext array
+     */
+    function loadFiles(patterns) {
+        const files = [];
+        const seen = new Set();
+        for (const pattern of patterns) {
+            const paths = callbacks.glob(pattern);
+            for (const filePath of paths) {
+                if (seen.has(filePath))
+                    continue;
+                seen.add(filePath);
+                try {
+                    const content = callbacks.readFile(filePath);
+                    const language = detectLanguageFromPath(filePath);
+                    files.push({
+                        path: filePath,
+                        content,
+                        language,
+                    });
+                }
+                catch {
+                    // Skip files that can't be read
+                }
+            }
+        }
+        return files;
+    }
+    return {
+        /**
+         * Compress multiple files with cross-file deduplication
+         *
+         * @param patterns - Glob patterns for files to include
+         * @param options - Compression options
+         * @returns Compressed result with shared elements and stats
+         */
+        compress(patterns, options) {
+            const files = loadFiles(patterns);
+            if (files.length === 0) {
+                return {
+                    compressed: "// No files found matching patterns",
+                    filesIncluded: [],
+                    sharedElements: { imports: [], types: [], constants: [] },
+                    stats: {
+                        originalTokens: 0,
+                        compressedTokens: 0,
+                        filesProcessed: 0,
+                        deduplicatedItems: 0,
+                        reductionPercent: 0,
+                    },
+                };
+            }
+            return compressMultiFile(files, {
+                maxTokens: options?.maxTokens || 50000,
+                strategy: options?.strategy || "deduplicate",
+                preservePatterns: options?.preservePatterns,
+            });
+        },
+        /**
+         * Extract shared elements (imports, types) across files
+         *
+         * @param patterns - Glob patterns for files to analyze
+         * @returns Shared imports, types, and constants
+         */
+        extractShared(patterns) {
+            const files = loadFiles(patterns);
+            return extractSharedElements(files);
+        },
+        /**
+         * Create chunks from files for incremental processing
+         *
+         * @param patterns - Glob patterns for files to chunk
+         * @param maxTokensPerChunk - Maximum tokens per chunk
+         * @returns Array of chunk information
+         */
+        chunk(patterns, maxTokensPerChunk) {
+            const files = loadFiles(patterns);
+            return createChunks(files, maxTokensPerChunk);
+        },
+        /**
+         * Get dependency-aware skeletons for files
+         *
+         * @param patterns - Glob patterns for entry point files
+         * @param depth - Optional depth limit for dependency traversal
+         * @returns Compressed skeleton output
+         */
+        skeletons(patterns, depth) {
+            const files = loadFiles(patterns);
+            if (files.length === 0) {
+                return "// No files found matching patterns";
+            }
+            const result = compressMultiFile(files, {
+                strategy: "skeleton",
+                dependencyDepth: depth || 2,
+            });
+            return result.compressed;
+        },
+        /**
+         * Read multiple files into a combined context
+         *
+         * @param patterns - Glob patterns for files to read
+         * @returns Combined file contents with headers
+         */
+        readAll(patterns) {
+            const files = loadFiles(patterns);
+            if (files.length === 0) {
+                return "// No files found matching patterns";
+            }
+            const parts = [];
+            for (const file of files) {
+                parts.push(`// === ${file.path} ===`);
+                parts.push(file.content);
+                parts.push("");
+            }
+            return parts.join("\n");
+        },
+    };
+}

package/dist/sandbox/sdk/pipeline.d.ts

@@ -0,0 +1,16 @@
+/**
+ * SDK Pipeline Functions
+ *
+ * Composable pipelines for chaining operations on files and data.
+ * Provides declarative data processing with built-in templates.
+ */
+import type { PipelineStep, PipelineResult, CodebaseOverview, SymbolUsage, DependencyAnalysis, HostCallbacks } from "../types.js";
+/**
+ * Create Pipeline API for sandbox
+ */
+export declare function createPipelineAPI(workingDir: string, callbacks: HostCallbacks): ((steps: PipelineStep[]) => PipelineResult) & {
+    codebaseOverview: (dir?: string) => CodebaseOverview;
+    findUsages: (symbol: string, glob?: string) => SymbolUsage;
+    analyzeDeps: (file: string, depth?: number) => DependencyAnalysis;
+};
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/sandbox/sdk/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/sandbox/sdk/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EACV,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,kBAAkB,EAElB,aAAa,EACd,MAAM,aAAa,CAAC;AAsIrB;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,YAI5C,YAAY,EAAE,KAAG,cAAc;sBAiH3C,CAAC,GAAG,CAAC,EAAE,MAAM,KAAK,gBAAgB;gBACxC,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,KAAK,WAAW;iBAC7C,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,kBAAkB;EA4PpE"}

package/dist/sandbox/sdk/pipeline.js

@@ -0,0 +1,454 @@
+/**
+ * SDK Pipeline Functions
+ *
+ * Composable pipelines for chaining operations on files and data.
+ * Provides declarative data processing with built-in templates.
+ */
+import * as fs from "fs";
+import * as path from "path";
+import { compressAuto, compressSemantic, compressLogs } from "./compress.js";
+import { detectLanguageFromPath } from "../../utils/language-detector.js";
+import { validateGlobPattern } from "../security/path-validator.js";
+const MAX_ITEMS = 1000;
+const CACHE_TTL_GLOB = 2 * 60 * 1000; // 2 minutes for glob results
+const CACHE_TTL_TEMPLATE = 5 * 60 * 1000; // 5 minutes for template methods
+const pipelineCache = new Map();
+function getCached(key) {
+    const entry = pipelineCache.get(key);
+    if (!entry)
+        return null;
+    if (Date.now() > entry.expiresAt) {
+        pipelineCache.delete(key);
+        return null;
+    }
+    return entry.value;
+}
+function setCached(key, value, ttl) {
+    // Limit cache size to prevent memory issues
+    if (pipelineCache.size > 100) {
+        // Evict oldest entries
+        const now = Date.now();
+        for (const [k, v] of pipelineCache.entries()) {
+            if (v.expiresAt < now || pipelineCache.size > 50) {
+                pipelineCache.delete(k);
+            }
+        }
+    }
+    pipelineCache.set(key, { value, expiresAt: Date.now() + ttl });
+}
+const MAX_FILE_SIZE = 1024 * 1024; // 1MB
+/**
+ * Simple glob pattern matcher with brace expansion support
+ */
+function matchGlob(filepath, pattern) {
+    // Handle brace expansion: {a,b,c} -> (a|b|c)
+    let regexPattern = pattern
+        .replace(/\{([^}]+)\}/g, (_match, group) => {
+        const alternatives = group.split(",").map((s) => s.trim());
+        return `(${alternatives.join("|")})`;
+    });
+    // Use placeholders to avoid order-dependent replacement issues
+    regexPattern = regexPattern
+        .replace(/\*\*\//g, "{{GLOBSTAR_SLASH}}") // **/ -> matches zero or more dirs
+        .replace(/\*\*/g, "{{GLOBSTAR}}") // ** -> matches anything
+        .replace(/\*/g, "{{STAR}}") // * -> matches within segment
+        .replace(/\?/g, "{{QUESTION}}") // ? -> matches single char
+        .replace(/\./g, "\\.") // escape dots
+        .replace(/{{GLOBSTAR_SLASH}}/g, "(.*/)?") // **/ can match empty or paths
+        .replace(/{{GLOBSTAR}}/g, ".*") // ** matches anything
+        .replace(/{{STAR}}/g, "[^/]*") // * matches non-slash chars
+        .replace(/{{QUESTION}}/g, "."); // ? matches any single char
+    const regex = new RegExp(`^${regexPattern}$`);
+    return regex.test(filepath);
+}
+/**
+ * Walk directory and find files matching pattern (with caching)
+ */
+function walkDirectory(dir, pattern, maxFiles = MAX_ITEMS) {
+    const cacheKey = `pipeline:glob:${dir}:${pattern}:${maxFiles}`;
+    // Check cache first
+    const cached = getCached(cacheKey);
+    if (cached) {
+        return cached;
+    }
+    const results = [];
+    function walk(currentDir, relativePath = "") {
+        if (results.length >= maxFiles)
+            return;
+        try {
+            const entries = fs.readdirSync(currentDir, { withFileTypes: true });
+            for (const entry of entries) {
+                if (results.length >= maxFiles)
+                    break;
+                const fullPath = path.join(currentDir, entry.name);
+                const relPath = path.join(relativePath, entry.name);
+                if (entry.name.startsWith(".") || entry.name === "node_modules") {
+                    continue;
+                }
+                if (entry.isDirectory()) {
+                    walk(fullPath, relPath);
+                }
+                else if (entry.isFile()) {
+                    if (matchGlob(relPath, pattern)) {
+                        results.push(relPath);
+                    }
+                }
+            }
+        }
+        catch {
+            // Skip directories we can't read
+        }
+    }
+    walk(dir);
+    // Cache the results
+    setCached(cacheKey, results, CACHE_TTL_GLOB);
+    return results;
+}
+/**
+ * Count lines in content
+ */
+function countLines(content) {
+    return content.split("\n").length;
+}
+/**
+ * Create Pipeline API for sandbox
+ */
+export function createPipelineAPI(workingDir, callbacks) {
+    /**
+     * Execute a pipeline of steps
+     */
+    function executePipeline(steps) {
+        const startTime = Date.now();
+        let data = [];
+        let stepsExecuted = 0;
+        let itemsProcessed = 0;
+        for (const step of steps) {
+            stepsExecuted++;
+            if ("glob" in step) {
+                // Glob step - find files
+                const validation = validateGlobPattern(step.glob, workingDir);
+                if (!validation.safe) {
+                    throw new Error(validation.error ?? "Invalid glob pattern");
+                }
+                data = walkDirectory(workingDir, step.glob);
+                itemsProcessed = data.length;
+            }
+            else if ("filter" in step) {
+                // Filter step
+                data = data.filter(step.filter);
+                itemsProcessed = data.length;
+            }
+            else if ("read" in step && step.read) {
+                // Read step - read file contents
+                data = data.map((file) => {
+                    if (typeof file !== "string")
+                        return file;
+                    try {
+                        const fullPath = path.join(workingDir, file);
+                        const stats = fs.statSync(fullPath);
+                        if (stats.size > MAX_FILE_SIZE) {
+                            return { file, content: null, error: "File too large" };
+                        }
+                        const content = fs.readFileSync(fullPath, "utf-8");
+                        return { file, content };
+                    }
+                    catch (err) {
+                        return { file, content: null, error: String(err) };
+                    }
+                });
+            }
+            else if ("map" in step) {
+                // Map step
+                data = data.map(step.map);
+            }
+            else if ("reduce" in step) {
+                // Reduce step
+                const result = data.reduce(step.reduce, step.initial);
+                data = [result];
+            }
+            else if ("compress" in step) {
+                // Compress step
+                const content = JSON.stringify(data, null, 2);
+                let compressed;
+                if (step.compress === "semantic") {
+                    const result = compressSemantic(content, step.ratio);
+                    compressed = result.compressed;
+                }
+                else if (step.compress === "logs") {
+                    const result = compressLogs(content);
+                    compressed = result.summary;
+                }
+                else {
+                    const result = compressAuto(content);
+                    compressed = result.compressed;
+                }
+                data = [compressed];
+            }
+            else if ("limit" in step) {
+                // Limit step
+                data = data.slice(0, step.limit);
+                itemsProcessed = data.length;
+            }
+            else if ("sort" in step) {
+                // Sort step
+                const direction = step.sort === "desc" ? -1 : 1;
+                const key = step.by;
+                data = [...data].sort((a, b) => {
+                    let valA = key ? a[key] : a;
+                    let valB = key ? b[key] : b;
+                    if (typeof valA === "string" && typeof valB === "string") {
+                        return valA.localeCompare(valB) * direction;
+                    }
+                    if (typeof valA === "number" && typeof valB === "number") {
+                        return (valA - valB) * direction;
+                    }
+                    return 0;
+                });
+            }
+            else if ("unique" in step) {
+                // Unique step
+                if (typeof step.unique === "string") {
+                    const key = step.unique;
+                    const seen = new Set();
+                    data = data.filter((item) => {
+                        const val = item[key];
+                        if (seen.has(val))
+                            return false;
+                        seen.add(val);
+                        return true;
+                    });
+                }
+                else if (step.unique) {
+                    data = [...new Set(data.map((d) => JSON.stringify(d)))].map((s) => JSON.parse(s));
+                }
+            }
+        }
+        return {
+            data,
+            stats: {
+                stepsExecuted,
+                itemsProcessed,
+                executionTimeMs: Date.now() - startTime,
+            },
+        };
+    }
+    // Add template methods to the pipeline function
+    const pipeline = executePipeline;
+    /**
+     * Get codebase overview with statistics (cached)
+     */
+    pipeline.codebaseOverview = (dir) => {
+        const targetDir = dir ?? ".";
+        const fullPath = path.join(workingDir, targetDir);
+        const cacheKey = `pipeline:overview:${fullPath}`;
+        // Check cache first
+        const cached = getCached(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const files = walkDirectory(fullPath, "**/*.{ts,tsx,js,jsx,py,go,rs,php,swift}");
+        const languages = {};
+        const fileSizes = [];
+        let totalLines = 0;
+        for (const file of files) {
+            try {
+                const filePath = path.join(fullPath, file);
+                const content = fs.readFileSync(filePath, "utf-8");
+                const lines = countLines(content);
+                const lang = detectLanguageFromPath(file);
+                totalLines += lines;
+                if (lang !== "unknown") {
+                    languages[lang] = (languages[lang] ?? 0) + 1;
+                }
+                fileSizes.push({ path: file, lines });
+            }
+            catch {
+                // Skip files we can't read
+            }
+        }
+        // Sort by lines descending
+        fileSizes.sort((a, b) => b.lines - a.lines);
+        // Build structure (simplified)
+        const structure = {
+            path: targetDir,
+            type: "directory",
+            name: path.basename(targetDir) || "root",
+            children: files.slice(0, 20).map((f) => ({
+                path: f,
+                type: "file",
+                name: path.basename(f),
+            })),
+        };
+        const result = {
+            totalFiles: files.length,
+            totalLines,
+            languages,
+            largestFiles: fileSizes.slice(0, 10),
+            structure,
+        };
+        // Cache the result
+        setCached(cacheKey, result, CACHE_TTL_TEMPLATE);
+        return result;
+    };
+    /**
+     * Find all usages of a symbol (cached)
+     */
+    pipeline.findUsages = (symbol, glob) => {
+        const pattern = glob ?? "**/*.{ts,tsx,js,jsx,py,go,rs}";
+        const cacheKey = `pipeline:usages:${symbol}:${pattern}`;
+        // Check cache first
+        const cached = getCached(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const files = walkDirectory(workingDir, pattern);
+        const definitions = [];
+        const usages = [];
+        // Definition patterns
+        const defPatterns = [
+            new RegExp(`(?:function|class|const|let|var|interface|type)\\s+${escapeRegex(symbol)}\\b`),
+            new RegExp(`${escapeRegex(symbol)}\\s*[=:]\\s*(?:function|\\(|async)`),
+        ];
+        // Usage pattern
+        const usagePattern = new RegExp(`\\b${escapeRegex(symbol)}\\b`, "g");
+        for (const file of files) {
+            try {
+                const filePath = path.join(workingDir, file);
+                const content = fs.readFileSync(filePath, "utf-8");
+                const lines = content.split("\n");
+                for (let i = 0; i < lines.length; i++) {
+                    const line = lines[i] ?? "";
+                    const lineNum = i + 1;
+                    // Check for definitions
+                    let isDef = false;
+                    for (const defPattern of defPatterns) {
+                        if (defPattern.test(line)) {
+                            definitions.push({ file, line: lineNum });
+                            isDef = true;
+                            break;
+                        }
+                    }
+                    // Check for usages (excluding definitions)
+                    if (!isDef && usagePattern.test(line)) {
+                        usages.push({
+                            file,
+                            line: lineNum,
+                            context: line.trim().slice(0, 100),
+                        });
+                    }
+                    // Reset regex lastIndex
+                    usagePattern.lastIndex = 0;
+                }
+            }
+            catch {
+                // Skip files we can't read
+            }
+        }
+        const result = {
+            symbol,
+            definitions,
+            usages,
+            totalReferences: definitions.length + usages.length,
+        };
+        // Cache the result
+        setCached(cacheKey, result, CACHE_TTL_TEMPLATE);
+        return result;
+    };
+    /**
+     * Analyze dependencies transitively (cached)
+     */
+    pipeline.analyzeDeps = (file, depth) => {
+        const maxDepth = Math.min(depth ?? 3, 5);
+        const cacheKey = `pipeline:deps:${file}:${maxDepth}`;
+        // Check cache first
+        const cached = getCached(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const directDeps = [];
+        const transitiveDeps = [];
+        const externalPackages = [];
+        const circularDeps = [];
+        const visited = new Set();
+        function analyzeFile(filePath, currentDepth) {
+            if (currentDepth > maxDepth || visited.has(filePath)) {
+                if (visited.has(filePath) && currentDepth > 0) {
+                    circularDeps.push(filePath);
+                }
+                return;
+            }
+            visited.add(filePath);
+            try {
+                const fullPath = path.join(workingDir, filePath);
+                const content = fs.readFileSync(fullPath, "utf-8");
+                // Parse imports
+                const importRegex = /import\s+(?:.*?\s+from\s+)?['"]([^'"]+)['"]/g;
+                let match;
+                while ((match = importRegex.exec(content)) !== null) {
+                    const importSource = match[1] ?? "";
+                    if (importSource.startsWith(".")) {
+                        // Relative import - resolve path
+                        const currentDir = path.dirname(fullPath);
+                        let resolved = path.resolve(currentDir, importSource);
+                        // Try with extensions
+                        const extensions = [".ts", ".tsx", ".js", ".jsx", ""];
+                        for (const ext of extensions) {
+                            const withExt = resolved + ext;
+                            if (fs.existsSync(withExt)) {
+                                resolved = withExt;
+                                break;
+                            }
+                            // Try index file
+                            const indexPath = path.join(resolved, `index${ext || ".ts"}`);
+                            if (fs.existsSync(indexPath)) {
+                                resolved = indexPath;
+                                break;
+                            }
+                        }
+                        const relativePath = path.relative(workingDir, resolved);
+                        if (currentDepth === 0) {
+                            if (!directDeps.includes(relativePath)) {
+                                directDeps.push(relativePath);
+                            }
+                        }
+                        else {
+                            if (!transitiveDeps.includes(relativePath) && !directDeps.includes(relativePath)) {
+                                transitiveDeps.push(relativePath);
+                            }
+                        }
+                        // Recurse
+                        analyzeFile(relativePath, currentDepth + 1);
+                    }
+                    else {
+                        // External package
+                        const pkgName = importSource.split("/")[0] ?? importSource;
+                        if (!externalPackages.includes(pkgName)) {
+                            externalPackages.push(pkgName);
+                        }
+                    }
+                }
+            }
+            catch {
+                // Skip files we can't read
+            }
+        }
+        analyzeFile(file, 0);
+        const result = {
+            file,
+            directDeps,
+            transitiveDeps,
+            externalPackages,
+            circularDeps: [...new Set(circularDeps)],
+        };
+        // Cache the result
+        setCached(cacheKey, result, CACHE_TTL_TEMPLATE);
+        return result;
+    };
+    return pipeline;
+}
+/**
+ * Escape special regex characters
+ */
+function escapeRegex(str) {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}

package/dist/sandbox/sdk/pipeline.test.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Pipeline SDK Tests
+ *
+ * Tests for ctx.pipeline.* functions.
+ */
+export {};
+//# sourceMappingURL=pipeline.test.d.ts.map

package/dist/sandbox/sdk/pipeline.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.test.d.ts","sourceRoot":"","sources":["../../../src/sandbox/sdk/pipeline.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}