distill-mcp 0.6.0-beta

package/dist/compressors/diff.js

@@ -0,0 +1,389 @@
+/**
+ * Diff Compressor
+ *
+ * Compresses git diff output to reduce tokens while preserving essential changes.
+ * Supports three strategies: hunks-only, summary, and semantic.
+ */
+import { countTokens } from "../utils/token-counter.js";
+import { calculateTFIDF, getSegmentTFIDFScore } from "../utils/tfidf.js";
+// =============================================================================
+// Diff Parser
+// =============================================================================
+/**
+ * Parse unified diff format into structured data
+ *
+ * Handles:
+ * - Standard unified diff (git diff, diff -u)
+ * - File headers: diff --git a/path b/path
+ * - Index lines: index abc123..def456 100644
+ * - --- / +++ file markers
+ * - @@ hunk headers
+ * - Context lines (space prefix)
+ * - Addition lines (+ prefix)
+ * - Deletion lines (- prefix)
+ * - Binary file indicators
+ * - Renamed files (similarity index)
+ */
+export function parseDiff(diff) {
+    const files = [];
+    const lines = diff.split("\n");
+    let currentFile = null;
+    let currentHunk = null;
+    let hunkLines = [];
+    for (const line of lines) {
+        // Detect file header: diff --git a/path b/path
+        if (line.startsWith("diff --git ")) {
+            // Save previous file
+            if (currentFile) {
+                if (currentHunk) {
+                    currentHunk.content = hunkLines.join("\n");
+                    currentFile.hunks.push(currentHunk);
+                }
+                files.push(currentFile);
+            }
+            // Parse paths from header
+            const match = line.match(/diff --git a\/(.+) b\/(.+)/);
+            currentFile = {
+                oldPath: match?.[1] ?? null,
+                newPath: match?.[2] ?? null,
+                status: "modified",
+                isBinary: false,
+                hunks: [],
+                additions: 0,
+                deletions: 0,
+            };
+            currentHunk = null;
+            hunkLines = [];
+            continue;
+        }
+        // Detect new file mode
+        if (line.startsWith("new file mode")) {
+            if (currentFile)
+                currentFile.status = "added";
+            continue;
+        }
+        // Detect deleted file mode
+        if (line.startsWith("deleted file mode")) {
+            if (currentFile)
+                currentFile.status = "deleted";
+            continue;
+        }
+        // Detect rename
+        if (line.startsWith("similarity index") || line.startsWith("rename from")) {
+            if (currentFile)
+                currentFile.status = "renamed";
+            continue;
+        }
+        // Detect binary file
+        if (line.startsWith("Binary files")) {
+            if (currentFile)
+                currentFile.isBinary = true;
+            continue;
+        }
+        // Detect hunk header: @@ -oldStart,oldCount +newStart,newCount @@
+        if (line.startsWith("@@")) {
+            // Save previous hunk
+            if (currentHunk && currentFile) {
+                currentHunk.content = hunkLines.join("\n");
+                currentFile.hunks.push(currentHunk);
+            }
+            const match = line.match(/@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/);
+            currentHunk = {
+                oldStart: parseInt(match?.[1] ?? "0"),
+                oldCount: parseInt(match?.[2] ?? "1"),
+                newStart: parseInt(match?.[3] ?? "0"),
+                newCount: parseInt(match?.[4] ?? "1"),
+                content: "",
+                additions: 0,
+                deletions: 0,
+            };
+            hunkLines = [line]; // Include @@ header
+            continue;
+        }
+        // Hunk content lines
+        if (currentHunk) {
+            hunkLines.push(line);
+            if (line.startsWith("+") && !line.startsWith("+++")) {
+                currentHunk.additions++;
+                if (currentFile)
+                    currentFile.additions++;
+            }
+            else if (line.startsWith("-") && !line.startsWith("---")) {
+                currentHunk.deletions++;
+                if (currentFile)
+                    currentFile.deletions++;
+            }
+        }
+    }
+    // Save final file/hunk
+    if (currentFile) {
+        if (currentHunk) {
+            currentHunk.content = hunkLines.join("\n");
+            currentFile.hunks.push(currentHunk);
+        }
+        files.push(currentFile);
+    }
+    return {
+        files,
+        totalAdditions: files.reduce((sum, f) => sum + f.additions, 0),
+        totalDeletions: files.reduce((sum, f) => sum + f.deletions, 0),
+    };
+}
+// =============================================================================
+// Compression Strategies
+// =============================================================================
+/**
+ * Strategy 1: hunks-only
+ * Keep only changed lines with minimal context
+ */
+function compressHunksOnly(parsed, contextLines = 1) {
+    const output = [];
+    for (const file of parsed.files) {
+        // File header
+        const path = file.newPath ?? file.oldPath ?? "unknown";
+        const statusIcon = {
+            modified: "M",
+            added: "A",
+            deleted: "D",
+            renamed: "R",
+        }[file.status];
+        output.push(`${statusIcon} ${path}`);
+        if (file.isBinary) {
+            output.push("  (binary file)");
+            continue;
+        }
+        // Process hunks
+        for (const hunk of file.hunks) {
+            // Simplified hunk header
+            output.push(`@@ -${hunk.oldStart},${hunk.oldCount} +${hunk.newStart},${hunk.newCount} @@`);
+            // Filter to only changed lines + minimal context
+            const hunkContentLines = hunk.content.split("\n").slice(1); // Skip @@ header
+            const relevantLines = [];
+            const changeIndices = [];
+            // Find all change indices
+            hunkContentLines.forEach((line, idx) => {
+                if (line.startsWith("+") || line.startsWith("-")) {
+                    changeIndices.push(idx);
+                }
+            });
+            // Include lines near changes
+            hunkContentLines.forEach((line, i) => {
+                if (line.startsWith("+") || line.startsWith("-")) {
+                    relevantLines.push(line);
+                }
+                else if (line.startsWith(" ")) {
+                    // Context line - check if near a change
+                    const nearChange = changeIndices.some((idx) => Math.abs(idx - i) <= contextLines);
+                    if (nearChange) {
+                        relevantLines.push(line);
+                    }
+                }
+            });
+            output.push(...relevantLines);
+        }
+        output.push(""); // Blank line between files
+    }
+    return output.join("\n").trim();
+}
+/**
+ * Strategy 2: summary
+ * Generate human-readable summary without diff content
+ */
+function compressSummary(parsed) {
+    const output = [];
+    // Overall summary
+    output.push("## Diff Summary");
+    output.push(`- Files changed: ${parsed.files.length}`);
+    output.push(`- Additions: +${parsed.totalAdditions}`);
+    output.push(`- Deletions: -${parsed.totalDeletions}`);
+    output.push("");
+    // Group by status
+    const byStatus = {
+        added: parsed.files.filter((f) => f.status === "added"),
+        modified: parsed.files.filter((f) => f.status === "modified"),
+        deleted: parsed.files.filter((f) => f.status === "deleted"),
+        renamed: parsed.files.filter((f) => f.status === "renamed"),
+    };
+    // List new files
+    if (byStatus.added.length > 0) {
+        output.push(`### Added (${byStatus.added.length})`);
+        for (const file of byStatus.added) {
+            output.push(`- ${file.newPath} (+${file.additions})`);
+        }
+        output.push("");
+    }
+    // List modified files with change description
+    if (byStatus.modified.length > 0) {
+        output.push(`### Modified (${byStatus.modified.length})`);
+        for (const file of byStatus.modified) {
+            const changeDesc = `+${file.additions}/-${file.deletions}, ${file.hunks.length} hunk${file.hunks.length !== 1 ? "s" : ""}`;
+            output.push(`- ${file.newPath}: ${changeDesc}`);
+        }
+        output.push("");
+    }
+    // List deleted files
+    if (byStatus.deleted.length > 0) {
+        output.push(`### Deleted (${byStatus.deleted.length})`);
+        for (const file of byStatus.deleted) {
+            output.push(`- ${file.oldPath} (-${file.deletions})`);
+        }
+        output.push("");
+    }
+    // List renamed files
+    if (byStatus.renamed.length > 0) {
+        output.push(`### Renamed (${byStatus.renamed.length})`);
+        for (const file of byStatus.renamed) {
+            output.push(`- ${file.oldPath} -> ${file.newPath}`);
+        }
+        output.push("");
+    }
+    return output.join("\n").trim();
+}
+/**
+ * Strategy 3: semantic
+ * Use TF-IDF to rank hunks by importance, keep most significant
+ */
+function compressSemantic(parsed, maxTokens) {
+    const allHunks = [];
+    // Flatten hunks with file context
+    for (const file of parsed.files) {
+        for (const hunk of file.hunks) {
+            allHunks.push({
+                file,
+                hunk,
+                score: 0,
+                tokens: countTokens(hunk.content),
+            });
+        }
+    }
+    if (allHunks.length === 0) {
+        return compressSummary(parsed);
+    }
+    // Calculate TF-IDF scores for hunk content
+    const hunkTexts = allHunks.map((h) => h.hunk.content);
+    const tfidfMap = calculateTFIDF(hunkTexts);
+    // Score each hunk
+    allHunks.forEach((item, i) => {
+        let score = getSegmentTFIDFScore(i, tfidfMap);
+        const content = item.hunk.content;
+        // Boost for certain patterns
+        // Errors/exceptions are critical
+        if (/error|exception|fail|throw|panic/i.test(content)) {
+            score += 0.3;
+        }
+        // Function/class definitions are important
+        if (/^[+-]\s*(export\s+)?(function|class|interface|type|const|let|var|def |fn |pub fn|func )/m.test(content)) {
+            score += 0.2;
+        }
+        // Test files have lower priority (usually verbose)
+        if (/\.(test|spec)\.[jt]sx?$/.test(item.file.newPath ?? "")) {
+            score -= 0.1;
+        }
+        // More changes = more important
+        score += Math.min((item.hunk.additions + item.hunk.deletions) / 50, 0.2);
+        item.score = Math.min(Math.max(score, 0), 1);
+    });
+    // Sort by score descending
+    allHunks.sort((a, b) => b.score - a.score);
+    // Select hunks until maxTokens
+    const selected = [];
+    let tokensUsed = 0;
+    for (const item of allHunks) {
+        if (tokensUsed + item.tokens <= maxTokens) {
+            selected.push(item);
+            tokensUsed += item.tokens;
+        }
+    }
+    // If nothing selected, at least include summary
+    if (selected.length === 0) {
+        return compressSummary(parsed);
+    }
+    // Group by file and output
+    const fileMap = new Map();
+    for (const item of selected) {
+        const path = item.file.newPath ?? item.file.oldPath ?? "unknown";
+        if (!fileMap.has(path)) {
+            fileMap.set(path, []);
+        }
+        fileMap.get(path).push(item);
+    }
+    const output = [];
+    output.push("## Semantic Diff Compression");
+    output.push(`Showing ${selected.length} of ${allHunks.length} hunks (most important)`);
+    output.push("");
+    for (const [path, items] of fileMap) {
+        output.push(`### ${path}`);
+        output.push("```diff");
+        for (const item of items) {
+            output.push(item.hunk.content);
+        }
+        output.push("```");
+        output.push("");
+    }
+    return output.join("\n").trim();
+}
+// =============================================================================
+// Main Compressor
+// =============================================================================
+/**
+ * Compress a git diff using the specified strategy
+ */
+export function compressDiff(diff, options) {
+    const parsed = parseDiff(diff);
+    // Extract file paths
+    const filesChanged = parsed.files
+        .map((f) => f.newPath ?? f.oldPath ?? "unknown")
+        .filter((p) => p !== "unknown");
+    // Calculate original tokens
+    const originalTokens = countTokens(diff);
+    // Apply compression strategy
+    let compressed;
+    let technique;
+    switch (options.strategy) {
+        case "hunks-only":
+            compressed = compressHunksOnly(parsed, options.contextLines ?? 1);
+            technique = "diff:hunks-only";
+            break;
+        case "summary":
+            compressed = compressSummary(parsed);
+            technique = "diff:summary";
+            break;
+        case "semantic":
+            const maxTokens = options.maxTokens ?? Math.ceil(originalTokens * 0.5);
+            compressed = compressSemantic(parsed, maxTokens);
+            technique = "diff:semantic";
+            break;
+        default:
+            compressed = compressHunksOnly(parsed);
+            technique = "diff:hunks-only";
+    }
+    const compressedTokens = countTokens(compressed);
+    // Generate summary
+    const summary = `${parsed.files.length} file${parsed.files.length !== 1 ? "s" : ""} changed, +${parsed.totalAdditions}/-${parsed.totalDeletions}`;
+    // Calculate stats
+    const stats = {
+        originalLines: diff.split("\n").length,
+        compressedLines: compressed.split("\n").length,
+        originalTokens,
+        compressedTokens,
+        reductionPercent: originalTokens > 0
+            ? Math.round((1 - compressedTokens / originalTokens) * 100)
+            : 0,
+        technique,
+    };
+    return {
+        compressed,
+        filesChanged,
+        summary,
+        additions: parsed.totalAdditions,
+        deletions: parsed.totalDeletions,
+        stats,
+    };
+}
+/**
+ * Diff compressor instance for use with tool system
+ */
+export const diffCompressor = {
+    name: "diff",
+    compress: compressDiff,
+};

package/dist/compressors/generic.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Generic Compressor
+ *
+ * Fallback compressor for unrecognized content types.
+ * Uses line deduplication and repetition detection.
+ */
+import type { Compressor } from "./types.js";
+export declare const genericCompressor: Compressor;
+//# sourceMappingURL=generic.d.ts.map

package/dist/compressors/generic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generic.d.ts","sourceRoot":"","sources":["../../src/compressors/generic.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAgD,MAAM,YAAY,CAAC;AAoK3F,eAAO,MAAM,iBAAiB,EAAE,UA4C/B,CAAC"}

package/dist/compressors/generic.js

@@ -0,0 +1,188 @@
+/**
+ * Generic Compressor
+ *
+ * Fallback compressor for unrecognized content types.
+ * Uses line deduplication and repetition detection.
+ */
+import { encodingForModel } from "js-tiktoken";
+const encoding = encodingForModel("gpt-4");
+function countTokens(text) {
+    return encoding.encode(text).length;
+}
+/**
+ * Normalize a line for pattern matching (remove numbers, normalize whitespace)
+ */
+function normalizeLine(line) {
+    return line
+        .replace(/\d+/g, "N") // Replace numbers
+        .replace(/0x[a-fA-F0-9]+/g, "HEX") // Replace hex values
+        .replace(/[a-f0-9]{8,}/gi, "HASH") // Replace hash-like strings
+        .replace(/\s+/g, " ") // Normalize whitespace
+        .trim();
+}
+/**
+ * Check if a line contains error indicators
+ */
+function isErrorLine(line) {
+    return /\b(error|Error|ERROR|fail|Fail|FAIL|fatal|Fatal|FATAL)\b/.test(line);
+}
+/**
+ * Check if a line contains warning indicators
+ */
+function isWarningLine(line) {
+    return /\b(warn|Warn|WARN|warning|Warning|WARNING)\b/.test(line);
+}
+/**
+ * Check if a line matches any preserve pattern
+ */
+function shouldPreserve(line, patterns) {
+    if (!patterns || patterns.length === 0)
+        return false;
+    return patterns.some((pattern) => pattern.test(line));
+}
+/**
+ * Group lines by their normalized pattern
+ */
+function groupLines(lines, preservePatterns) {
+    const groups = new Map();
+    const result = [];
+    for (const line of lines) {
+        if (!line.trim())
+            continue;
+        // Always keep preserved lines as individual groups
+        if (shouldPreserve(line, preservePatterns)) {
+            result.push({
+                pattern: line,
+                sample: line,
+                lines: [line],
+                count: 1,
+                hasError: isErrorLine(line),
+                hasWarning: isWarningLine(line),
+            });
+            continue;
+        }
+        const normalized = normalizeLine(line);
+        const existing = groups.get(normalized);
+        if (existing) {
+            existing.lines.push(line);
+            existing.count++;
+            if (isErrorLine(line))
+                existing.hasError = true;
+            if (isWarningLine(line))
+                existing.hasWarning = true;
+        }
+        else {
+            const group = {
+                pattern: normalized,
+                sample: line,
+                lines: [line],
+                count: 1,
+                hasError: isErrorLine(line),
+                hasWarning: isWarningLine(line),
+            };
+            groups.set(normalized, group);
+        }
+    }
+    // Add grouped lines to result
+    result.push(...groups.values());
+    return result;
+}
+/**
+ * Compress grouped lines based on detail level
+ */
+function compressGroups(groups, detail) {
+    const output = [];
+    const threshold = detail === "minimal" ? 2 : detail === "normal" ? 3 : 5;
+    for (const group of groups) {
+        // Always show errors and warnings fully
+        if (group.hasError || group.hasWarning) {
+            if (detail === "detailed" || group.count <= threshold) {
+                output.push(...group.lines);
+            }
+            else {
+                output.push(group.sample);
+                if (group.count > 1) {
+                    output.push(`  ... (${group.count - 1} similar ${group.hasError ? "errors" : "warnings"})`);
+                }
+            }
+            continue;
+        }
+        // Regular lines: show sample and count if above threshold
+        if (group.count <= threshold) {
+            output.push(...group.lines);
+        }
+        else {
+            output.push(group.sample);
+            output.push(`  ... (${group.count - 1} similar lines omitted)`);
+        }
+    }
+    return output;
+}
+/**
+ * Detect consecutive repeating lines
+ */
+function compressConsecutiveRepeats(lines, threshold = 3) {
+    const output = [];
+    let i = 0;
+    while (i < lines.length) {
+        const currentLine = lines[i];
+        let repeatCount = 1;
+        // Count consecutive identical or similar lines
+        while (i + repeatCount < lines.length) {
+            const nextLine = lines[i + repeatCount];
+            if (nextLine === currentLine ||
+                normalizeLine(nextLine ?? "") === normalizeLine(currentLine ?? "")) {
+                repeatCount++;
+            }
+            else {
+                break;
+            }
+        }
+        if (repeatCount >= threshold) {
+            output.push(currentLine ?? "");
+            output.push(`  ... (repeated ${repeatCount - 1} more times)`);
+        }
+        else {
+            for (let j = 0; j < repeatCount; j++) {
+                output.push(lines[i + j] ?? "");
+            }
+        }
+        i += repeatCount;
+    }
+    return output;
+}
+export const genericCompressor = {
+    name: "generic",
+    supportedTypes: ["generic", "code"],
+    canCompress(_content) {
+        // Generic compressor can always compress
+        return true;
+    },
+    compress(content, options) {
+        const lines = content.split("\n");
+        const originalTokens = countTokens(content);
+        // First pass: compress consecutive repeats
+        const afterRepeats = compressConsecutiveRepeats(lines);
+        // Second pass: group and deduplicate similar lines
+        const groups = groupLines(afterRepeats, options.preservePatterns);
+        const compressedLines = compressGroups(groups, options.detail);
+        const compressed = compressedLines.join("\n");
+        const compressedTokens = countTokens(compressed);
+        const reductionPercent = originalTokens > 0 ? Math.round((1 - compressedTokens / originalTokens) * 100) : 0;
+        // Build omitted info
+        const totalOmitted = lines.length - compressedLines.length;
+        const omittedInfo = totalOmitted > 0 ? `${totalOmitted} lines omitted through deduplication` : undefined;
+        return {
+            compressed,
+            stats: {
+                originalLines: lines.length,
+                compressedLines: compressedLines.length,
+                originalTokens,
+                compressedTokens,
+                reductionPercent,
+                technique: "line-deduplication",
+            },
+            omittedInfo,
+        };
+    },
+};

package/dist/compressors/index.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Compressors Index
+ *
+ * Orchestrates content compression with auto-detection and routing.
+ */
+export * from "./types.js";
+import type { Compressor, CompressOptions, CompressedResult, ContentType } from "./types.js";
+/**
+ * Get the appropriate compressor for a content type
+ */
+export declare function getCompressor(contentType: ContentType): Compressor;
+/**
+ * Compress content with auto-detection
+ */
+export declare function compressContent(content: string, options?: Partial<CompressOptions> & {
+    contentType?: ContentType;
+}): CompressedResult;
+/**
+ * Analyze content and suggest compression approach
+ */
+export declare function analyzeContent(content: string): {
+    detectedType: ContentType;
+    suggestedCompressor: string;
+    estimatedReduction: string;
+};
+export { logsCompressor } from "./logs.js";
+export { stacktraceCompressor } from "./stacktrace.js";
+export { configCompressor } from "./config.js";
+export { genericCompressor } from "./generic.js";
+export { semanticCompressor } from "./semantic.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/compressors/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/compressors/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,YAAY,CAAC;AAE3B,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAe7F;;GAEG;AACH,wBAAgB,aAAa,CAAC,WAAW,EAAE,WAAW,GAAG,UAAU,CAOlE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,OAAO,CAAC,eAAe,CAAC,GAAG;IAAE,WAAW,CAAC,EAAE,WAAW,CAAA;CAAO,GACrE,gBAAgB,CAqBlB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG;IAC/C,YAAY,EAAE,WAAW,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAyBA;AAGD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC"}