distill-mcp 0.6.0-beta

package/dist/compressors/index.js

@@ -0,0 +1,82 @@
+/**
+ * Compressors Index
+ *
+ * Orchestrates content compression with auto-detection and routing.
+ */
+export * from "./types.js";
+import { detectContentType } from "../utils/content-detector.js";
+import { logsCompressor } from "./logs.js";
+import { stacktraceCompressor } from "./stacktrace.js";
+import { configCompressor } from "./config.js";
+import { genericCompressor } from "./generic.js";
+// Registered compressors in priority order
+const compressors = [
+    logsCompressor,
+    stacktraceCompressor,
+    configCompressor,
+    genericCompressor, // Fallback - always last
+];
+/**
+ * Get the appropriate compressor for a content type
+ */
+export function getCompressor(contentType) {
+    for (const compressor of compressors) {
+        if (compressor.supportedTypes.includes(contentType)) {
+            return compressor;
+        }
+    }
+    return genericCompressor;
+}
+/**
+ * Compress content with auto-detection
+ */
+export function compressContent(content, options = {}) {
+    // Merge with defaults
+    const opts = {
+        detail: options.detail ?? "normal",
+        targetRatio: options.targetRatio,
+        preservePatterns: options.preservePatterns,
+    };
+    // Detect or use provided content type
+    const contentType = options.contentType ?? detectContentType(content);
+    // Get appropriate compressor
+    const compressor = getCompressor(contentType);
+    // Compress
+    const result = compressor.compress(content, opts);
+    // Add content type info to technique
+    result.stats.technique = `${contentType}:${result.stats.technique}`;
+    return result;
+}
+/**
+ * Analyze content and suggest compression approach
+ */
+export function analyzeContent(content) {
+    const detectedType = detectContentType(content);
+    const compressor = getCompressor(detectedType);
+    // Estimate reduction based on content type
+    let estimatedReduction;
+    switch (detectedType) {
+        case "logs":
+            estimatedReduction = "70-90%";
+            break;
+        case "stacktrace":
+            estimatedReduction = "50-80%";
+            break;
+        case "config":
+            estimatedReduction = "30-60%";
+            break;
+        default:
+            estimatedReduction = "20-50%";
+    }
+    return {
+        detectedType,
+        suggestedCompressor: compressor.name,
+        estimatedReduction,
+    };
+}
+// Re-export individual compressors for direct use
+export { logsCompressor } from "./logs.js";
+export { stacktraceCompressor } from "./stacktrace.js";
+export { configCompressor } from "./config.js";
+export { genericCompressor } from "./generic.js";
+export { semanticCompressor } from "./semantic.js";

package/dist/compressors/logs.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Logs Compressor
+ *
+ * Specialized compressor for application logs.
+ * Groups similar log entries and summarizes repetitive patterns.
+ */
+import type { Compressor } from "./types.js";
+export declare const logsCompressor: Compressor;
+//# sourceMappingURL=logs.d.ts.map

package/dist/compressors/logs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logs.d.ts","sourceRoot":"","sources":["../../src/compressors/logs.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAgD,MAAM,YAAY,CAAC;AA8N3F,eAAO,MAAM,cAAc,EAAE,UAuD5B,CAAC"}

package/dist/compressors/logs.js

@@ -0,0 +1,245 @@
+/**
+ * Logs Compressor
+ *
+ * Specialized compressor for application logs.
+ * Groups similar log entries and summarizes repetitive patterns.
+ */
+import { encodingForModel } from "js-tiktoken";
+const encoding = encodingForModel("gpt-4");
+function countTokens(text) {
+    return encoding.encode(text).length;
+}
+// Log level patterns
+const LOG_LEVELS = ["TRACE", "DEBUG", "INFO", "WARN", "WARNING", "ERROR", "FATAL"];
+/**
+ * Extract timestamp from log line
+ */
+function extractTimestamp(line) {
+    // ISO format: 2024-01-15T10:00:01
+    const isoMatch = line.match(/\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:\.\d+)?/);
+    if (isoMatch)
+        return isoMatch[0];
+    // Bracket format: [10:00:01]
+    const bracketMatch = line.match(/\[(\d{2}:\d{2}:\d{2})\]/);
+    if (bracketMatch)
+        return bracketMatch[1];
+    // Syslog format: Jan 15 10:00:01
+    const syslogMatch = line.match(/[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}/);
+    if (syslogMatch)
+        return syslogMatch[0];
+    return undefined;
+}
+/**
+ * Extract log level from line
+ */
+function extractLogLevel(line) {
+    const upperLine = line.toUpperCase();
+    for (const level of LOG_LEVELS) {
+        if (upperLine.includes(`[${level}]`) || upperLine.includes(` ${level} `)) {
+            return level;
+        }
+    }
+    return undefined;
+}
+/**
+ * Normalize log message for grouping
+ */
+function normalizeLogMessage(message) {
+    return message
+        .replace(/\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:\.\d+)?/g, "TIMESTAMP") // Remove timestamps
+        .replace(/\[\d{2}:\d{2}:\d{2}\]/g, "[TIMESTAMP]")
+        .replace(/\d+\.\d+\.\d+\.\d+/g, "IP") // IP addresses
+        .replace(/\d+ms/g, "Nms") // Durations
+        .replace(/\d+/g, "N") // Numbers
+        .replace(/[a-f0-9]{8,}/gi, "HASH") // Hashes/IDs
+        .replace(/"[^"]*"/g, '"STR"') // Quoted strings
+        .replace(/'[^']*'/g, "'STR'")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+/**
+ * Parse a log line into structured data
+ */
+function parseLogLine(line) {
+    const timestamp = extractTimestamp(line);
+    const level = extractLogLevel(line);
+    // Remove timestamp and level to get message
+    let message = line;
+    if (timestamp) {
+        message = message.replace(timestamp, "").trim();
+    }
+    return {
+        raw: line,
+        timestamp,
+        level,
+        message,
+        normalizedMessage: normalizeLogMessage(message),
+    };
+}
+/**
+ * Group log lines by their normalized message
+ */
+function groupLogLines(lines, preservePatterns) {
+    const groups = new Map();
+    for (const line of lines) {
+        if (!line.trim())
+            continue;
+        // Check if line should be preserved
+        const shouldPreserve = preservePatterns?.some((p) => p.test(line));
+        if (shouldPreserve) {
+            // Add as unique group
+            const key = `preserve_${groups.size}`;
+            groups.set(key, {
+                pattern: line,
+                sample: line,
+                lines: [line],
+                count: 1,
+                hasError: false,
+                hasWarning: false,
+            });
+            continue;
+        }
+        const parsed = parseLogLine(line);
+        const key = parsed.normalizedMessage;
+        const existing = groups.get(key);
+        if (existing) {
+            existing.lines.push(line);
+            existing.count++;
+        }
+        else {
+            const isError = parsed.level === "ERROR" || parsed.level === "FATAL";
+            const isWarning = parsed.level === "WARN" || parsed.level === "WARNING";
+            groups.set(key, {
+                pattern: parsed.normalizedMessage,
+                sample: line,
+                lines: [line],
+                count: 1,
+                hasError: isError,
+                hasWarning: isWarning,
+            });
+        }
+    }
+    return groups;
+}
+/**
+ * Generate compressed output from groups
+ */
+function generateCompressedOutput(groups, detail) {
+    const output = [];
+    // Sort: errors first, then warnings, then by count (descending)
+    const sortedGroups = Array.from(groups.values()).sort((a, b) => {
+        if (a.hasError && !b.hasError)
+            return -1;
+        if (!a.hasError && b.hasError)
+            return 1;
+        if (a.hasWarning && !b.hasWarning)
+            return -1;
+        if (!a.hasWarning && b.hasWarning)
+            return 1;
+        return b.count - a.count;
+    });
+    const threshold = detail === "minimal" ? 2 : detail === "normal" ? 3 : 5;
+    for (const group of sortedGroups) {
+        // Always show errors and warnings
+        if (group.hasError || group.hasWarning) {
+            if (detail === "detailed") {
+                output.push(...group.lines);
+            }
+            else {
+                output.push(group.sample);
+                if (group.count > 1) {
+                    output.push(`  ... (${group.count - 1} similar entries)`);
+                }
+            }
+            continue;
+        }
+        // Regular logs
+        if (group.count <= threshold) {
+            if (detail === "detailed") {
+                output.push(...group.lines);
+            }
+            else {
+                output.push(group.sample);
+                if (group.count > 1) {
+                    output.push(`  ... (${group.count - 1} similar entries)`);
+                }
+            }
+        }
+        else {
+            output.push(group.sample);
+            output.push(`  ... (${group.count - 1} similar entries omitted)`);
+        }
+    }
+    return output;
+}
+/**
+ * Generate summary statistics
+ */
+function generateSummary(groups) {
+    let errorCount = 0;
+    let warnCount = 0;
+    let infoCount = 0;
+    for (const group of groups.values()) {
+        if (group.hasError) {
+            errorCount += group.count;
+        }
+        else if (group.hasWarning) {
+            warnCount += group.count;
+        }
+        else {
+            infoCount += group.count;
+        }
+    }
+    const parts = [];
+    parts.push("### Log Summary");
+    parts.push(`- **Errors:** ${errorCount}`);
+    parts.push(`- **Warnings:** ${warnCount}`);
+    parts.push(`- **Info/Other:** ${infoCount}`);
+    parts.push(`- **Unique patterns:** ${groups.size}`);
+    return parts.join("\n");
+}
+export const logsCompressor = {
+    name: "logs",
+    supportedTypes: ["logs"],
+    canCompress(content) {
+        // Check for log-like patterns
+        const logPatterns = [
+            /\[\s*(INFO|DEBUG|WARN|WARNING|ERROR|TRACE|FATAL)\s*\]/i,
+            /\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}/,
+            /^[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}/m,
+        ];
+        return logPatterns.some((pattern) => pattern.test(content));
+    },
+    compress(content, options) {
+        const lines = content.split("\n");
+        const originalTokens = countTokens(content);
+        // Group log lines
+        const groups = groupLogLines(lines, options.preservePatterns);
+        // Generate compressed output
+        const compressedLines = generateCompressedOutput(groups, options.detail);
+        // Add summary if not minimal
+        if (options.detail !== "minimal") {
+            compressedLines.push("");
+            compressedLines.push("---");
+            compressedLines.push(generateSummary(groups));
+        }
+        const compressed = compressedLines.join("\n");
+        const compressedTokens = countTokens(compressed);
+        const reductionPercent = originalTokens > 0 ? Math.round((1 - compressedTokens / originalTokens) * 100) : 0;
+        // Count omitted
+        const totalOriginal = Array.from(groups.values()).reduce((sum, g) => sum + g.count, 0);
+        const omittedCount = totalOriginal - compressedLines.filter((l) => l.trim() && !l.startsWith("  ...")).length;
+        return {
+            compressed,
+            stats: {
+                originalLines: lines.length,
+                compressedLines: compressedLines.length,
+                originalTokens,
+                compressedTokens,
+                reductionPercent,
+                technique: "log-grouping",
+            },
+            omittedInfo: omittedCount > 0 ? `${omittedCount} repetitive log entries summarized` : undefined,
+        };
+    },
+};

package/dist/compressors/multifile.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Multi-File Compressor
+ *
+ * Compresses multiple files with cross-file deduplication,
+ * dependency-aware skeleton extraction, and smart chunking.
+ */
+/**
+ * File context for multi-file compression
+ */
+export interface FileContext {
+    path: string;
+    content: string;
+    language?: string;
+}
+/**
+ * Import information extracted from files
+ */
+export interface ExtractedImport {
+    source: string;
+    names: string[];
+    isDefault: boolean;
+    isNamespace: boolean;
+    raw: string;
+}
+/**
+ * Type/interface information extracted from files
+ */
+export interface ExtractedType {
+    name: string;
+    kind: "type" | "interface" | "class" | "enum";
+    definition: string;
+    usedIn: string[];
+}
+/**
+ * Shared elements across files
+ */
+export interface SharedElements {
+    imports: Array<{
+        source: string;
+        names: string[];
+        usedIn: string[];
+    }>;
+    types: ExtractedType[];
+    constants: Array<{
+        name: string;
+        value: string;
+        usedIn: string[];
+    }>;
+}
+/**
+ * Multi-file compression options
+ */
+export interface MultiFileCompressOptions {
+    /** Maximum total tokens for output */
+    maxTokens?: number;
+    /** Compression strategy */
+    strategy: "deduplicate" | "skeleton" | "smart-chunk";
+    /** Patterns to preserve fully (glob patterns) */
+    preservePatterns?: string[];
+    /** Entry points for dependency analysis */
+    entryPoints?: string[];
+    /** Depth for dependency traversal */
+    dependencyDepth?: number;
+}
+/**
+ * Multi-file compression result
+ */
+export interface MultiFileCompressResult {
+    /** Compressed context */
+    compressed: string;
+    /** Files included in output */
+    filesIncluded: string[];
+    /** Shared elements extracted */
+    sharedElements: SharedElements;
+    /** Compression statistics */
+    stats: {
+        originalTokens: number;
+        compressedTokens: number;
+        filesProcessed: number;
+        deduplicatedItems: number;
+        reductionPercent: number;
+    };
+}
+/**
+ * Chunk information for smart chunking
+ */
+export interface ChunkInfo {
+    id: string;
+    files: string[];
+    tokens: number;
+    dependencies: string[];
+    content?: string;
+}
+/**
+ * Extract shared elements from files
+ */
+export declare function extractSharedElements(files: FileContext[]): SharedElements;
+/**
+ * Create chunks based on dependencies
+ */
+export declare function createChunks(files: FileContext[], maxTokensPerChunk: number): ChunkInfo[];
+/**
+ * Compress multiple files with cross-file deduplication
+ */
+export declare function compressMultiFile(files: FileContext[], options: MultiFileCompressOptions): MultiFileCompressResult;
+//# sourceMappingURL=multifile.d.ts.map

package/dist/compressors/multifile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multifile.d.ts","sourceRoot":"","sources":["../../src/compressors/multifile.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AASH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;IACnB,WAAW,EAAE,OAAO,CAAC;IACrB,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,OAAO,GAAG,MAAM,CAAC;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;IACtE,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,SAAS,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;CACrE;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,QAAQ,EAAE,aAAa,GAAG,UAAU,GAAG,aAAa,CAAC;IACrD,iDAAiD;IACjD,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,qCAAqC;IACrC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,yBAAyB;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,gCAAgC;IAChC,cAAc,EAAE,cAAc,CAAC;IAC/B,6BAA6B;IAC7B,KAAK,EAAE;QACL,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;QACzB,cAAc,EAAE,MAAM,CAAC;QACvB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAqQD;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,cAAc,CAkB1E;AAgKD;;GAEG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,WAAW,EAAE,EACpB,iBAAiB,EAAE,MAAM,GACxB,SAAS,EAAE,CA0Db;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,WAAW,EAAE,EACpB,OAAO,EAAE,wBAAwB,GAChC,uBAAuB,CA0GzB"}