@blockrun/clawrouter 0.12.102 → 0.12.104

package/dist/compression/index.js

@@ -1,246 +0,0 @@
-/**
- * LLM-Safe Context Compression
- *
- * Reduces token usage by 15-40% while preserving semantic meaning.
- * Implements 7 compression layers inspired by claw-compactor.
- *
- * Usage:
- *   const result = await compressContext(messages);
- *   // result.messages -> compressed version to send to provider
- *   // result.originalMessages -> original for logging
- */
-import { DEFAULT_COMPRESSION_CONFIG, } from "./types.js";
-import { deduplicateMessages } from "./layers/deduplication.js";
-import { normalizeMessagesWhitespace } from "./layers/whitespace.js";
-import { encodeMessages } from "./layers/dictionary.js";
-import { shortenPaths } from "./layers/paths.js";
-import { compactMessagesJson } from "./layers/json-compact.js";
-import { compressObservations } from "./layers/observation.js";
-import { applyDynamicCodebook, generateDynamicCodebookHeader } from "./layers/dynamic-codebook.js";
-import { generateCodebookHeader, STATIC_CODEBOOK } from "./codebook.js";
-export * from "./types.js";
-export { STATIC_CODEBOOK } from "./codebook.js";
-/**
- * Calculate total character count for messages.
- */
-function calculateTotalChars(messages) {
-    return messages.reduce((total, msg) => {
-        let chars = 0;
-        if (typeof msg.content === "string") {
-            chars = msg.content.length;
-        }
-        else if (Array.isArray(msg.content)) {
-            // For multimodal content, stringify to get approximate size
-            chars = JSON.stringify(msg.content).length;
-        }
-        if (msg.tool_calls) {
-            chars += JSON.stringify(msg.tool_calls).length;
-        }
-        return total + chars;
-    }, 0);
-}
-/**
- * Deep clone messages to preserve originals.
- */
-function cloneMessages(messages) {
-    return JSON.parse(JSON.stringify(messages));
-}
-/**
- * Prepend codebook header to the first USER message (not system).
- *
- * Why not system message?
- * - Google Gemini uses systemInstruction which doesn't support codebook format
- * - The codebook header in user message is still visible to all LLMs
- * - This ensures compatibility across all providers
- */
-function prependCodebookHeader(messages, usedCodes, pathMap) {
-    const header = generateCodebookHeader(usedCodes, pathMap);
-    if (!header)
-        return messages;
-    // Find first user message (not system - Google's systemInstruction doesn't support codebook)
-    const userIndex = messages.findIndex((m) => m.role === "user");
-    if (userIndex === -1) {
-        // No user message, add codebook as system (fallback)
-        return [{ role: "system", content: header }, ...messages];
-    }
-    // Prepend to first user message (only if content is a string)
-    return messages.map((msg, i) => {
-        if (i === userIndex) {
-            // Only prepend to string content - skip arrays (multimodal messages)
-            if (typeof msg.content === "string") {
-                return {
-                    ...msg,
-                    content: `${header}\n\n${msg.content}`,
-                };
-            }
-            // For non-string content, don't modify the message
-            // The codebook header would corrupt array content
-        }
-        return msg;
-    });
-}
-/**
- * Main compression function.
- *
- * Applies 5 layers in sequence:
- * 1. Deduplication - Remove exact duplicate messages
- * 2. Whitespace - Normalize excessive whitespace
- * 3. Dictionary - Replace common phrases with codes
- * 4. Paths - Shorten repeated file paths
- * 5. JSON - Compact JSON in tool calls
- *
- * Then prepends a codebook header for the LLM to decode in-context.
- */
-export async function compressContext(messages, config = {}) {
-    const fullConfig = {
-        ...DEFAULT_COMPRESSION_CONFIG,
-        ...config,
-        layers: {
-            ...DEFAULT_COMPRESSION_CONFIG.layers,
-            ...config.layers,
-        },
-        dictionary: {
-            ...DEFAULT_COMPRESSION_CONFIG.dictionary,
-            ...config.dictionary,
-        },
-    };
-    // If compression disabled, return as-is
-    if (!fullConfig.enabled) {
-        const originalChars = calculateTotalChars(messages);
-        return {
-            messages,
-            originalMessages: messages,
-            originalChars,
-            compressedChars: originalChars,
-            compressionRatio: 1,
-            stats: {
-                duplicatesRemoved: 0,
-                whitespaceSavedChars: 0,
-                dictionarySubstitutions: 0,
-                pathsShortened: 0,
-                jsonCompactedChars: 0,
-                observationsCompressed: 0,
-                observationCharsSaved: 0,
-                dynamicSubstitutions: 0,
-                dynamicCharsSaved: 0,
-            },
-            codebook: {},
-            pathMap: {},
-            dynamicCodes: {},
-        };
-    }
-    // Preserve originals for logging
-    const originalMessages = fullConfig.preserveRaw ? cloneMessages(messages) : messages;
-    const originalChars = calculateTotalChars(messages);
-    // Initialize stats
-    const stats = {
-        duplicatesRemoved: 0,
-        whitespaceSavedChars: 0,
-        dictionarySubstitutions: 0,
-        pathsShortened: 0,
-        jsonCompactedChars: 0,
-        observationsCompressed: 0,
-        observationCharsSaved: 0,
-        dynamicSubstitutions: 0,
-        dynamicCharsSaved: 0,
-    };
-    let result = cloneMessages(messages);
-    let usedCodes = new Set();
-    let pathMap = {};
-    let dynamicCodes = {};
-    // Layer 1: Deduplication
-    if (fullConfig.layers.deduplication) {
-        const dedupResult = deduplicateMessages(result);
-        result = dedupResult.messages;
-        stats.duplicatesRemoved = dedupResult.duplicatesRemoved;
-    }
-    // Layer 2: Whitespace normalization
-    if (fullConfig.layers.whitespace) {
-        const wsResult = normalizeMessagesWhitespace(result);
-        result = wsResult.messages;
-        stats.whitespaceSavedChars = wsResult.charsSaved;
-    }
-    // Layer 3: Dictionary encoding
-    if (fullConfig.layers.dictionary) {
-        const dictResult = encodeMessages(result);
-        result = dictResult.messages;
-        stats.dictionarySubstitutions = dictResult.substitutionCount;
-        usedCodes = dictResult.usedCodes;
-    }
-    // Layer 4: Path shortening
-    if (fullConfig.layers.paths) {
-        const pathResult = shortenPaths(result);
-        result = pathResult.messages;
-        pathMap = pathResult.pathMap;
-        stats.pathsShortened = Object.keys(pathMap).length;
-    }
-    // Layer 5: JSON compaction
-    if (fullConfig.layers.jsonCompact) {
-        const jsonResult = compactMessagesJson(result);
-        result = jsonResult.messages;
-        stats.jsonCompactedChars = jsonResult.charsSaved;
-    }
-    // Layer 6: Observation compression (BIG WIN - 97% on tool results)
-    if (fullConfig.layers.observation) {
-        const obsResult = compressObservations(result);
-        result = obsResult.messages;
-        stats.observationsCompressed = obsResult.observationsCompressed;
-        stats.observationCharsSaved = obsResult.charsSaved;
-    }
-    // Layer 7: Dynamic codebook (learns from actual content)
-    if (fullConfig.layers.dynamicCodebook) {
-        const dynResult = applyDynamicCodebook(result);
-        result = dynResult.messages;
-        stats.dynamicSubstitutions = dynResult.substitutions;
-        stats.dynamicCharsSaved = dynResult.charsSaved;
-        dynamicCodes = dynResult.dynamicCodes;
-    }
-    // Add codebook header if enabled and we have codes to include
-    if (fullConfig.dictionary.includeCodebookHeader &&
-        (usedCodes.size > 0 || Object.keys(pathMap).length > 0 || Object.keys(dynamicCodes).length > 0)) {
-        result = prependCodebookHeader(result, usedCodes, pathMap);
-        // Also add dynamic codebook header if we have dynamic codes
-        if (Object.keys(dynamicCodes).length > 0) {
-            const dynHeader = generateDynamicCodebookHeader(dynamicCodes);
-            if (dynHeader) {
-                const systemIndex = result.findIndex((m) => m.role === "system");
-                // Only prepend to string content - skip arrays (multimodal messages)
-                if (systemIndex >= 0 && typeof result[systemIndex].content === "string") {
-                    result[systemIndex] = {
-                        ...result[systemIndex],
-                        content: `${dynHeader}\n${result[systemIndex].content}`,
-                    };
-                }
-            }
-        }
-    }
-    // Calculate final stats
-    const compressedChars = calculateTotalChars(result);
-    const compressionRatio = compressedChars / originalChars;
-    // Build used codebook for logging
-    const usedCodebook = {};
-    usedCodes.forEach((code) => {
-        usedCodebook[code] = STATIC_CODEBOOK[code];
-    });
-    return {
-        messages: result,
-        originalMessages,
-        originalChars,
-        compressedChars,
-        compressionRatio,
-        stats,
-        codebook: usedCodebook,
-        pathMap,
-        dynamicCodes,
-    };
-}
-/**
- * Quick check if compression would benefit these messages.
- * Returns true if messages are large enough to warrant compression.
- */
-export function shouldCompress(messages) {
-    const chars = calculateTotalChars(messages);
-    // Only compress if > 5000 chars (roughly 1000 tokens)
-    return chars > 5000;
-}
-//# sourceMappingURL=index.js.map

package/dist/compression/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/compression/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAKL,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AACnG,OAAO,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAExE,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAA6B;IACxD,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACpC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACpC,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;QAC7B,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,4DAA4D;YAC5D,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAC7C,CAAC;QACD,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACnB,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;QACjD,CAAC;QACD,OAAO,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC,EAAE,CAAC,CAAC,CAAC;AACR,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,QAA6B;IAClD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,qBAAqB,CAC5B,QAA6B,EAC7B,SAAsB,EACtB,OAA+B;IAE/B,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC1D,IAAI,CAAC,MAAM;QAAE,OAAO,QAAQ,CAAC;IAE7B,6FAA6F;IAC7F,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAE/D,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;QACrB,qDAAqD;QACrD,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,QAAQ,CAAC,CAAC;IAC5D,CAAC;IAED,8DAA8D;IAC9D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QAC7B,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACpB,qEAAqE;YACrE,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACpC,OAAO;oBACL,GAAG,GAAG;oBACN,OAAO,EAAE,GAAG,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE;iBACvC,CAAC;YACJ,CAAC;YACD,mDAAmD;YACnD,kDAAkD;QACpD,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAA6B,EAC7B,SAAqC,EAAE;IAEvC,MAAM,UAAU,GAAsB;QACpC,GAAG,0BAA0B;QAC7B,GAAG,MAAM;QACT,MAAM,EAAE;YACN,GAAG,0BAA0B,CAAC,MAAM;YACpC,GAAG,MAAM,CAAC,MAAM;SACjB;QACD,UAAU,EAAE;YACV,GAAG,0BAA0B,CAAC,UAAU;YACxC,GAAG,MAAM,CAAC,UAAU;SACrB;KACF,CAAC;IAEF,wCAAwC;IACxC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,aAAa,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACpD,OAAO;YACL,QAAQ;YACR,gBAAgB,EAAE,QAAQ;YAC1B,aAAa;YACb,eAAe,EAAE,aAAa;YAC9B,gBAAgB,EAAE,CAAC;YACnB,KAAK,EAAE;gBACL,iBAAiB,EAAE,CAAC;gBACpB,oBAAoB,EAAE,CAAC;gBACvB,uBAAuB,EAAE,CAAC;gBAC1B,cAAc,EAAE,CAAC;gBACjB,kBAAkB,EAAE,CAAC;gBACrB,sBAAsB,EAAE,CAAC;gBACzB,qBAAqB,EAAE,CAAC;gBACxB,oBAAoB,EAAE,CAAC;gBACvB,iBAAiB,EAAE,CAAC;aACrB;YACD,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,EAAE;YACX,YAAY,EAAE,EAAE;SACjB,CAAC;IACJ,CAAC;IAED,iCAAiC;IACjC,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACrF,MAAM,aAAa,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAEpD,mBAAmB;IACnB,MAAM,KAAK,GAAqB;QAC9B,iBAAiB,EAAE,CAAC;QACpB,oBAAoB,EAAE,CAAC;QACvB,uBAAuB,EAAE,CAAC;QAC1B,cAAc,EAAE,CAAC;QACjB,kBAAkB,EAAE,CAAC;QACrB,sBAAsB,EAAE,CAAC;QACzB,qBAAqB,EAAE,CAAC;QACxB,oBAAoB,EAAE,CAAC;QACvB,iBAAiB,EAAE,CAAC;KACrB,CAAC;IAEF,IAAI,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACrC,IAAI,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,IAAI,OAAO,GAA2B,EAAE,CAAC;IACzC,IAAI,YAAY,GAA2B,EAAE,CAAC;IAE9C,yBAAyB;IACzB,IAAI,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,WAAW,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC;QAC9B,KAAK,CAAC,iBAAiB,GAAG,WAAW,CAAC,iBAAiB,CAAC;IAC1D,CAAC;IAED,oCAAoC;IACpC,IAAI,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;QACrD,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAC3B,KAAK,CAAC,oBAAoB,GAAG,QAAQ,CAAC,UAAU,CAAC;IACnD,CAAC;IAED,+BAA+B;IAC/B,IAAI,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;QAC7B,KAAK,CAAC,uBAAuB,GAAG,UAAU,CAAC,iBAAiB,CAAC;QAC7D,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;IACnC,CAAC;IAED,2BAA2B;IAC3B,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACxC,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;QAC7B,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAC7B,KAAK,CAAC,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACrD,CAAC;IAED,2BAA2B;IAC3B,IAAI,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;QAC7B,KAAK,CAAC,kBAAkB,GAAG,UAAU,CAAC,UAAU,CAAC;IACnD,CAAC;IAED,mEAAmE;IACnE,IAAI,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC;QAC5B,KAAK,CAAC,sBAAsB,GAAG,SAAS,CAAC,sBAAsB,CAAC;QAChE,KAAK,CAAC,qBAAqB,GAAG,SAAS,CAAC,UAAU,CAAC;IACrD,CAAC;IAED,yDAAyD;IACzD,IAAI,UAAU,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC;QAC5B,KAAK,CAAC,oBAAoB,GAAG,SAAS,CAAC,aAAa,CAAC;QACrD,KAAK,CAAC,iBAAiB,GAAG,SAAS,CAAC,UAAU,CAAC;QAC/C,YAAY,GAAG,SAAS,CAAC,YAAY,CAAC;IACxC,CAAC;IAED,8DAA8D;IAC9D,IACE,UAAU,CAAC,UAAU,CAAC,qBAAqB;QAC3C,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAC/F,CAAC;QACD,MAAM,GAAG,qBAAqB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC3D,4DAA4D;QAC5D,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,6BAA6B,CAAC,YAAY,CAAC,CAAC;YAC9D,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;gBACjE,qEAAqE;gBACrE,IAAI,WAAW,IAAI,CAAC,IAAI,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACxE,MAAM,CAAC,WAAW,CAAC,GAAG;wBACpB,GAAG,MAAM,CAAC,WAAW,CAAC;wBACtB,OAAO,EAAE,GAAG,SAAS,KAAK,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;qBACxD,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,eAAe,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,gBAAgB,GAAG,eAAe,GAAG,aAAa,CAAC;IAEzD,kCAAkC;IAClC,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,YAAY,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,QAAQ,EAAE,MAAM;QAChB,gBAAgB;QAChB,aAAa;QACb,eAAe;QACf,gBAAgB;QAChB,KAAK;QACL,QAAQ,EAAE,YAAY;QACtB,OAAO;QACP,YAAY;KACb,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,QAA6B;IAC1D,MAAM,KAAK,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAC5C,sDAAsD;IACtD,OAAO,KAAK,GAAG,IAAI,CAAC;AACtB,CAAC"}

package/dist/compression/layers/deduplication.d.ts

@@ -1,28 +0,0 @@
-/**
- * Layer 1: Message Deduplication
- *
- * Removes exact duplicate messages from conversation history.
- * Common in heartbeat patterns and repeated tool calls.
- *
- * Safe for LLM: Identical messages add no new information.
- * Expected savings: 2-5%
- */
-import { NormalizedMessage } from "../types.js";
-export interface DeduplicationResult {
-    messages: NormalizedMessage[];
-    duplicatesRemoved: number;
-    originalCount: number;
-}
-/**
- * Remove exact duplicate messages from the conversation.
- *
- * Strategy:
- * - Keep first occurrence of each unique message
- * - Preserve order for semantic coherence
- * - Never dedupe system messages (they set context)
- * - Allow duplicate user messages (user might repeat intentionally)
- * - CRITICAL: Never dedupe assistant messages with tool_calls that are
- *   referenced by subsequent tool messages (breaks Anthropic tool_use/tool_result pairing)
- */
-export declare function deduplicateMessages(messages: NormalizedMessage[]): DeduplicationResult;
-//# sourceMappingURL=deduplication.d.ts.map

package/dist/compression/layers/deduplication.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"deduplication.d.ts","sourceRoot":"","sources":["../../../src/compression/layers/deduplication.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAGhD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;CACvB;AAiCD;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,mBAAmB,CA+DtF"}

package/dist/compression/layers/deduplication.js

@@ -1,101 +0,0 @@
-/**
- * Layer 1: Message Deduplication
- *
- * Removes exact duplicate messages from conversation history.
- * Common in heartbeat patterns and repeated tool calls.
- *
- * Safe for LLM: Identical messages add no new information.
- * Expected savings: 2-5%
- */
-import crypto from "crypto";
-/**
- * Generate a hash for a message based on its semantic content.
- * Uses role + content + tool_call_id to identify duplicates.
- */
-function hashMessage(message) {
-    // Handle content - stringify arrays (multimodal), use string directly, or empty string
-    let contentStr = "";
-    if (typeof message.content === "string") {
-        contentStr = message.content;
-    }
-    else if (Array.isArray(message.content)) {
-        contentStr = JSON.stringify(message.content);
-    }
-    const parts = [message.role, contentStr, message.tool_call_id || "", message.name || ""];
-    // Include tool_calls if present
-    if (message.tool_calls) {
-        parts.push(JSON.stringify(message.tool_calls.map((tc) => ({
-            name: tc.function.name,
-            args: tc.function.arguments,
-        }))));
-    }
-    const content = parts.join("|");
-    return crypto.createHash("md5").update(content).digest("hex");
-}
-/**
- * Remove exact duplicate messages from the conversation.
- *
- * Strategy:
- * - Keep first occurrence of each unique message
- * - Preserve order for semantic coherence
- * - Never dedupe system messages (they set context)
- * - Allow duplicate user messages (user might repeat intentionally)
- * - CRITICAL: Never dedupe assistant messages with tool_calls that are
- *   referenced by subsequent tool messages (breaks Anthropic tool_use/tool_result pairing)
- */
-export function deduplicateMessages(messages) {
-    const seen = new Set();
-    const result = [];
-    let duplicatesRemoved = 0;
-    // First pass: collect all tool_call_ids that are referenced by tool messages
-    // These tool_calls MUST be preserved to maintain tool_use/tool_result pairing
-    const referencedToolCallIds = new Set();
-    for (const message of messages) {
-        if (message.role === "tool" && message.tool_call_id) {
-            referencedToolCallIds.add(message.tool_call_id);
-        }
-    }
-    for (const message of messages) {
-        // Always keep system messages (they set important context)
-        if (message.role === "system") {
-            result.push(message);
-            continue;
-        }
-        // Always keep user messages (user might repeat intentionally)
-        if (message.role === "user") {
-            result.push(message);
-            continue;
-        }
-        // Always keep tool messages (they are results of tool calls)
-        // Removing them would break the tool_use/tool_result pairing
-        if (message.role === "tool") {
-            result.push(message);
-            continue;
-        }
-        // For assistant messages with tool_calls, check if any are referenced
-        // by subsequent tool messages - if so, we MUST keep this message
-        if (message.role === "assistant" && message.tool_calls) {
-            const hasReferencedToolCall = message.tool_calls.some((tc) => referencedToolCallIds.has(tc.id));
-            if (hasReferencedToolCall) {
-                // This assistant message has tool_calls that are referenced - keep it
-                result.push(message);
-                continue;
-            }
-        }
-        // For other assistant messages, check for duplicates
-        const hash = hashMessage(message);
-        if (!seen.has(hash)) {
-            seen.add(hash);
-            result.push(message);
-        }
-        else {
-            duplicatesRemoved++;
-        }
-    }
-    return {
-        messages: result,
-        duplicatesRemoved,
-        originalCount: messages.length,
-    };
-}
-//# sourceMappingURL=deduplication.js.map

package/dist/compression/layers/deduplication.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"deduplication.js","sourceRoot":"","sources":["../../../src/compression/layers/deduplication.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAQ5B;;;GAGG;AACH,SAAS,WAAW,CAAC,OAA0B;IAC7C,uFAAuF;IACvF,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACxC,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC;IAC/B,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,YAAY,IAAI,EAAE,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAEzF,gCAAgC;IAChC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,SAAS,CACZ,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9B,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YACtB,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,SAAS;SAC5B,CAAC,CAAC,CACJ,CACF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChC,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAA6B;IAC/D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAwB,EAAE,CAAC;IACvC,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,6EAA6E;IAC7E,8EAA8E;IAC9E,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAC;IAChD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACpD,qBAAqB,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,2DAA2D;QAC3D,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,SAAS;QACX,CAAC;QAED,8DAA8D;QAC9D,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,SAAS;QACX,CAAC;QAED,6DAA6D;QAC7D,6DAA6D;QAC7D,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,SAAS;QACX,CAAC;QAED,sEAAsE;QACtE,iEAAiE;QACjE,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvD,MAAM,qBAAqB,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAC3D,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CACjC,CAAC;YACF,IAAI,qBAAqB,EAAE,CAAC;gBAC1B,sEAAsE;gBACtE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;QACH,CAAC;QAED,qDAAqD;QACrD,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAElC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,iBAAiB,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,MAAM;QAChB,iBAAiB;QACjB,aAAa,EAAE,QAAQ,CAAC,MAAM;KAC/B,CAAC;AACJ,CAAC"}

package/dist/compression/layers/dictionary.d.ts

@@ -1,21 +0,0 @@
-/**
- * Layer 3: Dictionary Encoding
- *
- * Replaces frequently repeated long phrases with short codes.
- * Uses a static codebook of common patterns from production logs.
- *
- * Safe for LLM: Reversible substitution with codebook header.
- * Expected savings: 4-8%
- */
-import { NormalizedMessage } from "../types.js";
-export interface DictionaryResult {
-    messages: NormalizedMessage[];
-    substitutionCount: number;
-    usedCodes: Set<string>;
-    charsSaved: number;
-}
-/**
- * Apply dictionary encoding to all messages.
- */
-export declare function encodeMessages(messages: NormalizedMessage[]): DictionaryResult;
-//# sourceMappingURL=dictionary.d.ts.map

package/dist/compression/layers/dictionary.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"dictionary.d.ts","sourceRoot":"","sources":["../../../src/compression/layers/dictionary.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAGhD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB;AA6CD;;GAEG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,gBAAgB,CA+B9E"}

package/dist/compression/layers/dictionary.js

@@ -1,73 +0,0 @@
-/**
- * Layer 3: Dictionary Encoding
- *
- * Replaces frequently repeated long phrases with short codes.
- * Uses a static codebook of common patterns from production logs.
- *
- * Safe for LLM: Reversible substitution with codebook header.
- * Expected savings: 4-8%
- */
-import { getInverseCodebook } from "../codebook.js";
-/**
- * Apply dictionary encoding to a string.
- * Returns the encoded string and stats.
- */
-function encodeContent(content, inverseCodebook) {
-    // Defensive type check - content might be array/object for multimodal messages
-    if (!content || typeof content !== "string") {
-        return { encoded: content, substitutions: 0, codes: new Set(), charsSaved: 0 };
-    }
-    let encoded = content;
-    let substitutions = 0;
-    let charsSaved = 0;
-    const codes = new Set();
-    // Sort phrases by length (longest first) to avoid partial matches
-    const phrases = Object.keys(inverseCodebook).sort((a, b) => b.length - a.length);
-    for (const phrase of phrases) {
-        const code = inverseCodebook[phrase];
-        const regex = new RegExp(escapeRegex(phrase), "g");
-        const matches = encoded.match(regex);
-        if (matches && matches.length > 0) {
-            encoded = encoded.replace(regex, code);
-            substitutions += matches.length;
-            charsSaved += matches.length * (phrase.length - code.length);
-            codes.add(code);
-        }
-    }
-    return { encoded, substitutions, codes, charsSaved };
-}
-/**
- * Escape special regex characters in a string.
- */
-function escapeRegex(str) {
-    return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-/**
- * Apply dictionary encoding to all messages.
- */
-export function encodeMessages(messages) {
-    const inverseCodebook = getInverseCodebook();
-    let totalSubstitutions = 0;
-    let totalCharsSaved = 0;
-    const allUsedCodes = new Set();
-    const result = messages.map((message) => {
-        // Only process string content (skip arrays for multimodal messages)
-        if (!message.content || typeof message.content !== "string")
-            return message;
-        const { encoded, substitutions, codes, charsSaved } = encodeContent(message.content, inverseCodebook);
-        totalSubstitutions += substitutions;
-        totalCharsSaved += charsSaved;
-        codes.forEach((code) => allUsedCodes.add(code));
-        return {
-            ...message,
-            content: encoded,
-        };
-    });
-    return {
-        messages: result,
-        substitutionCount: totalSubstitutions,
-        usedCodes: allUsedCodes,
-        charsSaved: totalCharsSaved,
-    };
-}
-//# sourceMappingURL=dictionary.js.map

package/dist/compression/layers/dictionary.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"dictionary.js","sourceRoot":"","sources":["../../../src/compression/layers/dictionary.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AASpD;;;GAGG;AACH,SAAS,aAAa,CACpB,OAAe,EACf,eAAuC;IAEvC,+EAA+E;IAC/E,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC5C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IACjF,CAAC;IACD,IAAI,OAAO,GAAG,OAAO,CAAC;IACtB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,kEAAkE;IAClE,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IAEjF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAErC,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACvC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;YAChC,UAAU,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7D,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;AACvD,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO,GAAG,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,QAA6B;IAC1D,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;IAC7C,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IAEvC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QACtC,oEAAoE;QACpE,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;YAAE,OAAO,OAAO,CAAC;QAE5E,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,aAAa,CACjE,OAAO,CAAC,OAAO,EACf,eAAe,CAChB,CAAC;QAEF,kBAAkB,IAAI,aAAa,CAAC;QACpC,eAAe,IAAI,UAAU,CAAC;QAC9B,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAEhD,OAAO;YACL,GAAG,OAAO;YACV,OAAO,EAAE,OAAO;SACjB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,QAAQ,EAAE,MAAM;QAChB,iBAAiB,EAAE,kBAAkB;QACrC,SAAS,EAAE,YAAY;QACvB,UAAU,EAAE,eAAe;KAC5B,CAAC;AACJ,CAAC"}

package/dist/compression/layers/dynamic-codebook.d.ts

@@ -1,26 +0,0 @@
-/**
- * L7: Dynamic Codebook Builder
- *
- * Inspired by claw-compactor's frequency-based codebook.
- * Builds codebook from actual content being compressed,
- * rather than relying on static patterns.
- *
- * Finds phrases that appear 3+ times and replaces with short codes.
- */
-import { NormalizedMessage } from "../types.js";
-interface DynamicCodebookResult {
-    messages: NormalizedMessage[];
-    charsSaved: number;
-    dynamicCodes: Record<string, string>;
-    substitutions: number;
-}
-/**
- * Apply dynamic codebook to messages.
- */
-export declare function applyDynamicCodebook(messages: NormalizedMessage[]): DynamicCodebookResult;
-/**
- * Generate header for dynamic codes (to include in system message).
- */
-export declare function generateDynamicCodebookHeader(codebook: Record<string, string>): string;
-export {};
-//# sourceMappingURL=dynamic-codebook.d.ts.map

package/dist/compression/layers/dynamic-codebook.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"dynamic-codebook.d.ts","sourceRoot":"","sources":["../../../src/compression/layers/dynamic-codebook.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,UAAU,qBAAqB;IAC7B,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;CACvB;AAyFD;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,qBAAqB,CAmDzF;AAED;;GAEG;AACH,wBAAgB,6BAA6B,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAatF"}