@blockrun/runcode 2.4.0 → 2.5.0

package/dist/compression/codebook.js

@@ -1,118 +0,0 @@
-/**
- * Dictionary Codebook
- *
- * Static dictionary of frequently repeated phrases observed in LLM prompts.
- * Built from analysis of BlockRun production logs.
- *
- * Format: Short code ($XX) -> Long phrase
- * The LLM receives a codebook header and decodes in-context.
- */
-// Static codebook - common patterns from system prompts
-// Ordered by expected frequency and impact
-export const STATIC_CODEBOOK = {
-    // High-impact: OpenClaw/Agent system prompt patterns (very common)
-    "$OC01": "unbrowse_", // Common prefix in tool names
-    "$OC02": "<location>",
-    "$OC03": "</location>",
-    "$OC04": "<name>",
-    "$OC05": "</name>",
-    "$OC06": "<description>",
-    "$OC07": "</description>",
-    "$OC08": "(may need login)",
-    "$OC09": "API skill for OpenClaw",
-    "$OC10": "endpoints",
-    // Skill/tool markers
-    "$SK01": "<available_skills>",
-    "$SK02": "</available_skills>",
-    "$SK03": "<skill>",
-    "$SK04": "</skill>",
-    // Schema patterns (very common in tool definitions)
-    "$T01": 'type: "function"',
-    "$T02": '"type": "function"',
-    "$T03": '"type": "string"',
-    "$T04": '"type": "object"',
-    "$T05": '"type": "array"',
-    "$T06": '"type": "boolean"',
-    "$T07": '"type": "number"',
-    // Common descriptions
-    "$D01": "description:",
-    "$D02": '"description":',
-    // Common instructions
-    "$I01": "You are a personal assistant",
-    "$I02": "Tool names are case-sensitive",
-    "$I03": "Call tools exactly as listed",
-    "$I04": "Use when",
-    "$I05": "without asking",
-    // Safety phrases
-    "$S01": "Do not manipulate or persuade",
-    "$S02": "Prioritize safety and human oversight",
-    "$S03": "unless explicitly requested",
-    // JSON patterns
-    "$J01": '"required": ["',
-    "$J02": '"properties": {',
-    "$J03": '"additionalProperties": false',
-    // Heartbeat patterns
-    "$H01": "HEARTBEAT_OK",
-    "$H02": "Read HEARTBEAT.md if it exists",
-    // Role markers
-    "$R01": '"role": "system"',
-    "$R02": '"role": "user"',
-    "$R03": '"role": "assistant"',
-    "$R04": '"role": "tool"',
-    // Common endings/phrases
-    "$E01": "would you like to",
-    "$E02": "Let me know if you",
-    "$E03": "internal APIs",
-    "$E04": "session cookies",
-    // BlockRun model aliases (common in prompts)
-    "$M01": "blockrun/",
-    "$M02": "openai/",
-    "$M03": "anthropic/",
-    "$M04": "google/",
-    "$M05": "xai/",
-};
-/**
- * Get the inverse codebook for decompression.
- */
-export function getInverseCodebook() {
-    const inverse = {};
-    for (const [code, phrase] of Object.entries(STATIC_CODEBOOK)) {
-        inverse[phrase] = code;
-    }
-    return inverse;
-}
-/**
- * Generate the codebook header for inclusion in system message.
- * LLMs can decode in-context using this header.
- */
-export function generateCodebookHeader(usedCodes, pathMap = {}) {
-    if (usedCodes.size === 0 && Object.keys(pathMap).length === 0) {
-        return "";
-    }
-    const parts = [];
-    // Add used dictionary codes
-    if (usedCodes.size > 0) {
-        const codeEntries = Array.from(usedCodes)
-            .map((code) => `${code}=${STATIC_CODEBOOK[code]}`)
-            .join(", ");
-        parts.push(`[Dict: ${codeEntries}]`);
-    }
-    // Add path map
-    if (Object.keys(pathMap).length > 0) {
-        const pathEntries = Object.entries(pathMap)
-            .map(([code, path]) => `${code}=${path}`)
-            .join(", ");
-        parts.push(`[Paths: ${pathEntries}]`);
-    }
-    return parts.join("\n");
-}
-/**
- * Decompress a string using the codebook (for logging).
- */
-export function decompressContent(content, codebook = STATIC_CODEBOOK) {
-    let result = content;
-    for (const [code, phrase] of Object.entries(codebook)) {
-        result = result.split(code).join(phrase);
-    }
-    return result;
-}

package/dist/compression/index.d.ts

@@ -1,32 +0,0 @@
-/**
- * LLM-Safe Context Compression
- *
- * Reduces token usage by 15-40% while preserving semantic meaning.
- * Implements 7 compression layers inspired by claw-compactor.
- *
- * Usage:
- *   const result = await compressContext(messages);
- *   // result.messages -> compressed version to send to provider
- *   // result.originalMessages -> original for logging
- */
-import { NormalizedMessage, CompressionConfig, CompressionResult } from "./types.js";
-export * from "./types.js";
-export { STATIC_CODEBOOK } from "./codebook.js";
-/**
- * Main compression function.
- *
- * Applies 5 layers in sequence:
- * 1. Deduplication - Remove exact duplicate messages
- * 2. Whitespace - Normalize excessive whitespace
- * 3. Dictionary - Replace common phrases with codes
- * 4. Paths - Shorten repeated file paths
- * 5. JSON - Compact JSON in tool calls
- *
- * Then prepends a codebook header for the LLM to decode in-context.
- */
-export declare function compressContext(messages: NormalizedMessage[], config?: Partial<CompressionConfig>): Promise<CompressionResult>;
-/**
- * Quick check if compression would benefit these messages.
- * Returns true if messages are large enough to warrant compression.
- */
-export declare function shouldCompress(messages: NormalizedMessage[]): boolean;

package/dist/compression/index.js

@@ -1,258 +0,0 @@
-/**
- * LLM-Safe Context Compression
- *
- * Reduces token usage by 15-40% while preserving semantic meaning.
- * Implements 7 compression layers inspired by claw-compactor.
- *
- * Usage:
- *   const result = await compressContext(messages);
- *   // result.messages -> compressed version to send to provider
- *   // result.originalMessages -> original for logging
- */
-import { DEFAULT_COMPRESSION_CONFIG, } from "./types.js";
-import { deduplicateMessages } from "./layers/deduplication.js";
-import { normalizeMessagesWhitespace } from "./layers/whitespace.js";
-import { encodeMessages } from "./layers/dictionary.js";
-import { shortenPaths } from "./layers/paths.js";
-import { compactMessagesJson } from "./layers/json-compact.js";
-import { compressObservations } from "./layers/observation.js";
-import { applyDynamicCodebook, generateDynamicCodebookHeader } from "./layers/dynamic-codebook.js";
-import { generateCodebookHeader, STATIC_CODEBOOK } from "./codebook.js";
-export * from "./types.js";
-export { STATIC_CODEBOOK } from "./codebook.js";
-/**
- * Calculate total character count for messages.
- */
-function calculateTotalChars(messages) {
-    return messages.reduce((total, msg) => {
-        let chars = 0;
-        if (Array.isArray(msg.content)) {
-            for (const part of msg.content) {
-                if (part.type === "text" && part.text)
-                    chars += part.text.length;
-                else if (part.type === "image_url")
-                    chars += 2500; // ~1000 tokens worth
-            }
-        }
-        else {
-            chars = msg.content?.length || 0;
-        }
-        if (msg.tool_calls) {
-            chars += JSON.stringify(msg.tool_calls).length;
-        }
-        return total + chars;
-    }, 0);
-}
-/**
- * Check if any message contains image_url content parts.
- */
-function hasVisionContent(messages) {
-    return messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === "image_url"));
-}
-/**
- * Deep clone messages to preserve originals.
- */
-function cloneMessages(messages) {
-    return JSON.parse(JSON.stringify(messages));
-}
-/**
- * Prepend codebook header to the first USER message (not system).
- *
- * Why not system message?
- * - Google Gemini uses systemInstruction which doesn't support codebook format
- * - The codebook header in user message is still visible to all LLMs
- * - This ensures compatibility across all providers
- */
-function prependCodebookHeader(messages, usedCodes, pathMap) {
-    const header = generateCodebookHeader(usedCodes, pathMap);
-    if (!header)
-        return messages;
-    // Find first user message (not system - Google's systemInstruction doesn't support codebook)
-    const userIndex = messages.findIndex((m) => m.role === "user");
-    if (userIndex === -1) {
-        // No user message, add codebook as system (fallback)
-        return [
-            { role: "system", content: header },
-            ...messages,
-        ];
-    }
-    // Prepend to first user message
-    return messages.map((msg, i) => {
-        if (i === userIndex) {
-            return {
-                ...msg,
-                content: `${header}\n\n${msg.content || ""}`,
-            };
-        }
-        return msg;
-    });
-}
-/**
- * Main compression function.
- *
- * Applies 5 layers in sequence:
- * 1. Deduplication - Remove exact duplicate messages
- * 2. Whitespace - Normalize excessive whitespace
- * 3. Dictionary - Replace common phrases with codes
- * 4. Paths - Shorten repeated file paths
- * 5. JSON - Compact JSON in tool calls
- *
- * Then prepends a codebook header for the LLM to decode in-context.
- */
-export async function compressContext(messages, config = {}) {
-    const fullConfig = {
-        ...DEFAULT_COMPRESSION_CONFIG,
-        ...config,
-        layers: {
-            ...DEFAULT_COMPRESSION_CONFIG.layers,
-            ...config.layers,
-        },
-        dictionary: {
-            ...DEFAULT_COMPRESSION_CONFIG.dictionary,
-            ...config.dictionary,
-        },
-    };
-    // If compression disabled, return as-is
-    if (!fullConfig.enabled) {
-        const originalChars = calculateTotalChars(messages);
-        return {
-            messages,
-            originalMessages: messages,
-            originalChars,
-            compressedChars: originalChars,
-            compressionRatio: 1,
-            stats: {
-                duplicatesRemoved: 0,
-                whitespaceSavedChars: 0,
-                dictionarySubstitutions: 0,
-                pathsShortened: 0,
-                jsonCompactedChars: 0,
-                observationsCompressed: 0,
-                observationCharsSaved: 0,
-                dynamicSubstitutions: 0,
-                dynamicCharsSaved: 0,
-            },
-            codebook: {},
-            pathMap: {},
-            dynamicCodes: {},
-        };
-    }
-    // Preserve originals for logging
-    const originalMessages = fullConfig.preserveRaw
-        ? cloneMessages(messages)
-        : messages;
-    const originalChars = calculateTotalChars(messages);
-    // Initialize stats
-    const stats = {
-        duplicatesRemoved: 0,
-        whitespaceSavedChars: 0,
-        dictionarySubstitutions: 0,
-        pathsShortened: 0,
-        jsonCompactedChars: 0,
-        observationsCompressed: 0,
-        observationCharsSaved: 0,
-        dynamicSubstitutions: 0,
-        dynamicCharsSaved: 0,
-    };
-    let result = cloneMessages(messages);
-    let usedCodes = new Set();
-    let pathMap = {};
-    let dynamicCodes = {};
-    // Layer 1: Deduplication
-    if (fullConfig.layers.deduplication) {
-        const dedupResult = deduplicateMessages(result);
-        result = dedupResult.messages;
-        stats.duplicatesRemoved = dedupResult.duplicatesRemoved;
-    }
-    // Layer 2: Whitespace normalization
-    if (fullConfig.layers.whitespace) {
-        const wsResult = normalizeMessagesWhitespace(result);
-        result = wsResult.messages;
-        stats.whitespaceSavedChars = wsResult.charsSaved;
-    }
-    // Layer 3: Dictionary encoding
-    if (fullConfig.layers.dictionary) {
-        const dictResult = encodeMessages(result);
-        result = dictResult.messages;
-        stats.dictionarySubstitutions = dictResult.substitutionCount;
-        usedCodes = dictResult.usedCodes;
-    }
-    // Layer 4: Path shortening
-    if (fullConfig.layers.paths) {
-        const pathResult = shortenPaths(result);
-        result = pathResult.messages;
-        pathMap = pathResult.pathMap;
-        stats.pathsShortened = Object.keys(pathMap).length;
-    }
-    // Layer 5: JSON compaction
-    if (fullConfig.layers.jsonCompact) {
-        const jsonResult = compactMessagesJson(result);
-        result = jsonResult.messages;
-        stats.jsonCompactedChars = jsonResult.charsSaved;
-    }
-    // Layer 6: Observation compression (BIG WIN - 97% on tool results)
-    if (fullConfig.layers.observation) {
-        const obsResult = compressObservations(result);
-        result = obsResult.messages;
-        stats.observationsCompressed = obsResult.observationsCompressed;
-        stats.observationCharsSaved = obsResult.charsSaved;
-    }
-    // Layer 7: Dynamic codebook (learns from actual content)
-    if (fullConfig.layers.dynamicCodebook) {
-        const dynResult = applyDynamicCodebook(result);
-        result = dynResult.messages;
-        stats.dynamicSubstitutions = dynResult.substitutions;
-        stats.dynamicCharsSaved = dynResult.charsSaved;
-        dynamicCodes = dynResult.dynamicCodes;
-    }
-    // Add codebook header if enabled and we have codes to include
-    if (fullConfig.dictionary.includeCodebookHeader &&
-        (usedCodes.size > 0 || Object.keys(pathMap).length > 0 || Object.keys(dynamicCodes).length > 0)) {
-        result = prependCodebookHeader(result, usedCodes, pathMap);
-        // Also add dynamic codebook header if we have dynamic codes
-        if (Object.keys(dynamicCodes).length > 0) {
-            const dynHeader = generateDynamicCodebookHeader(dynamicCodes);
-            if (dynHeader) {
-                const systemIndex = result.findIndex((m) => m.role === "system");
-                if (systemIndex >= 0) {
-                    result[systemIndex] = {
-                        ...result[systemIndex],
-                        content: `${dynHeader}\n${result[systemIndex].content || ""}`,
-                    };
-                }
-            }
-        }
-    }
-    // Calculate final stats
-    const compressedChars = calculateTotalChars(result);
-    const compressionRatio = compressedChars / originalChars;
-    // Build used codebook for logging
-    const usedCodebook = {};
-    usedCodes.forEach((code) => {
-        usedCodebook[code] = STATIC_CODEBOOK[code];
-    });
-    return {
-        messages: result,
-        originalMessages,
-        originalChars,
-        compressedChars,
-        compressionRatio,
-        stats,
-        codebook: usedCodebook,
-        pathMap,
-        dynamicCodes,
-    };
-}
-/**
- * Quick check if compression would benefit these messages.
- * Returns true if messages are large enough to warrant compression.
- */
-export function shouldCompress(messages) {
-    // Skip compression entirely when messages contain images —
-    // compression layers operate on string content and would corrupt image_url parts
-    if (hasVisionContent(messages))
-        return false;
-    const chars = calculateTotalChars(messages);
-    // Only compress if > 5000 chars (roughly 1000 tokens)
-    return chars > 5000;
-}

package/dist/compression/layers/deduplication.d.ts

@@ -1,27 +0,0 @@
-/**
- * Layer 1: Message Deduplication
- *
- * Removes exact duplicate messages from conversation history.
- * Common in heartbeat patterns and repeated tool calls.
- *
- * Safe for LLM: Identical messages add no new information.
- * Expected savings: 2-5%
- */
-import { NormalizedMessage } from "../types.js";
-export interface DeduplicationResult {
-    messages: NormalizedMessage[];
-    duplicatesRemoved: number;
-    originalCount: number;
-}
-/**
- * Remove exact duplicate messages from the conversation.
- *
- * Strategy:
- * - Keep first occurrence of each unique message
- * - Preserve order for semantic coherence
- * - Never dedupe system messages (they set context)
- * - Allow duplicate user messages (user might repeat intentionally)
- * - CRITICAL: Never dedupe assistant messages with tool_calls that are
- *   referenced by subsequent tool messages (breaks Anthropic tool_use/tool_result pairing)
- */
-export declare function deduplicateMessages(messages: NormalizedMessage[]): DeduplicationResult;

package/dist/compression/layers/deduplication.js

@@ -1,97 +0,0 @@
-/**
- * Layer 1: Message Deduplication
- *
- * Removes exact duplicate messages from conversation history.
- * Common in heartbeat patterns and repeated tool calls.
- *
- * Safe for LLM: Identical messages add no new information.
- * Expected savings: 2-5%
- */
-import { createHash } from 'node:crypto';
-/**
- * Generate a hash for a message based on its semantic content.
- * Uses role + content + tool_call_id to identify duplicates.
- */
-function hashMessage(message) {
-    const parts = [
-        message.role,
-        message.content || "",
-        message.tool_call_id || "",
-        message.name || "",
-    ];
-    // Include tool_calls if present
-    if (message.tool_calls) {
-        parts.push(JSON.stringify(message.tool_calls.map((tc) => ({
-            name: tc.function.name,
-            args: tc.function.arguments,
-        }))));
-    }
-    const content = parts.join("|");
-    return createHash("md5").update(content).digest("hex");
-}
-/**
- * Remove exact duplicate messages from the conversation.
- *
- * Strategy:
- * - Keep first occurrence of each unique message
- * - Preserve order for semantic coherence
- * - Never dedupe system messages (they set context)
- * - Allow duplicate user messages (user might repeat intentionally)
- * - CRITICAL: Never dedupe assistant messages with tool_calls that are
- *   referenced by subsequent tool messages (breaks Anthropic tool_use/tool_result pairing)
- */
-export function deduplicateMessages(messages) {
-    const seen = new Set();
-    const result = [];
-    let duplicatesRemoved = 0;
-    // First pass: collect all tool_call_ids that are referenced by tool messages
-    // These tool_calls MUST be preserved to maintain tool_use/tool_result pairing
-    const referencedToolCallIds = new Set();
-    for (const message of messages) {
-        if (message.role === "tool" && message.tool_call_id) {
-            referencedToolCallIds.add(message.tool_call_id);
-        }
-    }
-    for (const message of messages) {
-        // Always keep system messages (they set important context)
-        if (message.role === "system") {
-            result.push(message);
-            continue;
-        }
-        // Always keep user messages (user might repeat intentionally)
-        if (message.role === "user") {
-            result.push(message);
-            continue;
-        }
-        // Always keep tool messages (they are results of tool calls)
-        // Removing them would break the tool_use/tool_result pairing
-        if (message.role === "tool") {
-            result.push(message);
-            continue;
-        }
-        // For assistant messages with tool_calls, check if any are referenced
-        // by subsequent tool messages - if so, we MUST keep this message
-        if (message.role === "assistant" && message.tool_calls) {
-            const hasReferencedToolCall = message.tool_calls.some((tc) => referencedToolCallIds.has(tc.id));
-            if (hasReferencedToolCall) {
-                // This assistant message has tool_calls that are referenced - keep it
-                result.push(message);
-                continue;
-            }
-        }
-        // For other assistant messages, check for duplicates
-        const hash = hashMessage(message);
-        if (!seen.has(hash)) {
-            seen.add(hash);
-            result.push(message);
-        }
-        else {
-            duplicatesRemoved++;
-        }
-    }
-    return {
-        messages: result,
-        duplicatesRemoved,
-        originalCount: messages.length,
-    };
-}

package/dist/compression/layers/dictionary.d.ts

@@ -1,20 +0,0 @@
-/**
- * Layer 3: Dictionary Encoding
- *
- * Replaces frequently repeated long phrases with short codes.
- * Uses a static codebook of common patterns from production logs.
- *
- * Safe for LLM: Reversible substitution with codebook header.
- * Expected savings: 4-8%
- */
-import { NormalizedMessage } from "../types.js";
-export interface DictionaryResult {
-    messages: NormalizedMessage[];
-    substitutionCount: number;
-    usedCodes: Set<string>;
-    charsSaved: number;
-}
-/**
- * Apply dictionary encoding to all messages.
- */
-export declare function encodeMessages(messages: NormalizedMessage[]): DictionaryResult;

package/dist/compression/layers/dictionary.js

@@ -1,67 +0,0 @@
-/**
- * Layer 3: Dictionary Encoding
- *
- * Replaces frequently repeated long phrases with short codes.
- * Uses a static codebook of common patterns from production logs.
- *
- * Safe for LLM: Reversible substitution with codebook header.
- * Expected savings: 4-8%
- */
-import { getInverseCodebook } from "../codebook.js";
-/**
- * Apply dictionary encoding to a string.
- * Returns the encoded string and stats.
- */
-function encodeContent(content, inverseCodebook) {
-    let encoded = content;
-    let substitutions = 0;
-    let charsSaved = 0;
-    const codes = new Set();
-    // Sort phrases by length (longest first) to avoid partial matches
-    const phrases = Object.keys(inverseCodebook).sort((a, b) => b.length - a.length);
-    for (const phrase of phrases) {
-        const code = inverseCodebook[phrase];
-        const regex = new RegExp(escapeRegex(phrase), "g");
-        const matches = encoded.match(regex);
-        if (matches && matches.length > 0) {
-            encoded = encoded.replace(regex, code);
-            substitutions += matches.length;
-            charsSaved += matches.length * (phrase.length - code.length);
-            codes.add(code);
-        }
-    }
-    return { encoded, substitutions, codes, charsSaved };
-}
-/**
- * Escape special regex characters in a string.
- */
-function escapeRegex(str) {
-    return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-/**
- * Apply dictionary encoding to all messages.
- */
-export function encodeMessages(messages) {
-    const inverseCodebook = getInverseCodebook();
-    let totalSubstitutions = 0;
-    let totalCharsSaved = 0;
-    const allUsedCodes = new Set();
-    const result = messages.map((message) => {
-        if (!message.content || typeof message.content !== "string")
-            return message;
-        const { encoded, substitutions, codes, charsSaved } = encodeContent(message.content, inverseCodebook);
-        totalSubstitutions += substitutions;
-        totalCharsSaved += charsSaved;
-        codes.forEach((code) => allUsedCodes.add(code));
-        return {
-            ...message,
-            content: encoded,
-        };
-    });
-    return {
-        messages: result,
-        substitutionCount: totalSubstitutions,
-        usedCodes: allUsedCodes,
-        charsSaved: totalCharsSaved,
-    };
-}

package/dist/compression/layers/dynamic-codebook.d.ts

@@ -1,25 +0,0 @@
-/**
- * L7: Dynamic Codebook Builder
- *
- * Inspired by claw-compactor's frequency-based codebook.
- * Builds codebook from actual content being compressed,
- * rather than relying on static patterns.
- *
- * Finds phrases that appear 3+ times and replaces with short codes.
- */
-import { NormalizedMessage } from "../types.js";
-interface DynamicCodebookResult {
-    messages: NormalizedMessage[];
-    charsSaved: number;
-    dynamicCodes: Record<string, string>;
-    substitutions: number;
-}
-/**
- * Apply dynamic codebook to messages.
- */
-export declare function applyDynamicCodebook(messages: NormalizedMessage[]): DynamicCodebookResult;
-/**
- * Generate header for dynamic codes (to include in system message).
- */
-export declare function generateDynamicCodebookHeader(codebook: Record<string, string>): string;
-export {};