@tyvm/knowhow 0.0.61 → 0.0.63

package/ts_build/src/processors/JsonCompressor.js

@@ -0,0 +1,295 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JsonCompressor = void 0;
+class JsonCompressor {
+    deduplicationMap = new Map();
+    objectSeenCount = new Map();
+    propertyNamesMap = new Map();
+    propertyNamesSeenCount = new Map();
+    compressionThreshold;
+    maxTokens;
+    toolName;
+    storage;
+    constructor(storage, compressionThreshold, maxTokens, toolName) {
+        this.storage = storage;
+        this.compressionThreshold = compressionThreshold;
+        this.maxTokens = maxTokens;
+        this.toolName = toolName;
+    }
+    clearDeduplication() {
+        this.deduplicationMap.clear();
+        this.objectSeenCount.clear();
+        this.propertyNamesMap.clear();
+        this.propertyNamesSeenCount.clear();
+    }
+    updateSettings(compressionThreshold, maxTokens) {
+        this.compressionThreshold = compressionThreshold;
+        this.maxTokens = maxTokens;
+    }
+    tryParseJson(content) {
+        try {
+            const parsed = JSON.parse(content);
+            if (typeof parsed === 'string') {
+                try {
+                    return this.tryParseJson(parsed);
+                }
+                catch (e) {
+                    return parsed;
+                }
+            }
+            if (parsed &&
+                typeof parsed === 'object' &&
+                Array.isArray(parsed.content) &&
+                parsed.content.length > 0) {
+                const firstContent = parsed.content[0];
+                if (firstContent.type === 'text' && typeof firstContent.text === 'string') {
+                    try {
+                        const nestedData = this.tryParseJson(firstContent.text);
+                        return {
+                            _mcp_format: true,
+                            _raw_structure: { content: [{ type: 'text' }] },
+                            _data: nestedData
+                        };
+                    }
+                    catch (e) {
+                        return parsed;
+                    }
+                }
+            }
+            return parsed;
+        }
+        catch {
+            return null;
+        }
+    }
+    generateSchema(obj, maxDepth = 3, currentDepth = 0) {
+        if (currentDepth > maxDepth) {
+            return { type: 'any' };
+        }
+        if (obj && typeof obj === 'object' && obj._mcp_format === true && obj._data) {
+            const dataSchema = this.generateSchema(obj._data, maxDepth, currentDepth);
+            return {
+                type: 'mcp_response',
+                properties: {
+                    _data: dataSchema
+                }
+            };
+        }
+        if (obj === null) {
+            return { type: 'null' };
+        }
+        if (Array.isArray(obj)) {
+            if (obj.length === 0) {
+                return { type: 'array', items: { type: 'unknown' } };
+            }
+            const sample = obj.slice(0, 3);
+            const itemSchemas = sample.map(item => this.generateSchema(item, maxDepth, currentDepth + 1));
+            return { type: 'array', items: itemSchemas[0] };
+        }
+        if (typeof obj === 'object') {
+            const properties = {};
+            for (const [key, value] of Object.entries(obj)) {
+                properties[key] = this.generateSchema(value, maxDepth, currentDepth + 1);
+            }
+            return { type: 'object', properties };
+        }
+        return { type: typeof obj };
+    }
+    calculateSimilarity(str1, str2) {
+        const maxLen = Math.max(str1.length, str2.length);
+        if (maxLen === 0)
+            return 1.0;
+        let commonPrefixLen = 0;
+        const minLen = Math.min(str1.length, str2.length);
+        for (let i = 0; i < minLen; i++) {
+            if (str1[i] === str2[i]) {
+                commonPrefixLen++;
+            }
+            else {
+                break;
+            }
+        }
+        return commonPrefixLen / maxLen;
+    }
+    detectLowSignalProperties(obj) {
+        if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
+            return { lowSignal: [], metadata: {} };
+        }
+        const lowSignal = [];
+        const metadata = {};
+        const entries = Object.entries(obj);
+        const urlPattern = /^https?:\/\//;
+        const urlProps = [];
+        for (const [key, value] of entries) {
+            if (typeof value === 'string' && urlPattern.test(value)) {
+                urlProps.push(key);
+            }
+        }
+        if (urlProps.length >= 3) {
+            const urlValues = urlProps.map(key => obj[key]);
+            let totalSimilarity = 0;
+            let comparisons = 0;
+            for (let i = 0; i < urlValues.length - 1; i++) {
+                for (let j = i + 1; j < urlValues.length; j++) {
+                    totalSimilarity += this.calculateSimilarity(urlValues[i], urlValues[j]);
+                    comparisons++;
+                }
+            }
+            const avgSimilarity = comparisons > 0 ? totalSimilarity / comparisons : 0;
+            if (avgSimilarity > 0.6) {
+                lowSignal.push(...urlProps);
+                metadata.url_similarity = avgSimilarity;
+                metadata.url_count = urlProps.length;
+            }
+        }
+        const lowSignalPatterns = [/_url$/, /_id$/, /^node_id$/, /^avatar_url$/, /^gravatar_id$/];
+        for (const [key, value] of entries) {
+            if (lowSignalPatterns.some(pattern => pattern.test(key)) && !lowSignal.includes(key)) {
+                lowSignal.push(key);
+            }
+        }
+        return { lowSignal, metadata };
+    }
+    compressObjectWithLowSignalDetection(obj, path = "") {
+        if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
+            return obj;
+        }
+        const { lowSignal, metadata } = this.detectLowSignalProperties(obj);
+        if (lowSignal.length < 5) {
+            return obj;
+        }
+        const highSignal = {};
+        const compressed = {};
+        for (const [key, value] of Object.entries(obj)) {
+            if (lowSignal.includes(key)) {
+                compressed[key] = value;
+            }
+            else {
+                highSignal[key] = value;
+            }
+        }
+        const compressedHash = this.hashObject(compressed);
+        let compressedKey = this.deduplicationMap.get(compressedHash);
+        if (!compressedKey) {
+            compressedKey = this.storage.generateKey();
+            this.deduplicationMap.set(compressedHash, compressedKey);
+            const compressionMetadata = {
+                compressed_properties: compressed,
+                compression_reason: 'low_signal_detection',
+                similarity_score: metadata.url_similarity,
+            };
+            this.storage.storeString(compressedKey, JSON.stringify(compressionMetadata));
+        }
+        const propertyNamesHash = this.hashObject(lowSignal);
+        const propertyNamesSeenCount = this.propertyNamesSeenCount.get(propertyNamesHash) || 0;
+        this.propertyNamesSeenCount.set(propertyNamesHash, propertyNamesSeenCount + 1);
+        let propertyNamesValue = lowSignal;
+        if (propertyNamesSeenCount === 0) {
+            const propertyNamesKey = this.storage.generateKey();
+            this.propertyNamesMap.set(propertyNamesHash, propertyNamesKey);
+            this.storage.storeString(propertyNamesKey, JSON.stringify(lowSignal));
+            propertyNamesValue = lowSignal;
+        }
+        else if (propertyNamesSeenCount >= 1) {
+            const existingPropertyNamesKey = this.propertyNamesMap.get(propertyNamesHash);
+            propertyNamesValue = `[DEDUPLICATED_ARRAY]\nKey: ${existingPropertyNamesKey}`;
+        }
+        return {
+            ...highSignal,
+            _compressed_properties_key: compressedKey,
+            _compressed_property_names: propertyNamesValue,
+            _compression_info: `${lowSignal.length} low-signal properties compressed (URLs, IDs). Use expandTokens with key "${compressedKey}" to retrieve.`
+        };
+    }
+    hashObject(obj) {
+        const normalized = JSON.stringify(obj, Object.keys(obj).sort());
+        let hash = 0;
+        for (let i = 0; i < normalized.length; i++) {
+            const char = normalized.charCodeAt(i);
+            hash = ((hash << 5) - hash) + char;
+            hash = hash & hash;
+        }
+        return hash.toString(36);
+    }
+    compressJsonProperties(obj, path = "") {
+        if (path === "" &&
+            this.storage.estimateTokens(JSON.stringify(obj)) <= this.maxTokens) {
+            return obj;
+        }
+        if (Array.isArray(obj)) {
+            const processedItems = obj.map((item, index) => this.compressJsonProperties(item, `${path}[${index}]`));
+            const finalArray = [];
+            let currentChunk = [];
+            for (let i = processedItems.length - 1; i >= 0; i--) {
+                const item = processedItems[i];
+                currentChunk.unshift(item);
+                const chunkString = JSON.stringify(currentChunk);
+                const chunkTokens = this.storage.estimateTokens(chunkString);
+                if (chunkTokens > this.compressionThreshold) {
+                    const key = this.storage.generateKey();
+                    this.storage.storeString(key, chunkString);
+                    const stub = `[COMPRESSED_JSON_ARRAY_CHUNK - ${chunkTokens} tokens, ${currentChunk.length} items]\nKey: ${key}\nPath: ${path}[${i}...${i + currentChunk.length - 1}]\nPreview: ${chunkString.substring(0, 100)}...\n[Use ${this.toolName} tool with key "${key}" to retrieve this chunk]`;
+                    finalArray.unshift(stub);
+                    currentChunk = [];
+                }
+            }
+            if (currentChunk.length > 0) {
+                finalArray.unshift(...currentChunk);
+            }
+            return finalArray;
+        }
+        if (obj && typeof obj === "object") {
+            const objHash = this.hashObject(obj);
+            const existingKey = this.deduplicationMap.get(objHash);
+            if (existingKey) {
+                return `[DEDUPLICATED_OBJECT]\nKey: ${existingKey}\nPath: ${path}\n[Use ${this.toolName} tool with key "${existingKey}" to retrieve content]`;
+            }
+            const seenCount = this.objectSeenCount.get(objHash) || 0;
+            this.objectSeenCount.set(objHash, seenCount + 1);
+            const isFirstOccurrence = seenCount === 1;
+            const objWithLowSignalCompressed = this.compressObjectWithLowSignalDetection(obj, path);
+            const objToProcess = objWithLowSignalCompressed !== obj ? objWithLowSignalCompressed : obj;
+            const result = {};
+            for (const [key, value] of Object.entries(objToProcess)) {
+                const newPath = path ? `${path}.${key}` : key;
+                result[key] = this.compressJsonProperties(value, newPath);
+            }
+            const objectAsString = JSON.stringify(result);
+            const tokens = this.storage.estimateTokens(objectAsString);
+            if (isFirstOccurrence && tokens > 100) {
+                const key = this.storage.generateKey();
+                this.deduplicationMap.set(objHash, key);
+                this.storage.storeString(key, objectAsString);
+                return result;
+            }
+            if (tokens > this.compressionThreshold) {
+                const key = this.storage.generateKey();
+                this.storage.storeString(key, objectAsString);
+                return `[COMPRESSED_JSON_OBJECT - ${tokens} tokens]\nKey: ${key}\nPath: ${path}\nKeys: ${Object.keys(result).join(", ")}\nPreview: ${objectAsString.substring(0, 200)}...\n[Use ${this.toolName} tool with key "${key}" to retrieve full content]`;
+            }
+            return result;
+        }
+        if (typeof obj === "string") {
+            const parsedJson = this.tryParseJson(obj);
+            if (parsedJson) {
+                const compressedJson = this.compressJsonProperties(parsedJson, path);
+                const compressedJsonString = JSON.stringify(compressedJson, null, 2);
+                const originalTokens = this.storage.estimateTokens(obj);
+                const compressedTokens = this.storage.estimateTokens(compressedJsonString);
+                if (compressedTokens < originalTokens * 0.8) {
+                    return compressedJsonString;
+                }
+            }
+            const tokens = this.storage.estimateTokens(obj);
+            if (tokens > this.compressionThreshold) {
+                const key = this.storage.generateKey();
+                this.storage.storeString(key, obj);
+                return `[COMPRESSED_JSON_PROPERTY - ${tokens} tokens]\nKey: ${key}\nPath: ${path}\nPreview: ${obj.substring(0, 200)}...\n[Use ${this.toolName} tool with key "${key}" to retrieve full content]`;
+            }
+            return obj;
+        }
+        return obj;
+    }
+}
+exports.JsonCompressor = JsonCompressor;
+//# sourceMappingURL=JsonCompressor.js.map

package/ts_build/src/processors/JsonCompressor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"JsonCompressor.js","sourceRoot":"","sources":["../../../src/processors/JsonCompressor.ts"],"names":[],"mappings":";;;AAgCA,MAAa,cAAc;IAEjB,gBAAgB,GAAwB,IAAI,GAAG,EAAE,CAAC;IAClD,eAAe,GAAwB,IAAI,GAAG,EAAE,CAAC;IACjD,gBAAgB,GAAwB,IAAI,GAAG,EAAE,CAAC;IAClD,sBAAsB,GAAwB,IAAI,GAAG,EAAE,CAAC;IAExD,oBAAoB,CAAS;IAC7B,SAAS,CAAS;IAClB,QAAQ,CAAS;IACjB,OAAO,CAAwB;IAEvC,YACE,OAA8B,EAC9B,oBAA4B,EAC5B,SAAiB,EACjB,QAAgB;QAEhB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAMD,kBAAkB;QAChB,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAAE,CAAC;IACtC,CAAC;IAKD,cAAc,CAAC,oBAA4B,EAAE,SAAiB;QAC5D,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAMD,YAAY,CAAC,OAAe;QAC1B,IAAI;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAGnC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;gBAC9B,IAAI;oBACF,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;iBAClC;gBAAC,OAAO,CAAC,EAAE;oBACV,OAAO,MAAM,CAAC;iBACf;aACF;YAGD,IAAI,MAAM;gBACN,OAAO,MAAM,KAAK,QAAQ;gBAC1B,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;gBAC7B,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBAE7B,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAGvC,IAAI,YAAY,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,YAAY,CAAC,IAAI,KAAK,QAAQ,EAAE;oBACzE,IAAI;wBAEF,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;wBAGxD,OAAO;4BACL,WAAW,EAAE,IAAI;4BACjB,cAAc,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;4BAC/C,KAAK,EAAE,UAAU;yBAClB,CAAC;qBACH;oBAAC,OAAO,CAAC,EAAE;wBAEV,OAAO,MAAM,CAAC;qBACf;iBACF;aACF;YAED,OAAO,MAAM,CAAC;SACf;QAAC,MAAM;YACN,OAAO,IAAI,CAAC;SACb;IACH,CAAC;IAMM,cAAc,CAAC,GAAQ,EAAE,WAAmB,CAAC,EAAE,eAAuB,CAAC;QAC5E,IAAI,YAAY,GAAG,QAAQ,EAAE;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB;QAGD,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,WAAW,KAAK,IAAI,IAAI,GAAG,CAAC,KAAK,EAAE;YAE3E,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;YAC1E,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE;oBACV,KAAK,EAAE,UAAU;iBAClB;aACF,CAAC;SACH;QAED,IAAI,GAAG,KAAK,IAAI,EAAE;YAChB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;SACzB;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YACtB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE;gBACpB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;aACtD;YAED,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/B,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC;YAE9F,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;SACjD;QAED,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;YAC3B,MAAM,UAAU,GAA+B,EAAE,CAAC;YAClD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;gBAC9C,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;aAC1E;YACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC;SACvC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,CAAC;IAC9B,CAAC;IAMO,mBAAmB,CAAC,IAAY,EAAE,IAAY;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,MAAM,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QAG7B,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE;YAC/B,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE;gBACvB,eAAe,EAAE,CAAC;aACnB;iBAAM;gBACL,MAAM;aACP;SACF;QAED,OAAO,eAAe,GAAG,MAAM,CAAC;IAClC,CAAC;IAKO,yBAAyB,CAAC,GAAQ;QACxC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YACzD,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;SACxC;QAED,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAwB,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAGpC,MAAM,UAAU,GAAG,cAAc,CAAC;QAClC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE;YAClC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBACvD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aACpB;SACF;QAGD,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE;YACxB,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAW,CAAC,CAAC;YAC1D,IAAI,eAAe,GAAG,CAAC,CAAC;YACxB,IAAI,WAAW,GAAG,CAAC,CAAC;YAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;gBAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBAC7C,eAAe,IAAI,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;oBACxE,WAAW,EAAE,CAAC;iBACf;aACF;YAED,MAAM,aAAa,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YAG1E,IAAI,aAAa,GAAG,GAAG,EAAE;gBACvB,SAAS,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC5B,QAAQ,CAAC,cAAc,GAAG,aAAa,CAAC;gBACxC,QAAQ,CAAC,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC;aACtC;SACF;QAGD,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;QAC1F,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE;YAClC,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACpF,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aACrB;SACF;QAED,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;IACjC,CAAC;IAMD,oCAAoC,CAAC,GAAQ,EAAE,OAAe,EAAE;QAC9D,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YACzD,OAAO,GAAG,CAAC;SACZ;QAED,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,CAAC;QAGpE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;YACxB,OAAO,GAAG,CAAC;SACZ;QAED,MAAM,UAAU,GAAQ,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAQ,EAAE,CAAC;QAE3B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAC9C,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBAC3B,UAAU,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;aACzB;iBAAM;gBACL,UAAU,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;aACzB;SACF;QAGD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QACnD,IAAI,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAE9D,IAAI,CAAC,aAAa,EAAE;YAElB,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC3C,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;YAEzD,MAAM,mBAAmB,GAAwB;gBAC/C,qBAAqB,EAAE,UAAU;gBACjC,kBAAkB,EAAE,sBAAsB;gBAC1C,gBAAgB,EAAE,QAAQ,CAAC,cAAc;aAC1C,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC,CAAC;SAC9E;QAMD,MAAM,iBAAiB,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,sBAAsB,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACvF,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,iBAAiB,EAAE,sBAAsB,GAAG,CAAC,CAAC,CAAC;QAE/E,IAAI,kBAAkB,GAAmB,SAAS,CAAC;QAEnD,IAAI,sBAAsB,KAAK,CAAC,EAAE;YAEhC,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACpD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;YAC/D,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtE,kBAAkB,GAAG,SAAS,CAAC;SAChC;aAAM,IAAI,sBAAsB,IAAI,CAAC,EAAE;YAEtC,MAAM,wBAAwB,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;YAC9E,kBAAkB,GAAG,8BAA8B,wBAAwB,EAAE,CAAC;SAC/E;QAGD,OAAO;YACL,GAAG,UAAU;YACb,0BAA0B,EAAE,aAAa;YACzC,0BAA0B,EAAE,kBAAkB;YAC9C,iBAAiB,EAAE,GAAG,SAAS,CAAC,MAAM,6EAA6E,aAAa,gBAAgB;SACjJ,CAAC;IACJ,CAAC;IAMO,UAAU,CAAC,GAAQ;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAEhE,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC1C,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;YACnC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;SACpB;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAMD,sBAAsB,CAAC,GAAQ,EAAE,OAAe,EAAE;QAChD,IACE,IAAI,KAAK,EAAE;YACX,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,EAClE;YACA,OAAO,GAAG,CAAC;SACZ;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAEtB,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAC7C,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,CACvD,CAAC;YAMF,MAAM,UAAU,GAAU,EAAE,CAAC;YAC7B,IAAI,YAAY,GAAU,EAAE,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE;gBACnD,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;gBAC/B,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAE3B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;gBACjD,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBAE7D,IAAI,WAAW,GAAG,IAAI,CAAC,oBAAoB,EAAE;oBAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;oBACvC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;oBAE3C,MAAM,IAAI,GAAG,kCAAkC,WAAW,YACxD,YAAY,CAAC,MACf,iBAAiB,GAAG,WAAW,IAAI,IAAI,CAAC,MACtC,CAAC,GAAG,YAAY,CAAC,MAAM,GAAG,CAC5B,eAAe,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,aAC1C,IAAI,CAAC,QACP,mBAAmB,GAAG,2BAA2B,CAAC;oBAClD,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBAEzB,YAAY,GAAG,EAAE,CAAC;iBACnB;aACF;YAID,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC3B,UAAU,CAAC,OAAO,CAAC,GAAG,YAAY,CAAC,CAAC;aACrC;YACD,OAAO,UAAU,CAAC;SACnB;QAID,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;YAElC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAEvD,IAAI,WAAW,EAAE;gBAEb,OAAO,+BAA+B,WAAW,WAAW,IAAI,UAAU,IAAI,CAAC,QAAQ,mBAAmB,WAAW,wBAAwB,CAAC;aACjJ;YAGD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;YAQjD,MAAM,iBAAiB,GAAG,SAAS,KAAK,CAAC,CAAC;YAG1C,MAAM,0BAA0B,GAAG,IAAI,CAAC,oCAAoC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACxF,MAAM,YAAY,GAAG,0BAA0B,KAAK,GAAG,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,GAAG,CAAC;YAE3F,MAAM,MAAM,GAAQ,EAAE,CAAC;YACvB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE;gBACvD,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;gBAC9C,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;aAC3D;YAGD,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;YAG3D,IAAI,iBAAiB,IAAI,MAAM,GAAG,GAAG,EAAE;gBACnC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBACvC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBACxC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;gBAE9C,OAAO,MAAM,CAAC;aACjB;YAGD,IAAI,MAAM,GAAG,IAAI,CAAC,oBAAoB,EAAE;gBACtC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBACvC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;gBAE9C,OAAO,6BAA6B,MAAM,kBAAkB,GAAG,WAAW,IAAI,WAAW,MAAM,CAAC,IAAI,CAClG,MAAM,CACP,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,cAAc,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,aACxD,IAAI,CAAC,QACP,mBAAmB,GAAG,6BAA6B,CAAC;aACrD;YACD,OAAO,MAAM,CAAC;SACf;QAID,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;YAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,UAAU,EAAE;gBACd,MAAM,cAAc,GAAG,IAAI,CAAC,sBAAsB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACrE,MAAM,oBAAoB,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAErE,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;gBACxD,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;gBAE3E,IAAI,gBAAgB,GAAG,cAAc,GAAG,GAAG,EAAE;oBAC3C,OAAO,oBAAoB,CAAC;iBAC7B;aACF;YAGD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,MAAM,GAAG,IAAI,CAAC,oBAAoB,EAAE;gBACtC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBACvC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBAEnC,OAAO,+BAA+B,MAAM,kBAAkB,GAAG,WAAW,IAAI,cAAc,GAAG,CAAC,SAAS,CACzG,CAAC,EACD,GAAG,CACJ,aACC,IAAI,CAAC,QACP,mBAAmB,GAAG,6BAA6B,CAAC;aACrD;YACD,OAAO,GAAG,CAAC;SACZ;QAED,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AA/cD,wCA+cC"}

package/ts_build/src/processors/TokenCompressor.d.ts

@@ -1,22 +1,35 @@
 import { Message, Tool } from "../clients/types";
 import { MessageProcessorFunction } from "../services/MessageProcessor";
 import { ToolsService } from "../services";
-export declare class TokenCompressor {
+import { JsonSchema, JsonCompressorStorage } from "./JsonCompressor";
+export interface KeyInfo {
+    key: string;
+    size: number;
+    preview: string;
+    tokens?: number;
+    type?: string;
+    depth?: number;
+    childKeys?: string[];
+    nextChunkKey?: string;
+}
+export declare class TokenCompressor implements JsonCompressorStorage {
     private storage;
     private keyPrefix;
     private toolName;
     private compressionThreshold;
     private characterLimit;
-    private maxTokens;
+    maxTokens: number;
+    private jsonCompressor;
     constructor(toolsService?: ToolsService);
-    private estimateTokens;
+    estimateTokens(text: string): number;
     setCompressionThreshold(threshold: number): void;
     private setMaxTokens;
-    private tryParseJson;
+    tryParseJson(content: string): any | null;
     compressStringInChunks(content: string, path?: string): string;
+    private isAlreadyCompressed;
     compressContent(content: string, path?: string): string;
     compressJsonProperties(obj: any, path?: string): any;
-    private generateKey;
+    generateKey(): string;
     compressMessage(message: Message): Promise<void>;
     createProcessor(filterFn?: (msg: Message) => boolean): MessageProcessorFunction;
     retrieveString(key: string): string | null;
@@ -25,5 +38,10 @@ export declare class TokenCompressor {
     getStorageKeys(): string[];
     getStorageSize(): number;
     registerTool(toolsService?: ToolsService): void;
+    getSchema(key: string): JsonSchema | null;
+    getCompressedProperties(key: string): any | null;
+    getFullObject(mainObj: any, compressedKey: string): any;
+    extractKeys(content: string): string[];
+    getKeyChain(key: string): KeyInfo[];
 }
 export declare const expandTokensDefinition: Tool;

package/ts_build/src/processors/TokenCompressor.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.expandTokensDefinition = exports.TokenCompressor = void 0;
+const JsonCompressor_1 = require("./JsonCompressor");
 class TokenCompressor {
     storage = {};
     keyPrefix = "compressed_";
@@ -8,7 +9,9 @@ class TokenCompressor {
     compressionThreshold = 4000;
     characterLimit = this.compressionThreshold * 4;
     maxTokens = this.compressionThreshold * 2;
+    jsonCompressor;
     constructor(toolsService) {
+        this.jsonCompressor = new JsonCompressor_1.JsonCompressor(this, this.compressionThreshold, this.maxTokens, this.toolName);
         this.registerTool(toolsService);
     }
     estimateTokens(text) {
@@ -17,19 +20,16 @@ class TokenCompressor {
     setCompressionThreshold(threshold) {
         this.compressionThreshold = threshold;
         this.characterLimit = threshold * 4;
+        this.jsonCompressor.updateSettings(threshold, this.maxTokens);
     }
     setMaxTokens(maxTokens) {
         if (maxTokens > this.maxTokens) {
             this.maxTokens = maxTokens;
+            this.jsonCompressor.updateSettings(this.compressionThreshold, maxTokens);
         }
     }
     tryParseJson(content) {
-        try {
-            return JSON.parse(content);
-        }
-        catch {
-            return null;
-        }
+        return this.jsonCompressor.tryParseJson(content);
     }
     compressStringInChunks(content, path = "") {
         if (content.length <= this.characterLimit) {
@@ -64,7 +64,20 @@ class TokenCompressor {
         const chunkCount = chunks.length;
         return `[COMPRESSED_STRING - ${totalTokens} tokens in ${chunkCount} chunks]\nKey: ${firstKey}\nPath: ${path}\nPreview: ${content.substring(0, 200)}...\n[Use ${this.toolName} tool with key "${firstKey}" to retrieve content. Follow NEXT_CHUNK_KEY references for complete content]`;
     }
+    isAlreadyCompressed(content) {
+        if (content.includes("[COMPRESSED_STRING")) {
+            return true;
+        }
+        const parsed = this.tryParseJson(content);
+        if (parsed && parsed._schema_key && typeof parsed._schema_key === "string") {
+            return true;
+        }
+        return false;
+    }
     compressContent(content, path = "") {
+        if (this.isAlreadyCompressed(content)) {
+            return content;
+        }
         const tokens = this.estimateTokens(content);
         const thresholdToUse = path === "" ? this.compressionThreshold : this.maxTokens;
         if (tokens <= thresholdToUse) {
@@ -72,77 +85,33 @@ class TokenCompressor {
         }
         const jsonObj = this.tryParseJson(content);
         if (jsonObj) {
-            const compressedObj = this.compressJsonProperties(jsonObj, path);
-            const compressedContent = JSON.stringify(compressedObj, null, 2);
+            const dataToCompress = jsonObj._mcp_format ? jsonObj._data : jsonObj;
+            const schema = this.jsonCompressor.generateSchema(jsonObj);
+            const schemaKey = this.generateKey();
+            this.storeString(schemaKey, JSON.stringify(schema));
+            const compressedObj = this.compressJsonProperties(dataToCompress, path || "data");
+            const finalCompressedObj = jsonObj._mcp_format
+                ? {
+                    _mcp_format: true,
+                    _raw_structure: jsonObj._raw_structure,
+                    _data: compressedObj,
+                }
+                : compressedObj;
+            const resultWithSchema = typeof finalCompressedObj === "object" &&
+                !Array.isArray(finalCompressedObj)
+                ? { ...finalCompressedObj, _schema_key: schemaKey }
+                : { _schema_key: schemaKey, data: finalCompressedObj };
+            const compressedContent = JSON.stringify(resultWithSchema, null, 2);
             const compressedTokens = this.estimateTokens(compressedContent);
-            if (compressedTokens < tokens * 0.8) {
+            const compressionThreshold = 0.6;
+            if (compressedTokens < tokens * compressionThreshold) {
                 return compressedContent;
             }
         }
         return this.compressStringInChunks(content, path);
     }
     compressJsonProperties(obj, path = "") {
-        if (path === "" &&
-            this.estimateTokens(JSON.stringify(obj)) <= this.maxTokens) {
-            return obj;
-        }
-        if (Array.isArray(obj)) {
-            const processedItems = obj.map((item, index) => this.compressJsonProperties(item, `${path}[${index}]`));
-            const finalArray = [];
-            let currentChunk = [];
-            for (let i = processedItems.length - 1; i >= 0; i--) {
-                const item = processedItems[i];
-                currentChunk.unshift(item);
-                const chunkString = JSON.stringify(currentChunk);
-                const chunkTokens = this.estimateTokens(chunkString);
-                if (chunkTokens > this.compressionThreshold) {
-                    const key = this.generateKey();
-                    this.storeString(key, chunkString);
-                    const stub = `[COMPRESSED_JSON_ARRAY_CHUNK - ${chunkTokens} tokens, ${currentChunk.length} items]\nKey: ${key}\nPath: ${path}[${i}...${i + currentChunk.length - 1}]\nPreview: ${chunkString.substring(0, 100)}...\n[Use ${this.toolName} tool with key "${key}" to retrieve this chunk]`;
-                    finalArray.unshift(stub);
-                    currentChunk = [];
-                }
-            }
-            if (currentChunk.length > 0) {
-                finalArray.unshift(...currentChunk);
-            }
-            return finalArray;
-        }
-        if (obj && typeof obj === "object") {
-            const result = {};
-            for (const [key, value] of Object.entries(obj)) {
-                const newPath = path ? `${path}.${key}` : key;
-                result[key] = this.compressJsonProperties(value, newPath);
-            }
-            const objectAsString = JSON.stringify(result);
-            const tokens = this.estimateTokens(objectAsString);
-            if (tokens > this.compressionThreshold) {
-                const key = this.generateKey();
-                this.storeString(key, objectAsString);
-                return `[COMPRESSED_JSON_OBJECT - ${tokens} tokens]\nKey: ${key}\nPath: ${path}\nKeys: ${Object.keys(result).join(", ")}\nPreview: ${objectAsString.substring(0, 200)}...\n[Use ${this.toolName} tool with key "${key}" to retrieve full content]`;
-            }
-            return result;
-        }
-        if (typeof obj === "string") {
-            const parsedJson = this.tryParseJson(obj);
-            if (parsedJson) {
-                const compressedJson = this.compressJsonProperties(parsedJson, path);
-                const compressedJsonString = JSON.stringify(compressedJson, null, 2);
-                const originalTokens = this.estimateTokens(obj);
-                const compressedTokens = this.estimateTokens(compressedJsonString);
-                if (compressedTokens < originalTokens * 0.8) {
-                    return compressedJsonString;
-                }
-            }
-            const tokens = this.estimateTokens(obj);
-            if (tokens > this.compressionThreshold) {
-                const key = this.generateKey();
-                this.storeString(key, obj);
-                return `[COMPRESSED_JSON_PROPERTY - ${tokens} tokens]\nKey: ${key}\nPath: ${path}\nPreview: ${obj.substring(0, 200)}...\n[Use ${this.toolName} tool with key "${key}" to retrieve full content]`;
-            }
-            return obj;
-        }
-        return obj;
+        return this.jsonCompressor.compressJsonProperties(obj, path);
     }
     generateKey() {
         return `${this.keyPrefix}${Date.now()}_${Math.random()
@@ -182,6 +151,7 @@ class TokenCompressor {
     }
     clearStorage() {
         this.storage = {};
+        this.jsonCompressor.clearDeduplication();
     }
     getStorageKeys() {
         return Object.keys(this.storage);
@@ -203,6 +173,72 @@ class TokenCompressor {
             });
         }
     }
+    getSchema(key) {
+        const schemaKey = `${key}_schema`;
+        const schemaStr = this.storage[schemaKey];
+        if (!schemaStr) {
+            return null;
+        }
+        try {
+            return JSON.parse(schemaStr);
+        }
+        catch (e) {
+            return null;
+        }
+    }
+    getCompressedProperties(key) {
+        const content = this.storage[key];
+        if (!content) {
+            return null;
+        }
+        try {
+            const metadata = JSON.parse(content);
+            return metadata.compressed_properties || null;
+        }
+        catch (e) {
+            return null;
+        }
+    }
+    getFullObject(mainObj, compressedKey) {
+        if (!mainObj || typeof mainObj !== "object") {
+            return mainObj;
+        }
+        const compressed = this.getCompressedProperties(compressedKey);
+        if (!compressed) {
+            return mainObj;
+        }
+        const { _compressed_properties_key, _compressed_property_names, _compression_info, ...highSignal } = mainObj;
+        return { ...highSignal, ...compressed };
+    }
+    extractKeys(content) {
+        const keys = [];
+        const keyPattern = /\$expandTokens\[([^\]]+)\]|Key:\s*([^\s\n]+)/g;
+        let match;
+        while ((match = keyPattern.exec(content)) !== null) {
+            const key = match[1] || match[2];
+            if (key && !keys.includes(key)) {
+                keys.push(key);
+            }
+        }
+        return keys;
+    }
+    getKeyChain(key) {
+        const chain = [];
+        let currentKey = key;
+        while (currentKey) {
+            const content = this.storage[currentKey];
+            if (!content)
+                break;
+            chain.push({
+                key: currentKey,
+                size: content.length,
+                preview: content.substring(0, 100),
+            });
+            const nextMatch = content.match(/NEXT_CHUNK_KEY:\s*([^\s\n]+)/);
+            currentKey = nextMatch ? nextMatch[1] : null;
+        }
+        return chain;
+    }
 }
 exports.TokenCompressor = TokenCompressor;
 exports.expandTokensDefinition = {