llm-chat-msg-compressor 1.0.0 → 1.0.1

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2025
+Copyright (c) 2025 Sridharvn
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -1,5 +1,9 @@
 # llm-chat-msg-compressor 🚀
 
+[![NPM Version](https://img.shields.io/npm/v/llm-chat-msg-compressor.svg)](https://www.npmjs.com/package/llm-chat-msg-compressor)
+[![License](https://img.shields.io/npm/l/llm-chat-msg-compressor.svg)](https://github.com/Sridharvn/llm-chat-msg-compressor/blob/main/LICENSE)
+[![Build Status](https://github.com/Sridharvn/llm-chat-msg-compressor/actions/workflows/test.yml/badge.svg)](https://github.com/Sridharvn/llm-chat-msg-compressor/actions)
+
 Intelligent JSON optimizer for LLM APIs. Automatically reduces token usage by selecting the best compression strategy for your data payload.
 
 ## Features
@@ -66,3 +70,11 @@ optimize(data, {
 By default, the library is **Safe-by-Default**. It preserves all data types (including booleans), ensuring that downstream code (e.g., in your backend or strictly typed clients) works without modification.
 
 If you need maximum compression and your LLM can handle `1`/`0` instead of `true`/`false`, you can enable `unsafe: true`.
+
+## Contributing
+
+Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines and [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) for our code of conduct.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

package/dist/analyzer.js

@@ -3,8 +3,19 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Analyzer = void 0;
 class Analyzer {
     static analyze(data) {
-        const json = JSON.stringify(data);
-        const totalBytes = Buffer.byteLength(json, 'utf8');
+        // Pre-flight check for primitives or very small objects
+        if (data === null || typeof data !== 'object') {
+            return {
+                totalBytes: data === undefined ? 0 : Buffer.byteLength(JSON.stringify(data), 'utf8'),
+                arrayDensity: 0,
+                maxExampleArrayLength: 0,
+                nestingDepth: 0,
+                repeatedKeysEstimate: 0,
+                estimatedAbbrevSavings: 0,
+                estimatedSchemaSavings: 0
+            };
+        }
+        let totalBytes = 0;
         let arrayCount = 0;
         let objectCount = 0;
         let maxArrLen = 0;
@@ -18,17 +29,40 @@ class Analyzer {
             if (arr.length < 3 || typeof arr[0] !== 'object' || arr[0] === null)
                 return 0;
             // Check consistency of first few items
-            const keys = Object.keys(arr[0]);
-            const keyStr = keys.sort().join(',');
-            const sample = arr.slice(0, 5); // Check first 5 for speed
-            const isConsistent = sample.every(item => item && typeof item === 'object' && !Array.isArray(item) &&
-                Object.keys(item).sort().join(',') === keyStr);
+            const firstItem = arr[0];
+            const keys = Object.keys(firstItem);
+            const keyCount = keys.length;
+            if (keyCount === 0)
+                return 0;
+            const sampleSize = Math.min(arr.length, 5);
+            let isConsistent = true;
+            for (let i = 1; i < sampleSize; i++) {
+                const item = arr[i];
+                if (!item || typeof item !== 'object' || Array.isArray(item)) {
+                    isConsistent = false;
+                    break;
+                }
+                const itemKeys = Object.keys(item);
+                if (itemKeys.length !== keyCount) {
+                    isConsistent = false;
+                    break;
+                }
+                // Check if all keys from first item exist in this item
+                for (const key of keys) {
+                    if (!(key in item)) {
+                        isConsistent = false;
+                        break;
+                    }
+                }
+                if (!isConsistent)
+                    break;
+            }
             if (isConsistent) {
-                // Savings = (Items - 1) * (Sum of key lengths + overhead)
-                // Roughly: For N items, we write keys once instead of N times.
-                // Savings ~= (N - 1) * (total_key_chars + (keys.length * 3 chars for quotes/colon))
-                const keysLen = keys.reduce((sum, k) => sum + k.length, 0);
-                const perItemOverhead = keysLen + (keys.length * 2); // quotes + colon approx
+                let keysLen = 0;
+                for (const key of keys) {
+                    keysLen += key.length;
+                }
+                const perItemOverhead = keysLen + (keyCount * 2); // quotes + colon approx
                 return (arr.length - 1) * perItemOverhead;
             }
             return 0;
@@ -38,16 +72,42 @@ class Analyzer {
             if (Array.isArray(obj)) {
                 arrayCount++;
                 maxArrLen = Math.max(maxArrLen, obj.length);
+                totalBytes += 2; // []
+                if (obj.length > 1)
+                    totalBytes += obj.length - 1; // commas
                 // Check if this specific array offers schema savings
                 schemaSavings += calculateArraySchemaSavings(obj);
-                obj.forEach(i => traverse(i, currentDepth + 1));
+                for (let i = 0; i < obj.length; i++) {
+                    traverse(obj[i], currentDepth + 1);
+                }
             }
             else if (obj && typeof obj === 'object') {
                 objectCount++;
-                const keys = Object.keys(obj);
-                totalKeysCount += keys.length;
-                totalKeyLength += keys.reduce((sum, k) => sum + k.length, 0);
-                Object.values(obj).forEach(v => traverse(v, currentDepth + 1));
+                totalBytes += 2; // {}
+                let first = true;
+                for (const key in obj) {
+                    if (Object.prototype.hasOwnProperty.call(obj, key)) {
+                        if (!first)
+                            totalBytes += 1; // comma
+                        first = false;
+                        totalKeysCount++;
+                        totalKeyLength += key.length;
+                        totalBytes += key.length + 3; // "key":
+                        traverse(obj[key], currentDepth + 1);
+                    }
+                }
+            }
+            else {
+                // Primitive
+                if (typeof obj === 'string') {
+                    totalBytes += Buffer.byteLength(obj, 'utf8') + 2; // quotes
+                }
+                else if (typeof obj === 'number' || typeof obj === 'boolean') {
+                    totalBytes += String(obj).length;
+                }
+                else if (obj === null) {
+                    totalBytes += 4;
+                }
             }
         };
         traverse(data, 0);

package/dist/index.js

@@ -33,18 +33,32 @@ function restore(data) {
         return strat.decompress(data);
     }
     // Detect Schema Separation format anywhere in the structure
-    // We use a string check for existence of keys to decide if we should traverse.
-    if (JSON.stringify(data).includes('"$s"') && JSON.stringify(data).includes('"$d"')) {
+    if (hasSchemaMarker(data)) {
         const strat = new strategies_1.SchemaDataSeparationStrategy();
         return strat.decompress(data);
     }
     // Default: return as is
     return data;
 }
-function checkNestedSchema(obj) {
-    // Simple deep check (expensive, but safe for examples)
-    // In prod, rely on known structure
-    return JSON.stringify(obj).includes('"$s":') && JSON.stringify(obj).includes('"$d":');
+function hasSchemaMarker(obj) {
+    if (!obj || typeof obj !== 'object')
+        return false;
+    if (Array.isArray(obj)) {
+        for (let i = 0; i < obj.length; i++) {
+            if (hasSchemaMarker(obj[i]))
+                return true;
+        }
+        return false;
+    }
+    if ('$s' in obj && '$d' in obj)
+        return true;
+    for (const k in obj) {
+        if (Object.prototype.hasOwnProperty.call(obj, k)) {
+            if (hasSchemaMarker(obj[k]))
+                return true;
+        }
+    }
+    return false;
 }
 var optimizer_2 = require("./optimizer");
 Object.defineProperty(exports, "Optimizer", { enumerable: true, get: function () { return optimizer_2.Optimizer; } });

package/dist/optimizer.d.ts

@@ -5,6 +5,10 @@ export interface OptimizerOptions {
     unsafe?: boolean;
 }
 export declare class Optimizer {
+    private schemaStrat;
+    private abbrevStrat;
+    private ultraStratSafe;
+    private ultraStratUnsafe;
     private strategies;
     /**
      * Automatically selects and applies the best compression strategy

package/dist/optimizer.js

@@ -5,10 +5,14 @@ const strategies_1 = require("./strategies");
 const analyzer_1 = require("./analyzer");
 class Optimizer {
     constructor() {
+        this.schemaStrat = new strategies_1.SchemaDataSeparationStrategy();
+        this.abbrevStrat = new strategies_1.AbbreviatedKeysStrategy();
+        this.ultraStratSafe = new strategies_1.UltraCompactStrategy({ unsafe: false });
+        this.ultraStratUnsafe = new strategies_1.UltraCompactStrategy({ unsafe: true });
         this.strategies = [
-            new strategies_1.SchemaDataSeparationStrategy(),
-            new strategies_1.UltraCompactStrategy(),
-            new strategies_1.AbbreviatedKeysStrategy()
+            this.schemaStrat,
+            this.ultraStratSafe,
+            this.abbrevStrat
         ];
     }
     /**
@@ -20,31 +24,23 @@ class Optimizer {
         const metrics = analyzer_1.Analyzer.analyze(data);
         // 1. If too small, just minify
         if (metrics.totalBytes < thresholdBytes) {
-            console.log(`[Optimizer] Selected: minify (Size ${metrics.totalBytes} < ${thresholdBytes})`);
             return strategies_1.minify.compress(data);
         }
         // 2. Smart Strategy Selection
         // Compare estimated savings to pick the winner.
-        console.log(`[Optimizer] Analysis: SchemaSavings=${Math.round(metrics.estimatedSchemaSavings)} bytes, AbbrevSavings=${Math.round(metrics.estimatedAbbrevSavings)} bytes`);
         // Prefer SchemaSeparation if it saves MORE than AbbreviatedKeys (with a slight buffer for safety)
         // Schema Separation is "riskier" structure-wise (arrays vs maps), so we want it to be worth it.
         if (metrics.estimatedSchemaSavings > metrics.estimatedAbbrevSavings * 1.1) {
-            console.log('[Optimizer] Selected: schema-data-separation (Higher savings)');
-            const schemaStrat = new strategies_1.SchemaDataSeparationStrategy();
-            return schemaStrat.compress(data);
+            return this.schemaStrat.compress(data);
         }
         // 3. Fallback to UltraCompact if aggressive is set
         if (aggressive) {
-            console.log('[Optimizer] Selected: ultra-compact');
-            const ultra = new strategies_1.UltraCompactStrategy({ unsafe });
-            return ultra.compress(data);
+            return unsafe ? this.ultraStratUnsafe.compress(data) : this.ultraStratSafe.compress(data);
         }
         // 4. Default: Abbreviated Keys
         // If Schema Separation isn't significantly better, we default to this.
         // It handles mixed/nested payloads better and is "safer" structure-wise.
-        console.log('[Optimizer] Selected: abbreviated-keys');
-        const abbr = new strategies_1.AbbreviatedKeysStrategy();
-        return abbr.compress(data);
+        return this.abbrevStrat.compress(data);
     }
     /**
      * Helper to get a specific strategy
@@ -52,7 +48,13 @@ class Optimizer {
     getStrategy(name) {
         if (name === 'minify')
             return strategies_1.minify;
-        return this.strategies.find(s => s.name === name);
+        if (name === 'schema-data-separation')
+            return this.schemaStrat;
+        if (name === 'abbreviated-keys')
+            return this.abbrevStrat;
+        if (name === 'ultra-compact')
+            return this.ultraStratSafe; // Default to safe
+        return undefined;
     }
 }
 exports.Optimizer = Optimizer;

package/dist/strategies.d.ts

@@ -6,6 +6,10 @@ export interface CompressionStrategy {
     compress(data: any): any;
     decompress(data: any): any;
 }
+/**
+ * Shared utility for generating short keys (a, b, ... z, aa, ab ...)
+ */
+export declare const generateShortKey: (index: number) => string;
 /**
  * Strategy 1: Minify (Baseline)
  * Just standard JSON serialization (handled by default JSON.stringify)

package/dist/strategies.js

@@ -1,6 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UltraCompactStrategy = exports.SchemaDataSeparationStrategy = exports.AbbreviatedKeysStrategy = exports.minify = void 0;
+exports.UltraCompactStrategy = exports.SchemaDataSeparationStrategy = exports.AbbreviatedKeysStrategy = exports.minify = exports.generateShortKey = void 0;
+/**
+ * Shared utility for generating short keys (a, b, ... z, aa, ab ...)
+ */
+const generateShortKey = (index) => {
+    let shortKey = '';
+    let temp = index;
+    do {
+        shortKey = String.fromCharCode(97 + (temp % 26)) + shortKey;
+        temp = Math.floor(temp / 26) - 1;
+    } while (temp >= 0);
+    return shortKey;
+};
+exports.generateShortKey = generateShortKey;
 /**
  * Strategy 1: Minify (Baseline)
  * Just standard JSON serialization (handled by default JSON.stringify)
@@ -22,30 +35,30 @@ class AbbreviatedKeysStrategy {
         this.name = 'abbreviated-keys';
     }
     compress(data) {
-        // Implementation that returns { m: map, d: data }
         const keyMap = new Map();
-        const reverseMap = new Map();
         let counter = 0;
         const getShortKey = (key) => {
-            if (!keyMap.has(key)) {
-                let shortKey = '';
-                let temp = counter++;
-                do {
-                    shortKey = String.fromCharCode(97 + (temp % 26)) + shortKey;
-                    temp = Math.floor(temp / 26) - 1;
-                } while (temp >= 0);
+            let shortKey = keyMap.get(key);
+            if (shortKey === undefined) {
+                shortKey = (0, exports.generateShortKey)(counter++);
                 keyMap.set(key, shortKey);
-                reverseMap.set(shortKey, key);
             }
-            return keyMap.get(key);
+            return shortKey;
         };
         const traverse = (obj) => {
-            if (Array.isArray(obj))
-                return obj.map(traverse);
+            if (Array.isArray(obj)) {
+                const newArr = new Array(obj.length);
+                for (let i = 0; i < obj.length; i++) {
+                    newArr[i] = traverse(obj[i]);
+                }
+                return newArr;
+            }
             if (obj && typeof obj === 'object') {
                 const newObj = {};
                 for (const k in obj) {
-                    newObj[getShortKey(k)] = traverse(obj[k]);
+                    if (Object.prototype.hasOwnProperty.call(obj, k)) {
+                        newObj[getShortKey(k)] = traverse(obj[k]);
+                    }
                 }
                 return newObj;
             }
@@ -61,17 +74,26 @@ class AbbreviatedKeysStrategy {
         if (!pkg || !pkg.m || pkg.d === undefined)
             return pkg;
         const reverseMap = new Map();
-        for (const [k, v] of Object.entries(pkg.m)) {
-            reverseMap.set(v, k);
+        for (const k in pkg.m) {
+            if (Object.prototype.hasOwnProperty.call(pkg.m, k)) {
+                reverseMap.set(pkg.m[k], k);
+            }
         }
         const traverse = (obj) => {
-            if (Array.isArray(obj))
-                return obj.map(traverse);
+            if (Array.isArray(obj)) {
+                const newArr = new Array(obj.length);
+                for (let i = 0; i < obj.length; i++) {
+                    newArr[i] = traverse(obj[i]);
+                }
+                return newArr;
+            }
             if (obj && typeof obj === 'object') {
                 const newObj = {};
                 for (const k in obj) {
-                    const originalKey = reverseMap.get(k) || k;
-                    newObj[originalKey] = traverse(obj[k]);
+                    if (Object.prototype.hasOwnProperty.call(obj, k)) {
+                        const originalKey = reverseMap.get(k) || k;
+                        newObj[originalKey] = traverse(obj[k]);
+                    }
                 }
                 return newObj;
             }
@@ -94,11 +116,30 @@ class SchemaDataSeparationStrategy {
             if (Array.isArray(obj)) {
                 // Check if it's an array of objects
                 if (obj.length > 0 && typeof obj[0] === 'object' && obj[0] !== null && !Array.isArray(obj[0])) {
-                    const keys = Object.keys(obj[0]);
-                    const allMatch = obj.every(item => typeof item === 'object' &&
-                        item !== null &&
-                        !Array.isArray(item) &&
-                        JSON.stringify(Object.keys(item).sort()) === JSON.stringify(keys.sort()));
+                    const firstItem = obj[0];
+                    const keys = Object.keys(firstItem);
+                    const keyCount = keys.length;
+                    let allMatch = true;
+                    for (let i = 1; i < obj.length; i++) {
+                        const item = obj[i];
+                        if (typeof item !== 'object' || item === null || Array.isArray(item)) {
+                            allMatch = false;
+                            break;
+                        }
+                        const itemKeys = Object.keys(item);
+                        if (itemKeys.length !== keyCount) {
+                            allMatch = false;
+                            break;
+                        }
+                        for (const key of keys) {
+                            if (!(key in item)) {
+                                allMatch = false;
+                                break;
+                            }
+                        }
+                        if (!allMatch)
+                            break;
+                    }
                     if (allMatch) {
                         return {
                             $s: keys, // Schema
@@ -106,12 +147,18 @@ class SchemaDataSeparationStrategy {
                         };
                     }
                 }
-                return obj.map(traverse);
+                const newArr = new Array(obj.length);
+                for (let i = 0; i < obj.length; i++) {
+                    newArr[i] = traverse(obj[i]);
+                }
+                return newArr;
             }
             if (obj && typeof obj === 'object') {
                 const newObj = {};
                 for (const k in obj) {
-                    newObj[k] = traverse(obj[k]);
+                    if (Object.prototype.hasOwnProperty.call(obj, k)) {
+                        newObj[k] = traverse(obj[k]);
+                    }
                 }
                 return newObj;
             }
@@ -124,19 +171,30 @@ class SchemaDataSeparationStrategy {
             if (obj && typeof obj === 'object') {
                 if (obj.$s && obj.$d && Array.isArray(obj.$s) && Array.isArray(obj.$d)) {
                     const keys = obj.$s;
-                    return obj.$d.map((values) => {
+                    const dataArr = obj.$d;
+                    const result = new Array(dataArr.length);
+                    for (let i = 0; i < dataArr.length; i++) {
+                        const values = dataArr[i];
                         const item = {};
-                        keys.forEach((k, i) => {
-                            item[k] = traverse(values[i]);
-                        });
-                        return item;
-                    });
+                        for (let j = 0; j < keys.length; j++) {
+                            item[keys[j]] = traverse(values[j]);
+                        }
+                        result[i] = item;
+                    }
+                    return result;
+                }
+                if (Array.isArray(obj)) {
+                    const newArr = new Array(obj.length);
+                    for (let i = 0; i < obj.length; i++) {
+                        newArr[i] = traverse(obj[i]);
+                    }
+                    return newArr;
                 }
-                if (Array.isArray(obj))
-                    return obj.map(traverse);
                 const newObj = {};
                 for (const k in obj) {
-                    newObj[k] = traverse(obj[k]);
+                    if (Object.prototype.hasOwnProperty.call(obj, k)) {
+                        newObj[k] = traverse(obj[k]);
+                    }
                 }
                 return newObj;
             }
@@ -159,16 +217,12 @@ class UltraCompactStrategy {
         const keyMap = new Map();
         let counter = 0;
         const getShortKey = (key) => {
-            if (!keyMap.has(key)) {
-                let shortKey = '';
-                let temp = counter++;
-                do {
-                    shortKey = String.fromCharCode(97 + (temp % 26)) + shortKey;
-                    temp = Math.floor(temp / 26) - 1;
-                } while (temp >= 0);
+            let shortKey = keyMap.get(key);
+            if (shortKey === undefined) {
+                shortKey = (0, exports.generateShortKey)(counter++);
                 keyMap.set(key, shortKey);
             }
-            return keyMap.get(key);
+            return shortKey;
         };
         const traverse = (obj) => {
             // Bool optimization: Only if unsafe mode is enabled
@@ -178,12 +232,19 @@ class UltraCompactStrategy {
                 if (obj === false)
                     return 0;
             }
-            if (Array.isArray(obj))
-                return obj.map(traverse);
+            if (Array.isArray(obj)) {
+                const newArr = new Array(obj.length);
+                for (let i = 0; i < obj.length; i++) {
+                    newArr[i] = traverse(obj[i]);
+                }
+                return newArr;
+            }
             if (obj && typeof obj === 'object') {
                 const newObj = {};
                 for (const k in obj) {
-                    newObj[getShortKey(k)] = traverse(obj[k]);
+                    if (Object.prototype.hasOwnProperty.call(obj, k)) {
+                        newObj[getShortKey(k)] = traverse(obj[k]);
+                    }
                 }
                 return newObj;
             }
@@ -199,32 +260,26 @@ class UltraCompactStrategy {
         if (!pkg || !pkg.m || pkg.d === undefined)
             return pkg;
         const reverseMap = new Map();
-        for (const [k, v] of Object.entries(pkg.m)) {
-            reverseMap.set(v, k);
+        for (const k in pkg.m) {
+            if (Object.prototype.hasOwnProperty.call(pkg.m, k)) {
+                reverseMap.set(pkg.m[k], k);
+            }
         }
         const traverse = (obj) => {
-            // Removed automatic 1->true conversion to avoid corrupting numbers
-            // if (obj === 1) return true; 
-            // NOTE: Ultra Compact assumes usage where boolean restoration is preferred. 
-            // Strictly speaking, we lose type info between 1 and true if we just blindly map.
-            // For LLM context, usually 1/0 is fine. But for exact restoration, this is lossy for numbers.
-            // Let's make it smarter: logic handled by downstream consumers or accept fuzzy types.
-            // For now, let's NOT automatically convert 1->true to keep it safer, 
-            // unless we store type info effectively. 
-            // Actually, for LLM optimization, sending 1/0 is enough. 
-            // If we want exact restoration, we need a schema.
-            // Let's stick to key restoration for now, and leave values as is (1/0) or maybe keep booleans as is if size diff is minimal? 
-            // "true" is 4 bytes, "1" is 1 byte. 
-            // If we want to support full roundtrip without schema, we might skip bool optimization OR live with the type change.
-            // Let's keep 1/0 and NOT restore to boolean automatically to avoid corrupting actual numbers. 
-            // The user will receive 1/0 instead of true/false.
-            if (Array.isArray(obj))
-                return obj.map(traverse);
+            if (Array.isArray(obj)) {
+                const newArr = new Array(obj.length);
+                for (let i = 0; i < obj.length; i++) {
+                    newArr[i] = traverse(obj[i]);
+                }
+                return newArr;
+            }
             if (obj && typeof obj === 'object') {
                 const newObj = {};
                 for (const k in obj) {
-                    const originalKey = reverseMap.get(k) || k;
-                    newObj[originalKey] = traverse(obj[k]);
+                    if (Object.prototype.hasOwnProperty.call(obj, k)) {
+                        const originalKey = reverseMap.get(k) || k;
+                        newObj[originalKey] = traverse(obj[k]);
+                    }
                 }
                 return newObj;
             }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "llm-chat-msg-compressor",
-    "version": "1.0.0",
+    "version": "1.0.1",
     "description": "Intelligent JSON compression for LLM API optimization",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -35,9 +35,15 @@
         "type": "git",
         "url": "git+https://github.com/Sridharvn/llm-chat-msg-compressor.git"
     },
+    "bugs": {
+        "url": "https://github.com/Sridharvn/llm-chat-msg-compressor/issues"
+    },
+    "homepage": "https://github.com/Sridharvn/llm-chat-msg-compressor#readme",
+    "engines": {
+        "node": ">=18.0.0"
+    },
     "devDependencies": {
         "@types/jest": "^30.0.0",
-        "gpt-3-encoder": "^1.1.4",
         "jest": "^30.2.0",
         "ts-jest": "^29.4.6",
         "ts-node": "^10.9.2",