driftdetect-mcp 0.4.0 → 0.4.3

package/dist/infrastructure/token-estimator.js

@@ -0,0 +1,131 @@
+/**
+ * Token Estimator
+ *
+ * Estimates token count for responses to stay within context budgets.
+ * Uses heuristics based on typical tokenization patterns.
+ */
+export class TokenEstimator {
+    // Average characters per token for different content types
+    static CHARS_PER_TOKEN = {
+        english: 4,
+        code: 3.5,
+        json: 3,
+        mixed: 3.5,
+    };
+    // JSON structure overhead multiplier
+    static JSON_OVERHEAD = 1.15;
+    /**
+     * Estimate tokens for a string
+     */
+    estimate(text) {
+        if (!text)
+            return 0;
+        const contentType = this.detectContentType(text);
+        const charsPerToken = TokenEstimator.CHARS_PER_TOKEN[contentType];
+        // Base token count
+        let tokens = Math.ceil(text.length / charsPerToken);
+        // Add overhead for JSON structure
+        if (contentType === 'json') {
+            tokens = Math.ceil(tokens * TokenEstimator.JSON_OVERHEAD);
+        }
+        return tokens;
+    }
+    /**
+     * Estimate with detailed breakdown
+     */
+    estimateDetailed(text) {
+        if (!text) {
+            return { tokens: 0, confidence: 'high' };
+        }
+        const contentType = this.detectContentType(text);
+        const charsPerToken = TokenEstimator.CHARS_PER_TOKEN[contentType];
+        // Count different parts
+        const whitespaceCount = (text.match(/\s/g) || []).length;
+        const structureCount = (text.match(/[{}\[\]":,]/g) || []).length;
+        const textCount = text.length - whitespaceCount - structureCount;
+        // Estimate each part
+        const textTokens = Math.ceil(textCount / charsPerToken);
+        const structureTokens = Math.ceil(structureCount / 2); // Structure chars are often single tokens
+        const whitespaceTokens = Math.ceil(whitespaceCount / 10); // Whitespace is heavily compressed
+        const totalTokens = textTokens + structureTokens + whitespaceTokens;
+        // Confidence based on content type
+        const confidence = contentType === 'json' ? 'high' :
+            contentType === 'code' ? 'medium' : 'medium';
+        return {
+            tokens: totalTokens,
+            confidence,
+            breakdown: {
+                text: textTokens,
+                structure: structureTokens,
+                whitespace: whitespaceTokens,
+            },
+        };
+    }
+    /**
+     * Estimate tokens for an object (will be JSON serialized)
+     */
+    estimateObject(obj) {
+        return this.estimate(JSON.stringify(obj, null, 2));
+    }
+    /**
+     * Check if content fits within a token budget
+     */
+    fitsInBudget(text, budget) {
+        return this.estimate(text) <= budget;
+    }
+    /**
+     * Truncate text to fit within token budget
+     */
+    truncateToFit(text, budget, suffix = '...[truncated]') {
+        const currentTokens = this.estimate(text);
+        if (currentTokens <= budget) {
+            return text;
+        }
+        // Estimate how much to keep
+        const ratio = budget / currentTokens;
+        const targetLength = Math.floor(text.length * ratio * 0.9); // 10% safety margin
+        const suffixLength = suffix.length;
+        return text.slice(0, targetLength - suffixLength) + suffix;
+    }
+    /**
+     * Detect content type for better estimation
+     */
+    detectContentType(text) {
+        // Check for JSON
+        if (text.trim().startsWith('{') || text.trim().startsWith('[')) {
+            try {
+                JSON.parse(text);
+                return 'json';
+            }
+            catch {
+                // Not valid JSON, continue checking
+            }
+        }
+        // Check for code indicators
+        const codeIndicators = [
+            /function\s+\w+/,
+            /const\s+\w+\s*=/,
+            /class\s+\w+/,
+            /import\s+.*from/,
+            /def\s+\w+\(/,
+            /=>/,
+        ];
+        const codeMatches = codeIndicators.filter(pattern => pattern.test(text)).length;
+        if (codeMatches >= 2) {
+            return 'code';
+        }
+        // Check JSON-like structure ratio
+        const structureChars = (text.match(/[{}\[\]":,]/g) || []).length;
+        const structureRatio = structureChars / text.length;
+        if (structureRatio > 0.1) {
+            return 'json';
+        }
+        // Default to mixed
+        return 'mixed';
+    }
+}
+/**
+ * Singleton instance for convenience
+ */
+export const tokenEstimator = new TokenEstimator();
+//# sourceMappingURL=token-estimator.js.map

package/dist/infrastructure/token-estimator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-estimator.js","sourceRoot":"","sources":["../../src/infrastructure/token-estimator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAYH,MAAM,OAAO,cAAc;IACzB,2DAA2D;IACnD,MAAM,CAAU,eAAe,GAAG;QACxC,OAAO,EAAE,CAAC;QACV,IAAI,EAAE,GAAG;QACT,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,GAAG;KACX,CAAC;IAEF,qCAAqC;IAC7B,MAAM,CAAU,aAAa,GAAG,IAAI,CAAC;IAE7C;;OAEG;IACH,QAAQ,CAAC,IAAY;QACnB,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAEpB,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,aAAa,GAAG,cAAc,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAElE,mBAAmB;QACnB,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;QAEpD,kCAAkC;QAClC,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,IAAY;QAC3B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAC3C,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,aAAa,GAAG,cAAc,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAElE,wBAAwB;QACxB,MAAM,eAAe,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QACzD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,eAAe,GAAG,cAAc,CAAC;QAEjE,qBAAqB;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,aAAa,CAAC,CAAC;QACxD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,0CAA0C;QACjG,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC,CAAC,mCAAmC;QAE7F,MAAM,WAAW,GAAG,UAAU,GAAG,eAAe,GAAG,gBAAgB,CAAC;QAEpE,mCAAmC;QACnC,MAAM,UAAU,GAAG,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACjC,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QAEhE,OAAO;YACL,MAAM,EAAE,WAAW;YACnB,UAAU;YACV,SAAS,EAAE;gBACT,IAAI,EAAE,UAAU;gBAChB,SAAS,EAAE,eAAe;gBAC1B,UAAU,EAAE,gBAAgB;aAC7B;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,GAAY;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,IAAY,EAAE,MAAc;QACvC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,IAAY,EAAE,MAAc,EAAE,SAAiB,gBAAgB;QAC3E,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE1C,IAAI,aAAa,IAAI,MAAM,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,4BAA4B;QAC5B,MAAM,KAAK,GAAG,MAAM,GAAG,aAAa,CAAC;QACrC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,oBAAoB;QAChF,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;QAEnC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,GAAG,YAAY,CAAC,GAAG,MAAM,CAAC;IAC7D,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,IAAY;QACpC,iBAAiB;QACjB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC;gBACH,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjB,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,MAAM,CAAC;gBACP,oCAAoC;YACtC,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,MAAM,cAAc,GAAG;YACrB,gBAAgB;YAChB,iBAAiB;YACjB,aAAa;YACb,iBAAiB;YACjB,aAAa;YACb,IAAI;SACL,CAAC;QAEF,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;QAChF,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;YACrB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,kCAAkC;QAClC,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,cAAc,GAAG,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;QAEpD,IAAI,cAAc,GAAG,GAAG,EAAE,CAAC;YACzB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,mBAAmB;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;;AAGH;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC"}

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AA8BnE,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAqYD,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CA4JnE"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AAsCnE,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AA8eD,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAwLnE"}