codebuff 1.0.196 → 1.0.197

package/dist/common/util/token-counter.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.countTokens = countTokens;
+exports.countTokensJson = countTokensJson;
+exports.countTokensForFiles = countTokensForFiles;
+const gpt_4o_1 = require("gpt-tokenizer/esm/model/gpt-4o");
+const ANTHROPIC_TOKEN_FUDGE_FACTOR = 1.35;
+function countTokens(text) {
+    try {
+        return Math.floor((0, gpt_4o_1.encode)(text).length * ANTHROPIC_TOKEN_FUDGE_FACTOR);
+    }
+    catch (e) {
+        console.error('Error counting tokens', e);
+        return Math.ceil(text.length / 3);
+    }
+}
+function countTokensJson(text) {
+    return countTokens(JSON.stringify(text));
+}
+function countTokensForFiles(files) {
+    const tokenCounts = {};
+    for (const [filePath, content] of Object.entries(files)) {
+        tokenCounts[filePath] = content ? countTokens(content) : 0;
+    }
+    return tokenCounts;
+}
+//# sourceMappingURL=token-counter.js.map

package/dist/common/util/token-counter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-counter.js","sourceRoot":"","sources":["../../src/util/token-counter.ts"],"names":[],"mappings":";;AAIA,kCAOC;AAED,0CAEC;AAED,kDAQC;AAzBD,2DAAuD;AAEvD,MAAM,4BAA4B,GAAG,IAAI,CAAA;AAEzC,SAAgB,WAAW,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAA,eAAM,EAAC,IAAI,CAAC,CAAC,MAAM,GAAG,4BAA4B,CAAC,CAAA;IACvE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAA;QACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACnC,CAAC;AACH,CAAC;AAED,SAAgB,eAAe,CAAC,IAAqB;IACnD,OAAO,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAA;AAC1C,CAAC;AAED,SAAgB,mBAAmB,CACjC,KAAoC;IAEpC,MAAM,WAAW,GAA2B,EAAE,CAAA;IAC9C,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxD,WAAW,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5D,CAAC;IACD,OAAO,WAAW,CAAA;AACpB,CAAC"}