@kunal_dutt/token-lens 0.2.25 → 0.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +1 -1
- package/README.md +2 -1
- package/check-agent.mjs +14 -13
- package/gateway.mjs +73 -66
- package/package.json +1 -1
- package/sdk.mjs +2 -2
- package/server.mjs +86 -75
- package/token-lens.mcpb +0 -0
- package/types/src/core/compare.d.ts +22 -0
- package/types/src/core/data.d.ts +39 -0
- package/types/src/core/forecast.d.ts +43 -0
- package/types/src/core/index.d.ts +3 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kunal_dutt/token-lens",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.26",
|
|
4
4
|
"description": "Count and cut the tokens your prompts, chats and agents use, adapted to the model and task. MCP server and self-hosted gateway. Collects no data.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mcp",
|
package/sdk.mjs
CHANGED
|
@@ -205970,13 +205970,13 @@ var api = GptEncoding.getEncodingApi("o200k_base", () => bpe);
|
|
|
205970
205970
|
var { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, encodeChat, encodeChatGenerator, vocabularySize, setMergeCacheSize, clearMergeCache, estimateCost } = api;
|
|
205971
205971
|
|
|
205972
205972
|
// src/core/tokens.ts
|
|
205973
|
-
var WORD_RE = /[A-Za-z]+|\d+|[^\sA-Za-z\d]
|
|
205973
|
+
var WORD_RE = /[A-Za-z]+|\d+|[^\sA-Za-z\d]|\n[ \t]*/g;
|
|
205974
205974
|
function estimateTokens(text, charsPerToken = 4) {
|
|
205975
205975
|
if (!text) return 0;
|
|
205976
205976
|
const byChars = text.length / charsPerToken;
|
|
205977
205977
|
let pieces = 0;
|
|
205978
205978
|
for (const piece of text.match(WORD_RE) ?? []) {
|
|
205979
|
-
const size = /^[A-Za-z]/.test(piece) ? 6 : /^\d/.test(piece) ? 3 : 1;
|
|
205979
|
+
const size = /^[A-Za-z]/.test(piece) ? 6 : /^\d/.test(piece) ? 3 : piece[0] === "\n" ? 16 : 1;
|
|
205980
205980
|
pieces += Math.max(1, Math.ceil(piece.length / size));
|
|
205981
205981
|
}
|
|
205982
205982
|
return Math.max(1, Math.round(Math.max(byChars, pieces)));
|