npm - @pasko70/pibo - Versions diffs - 3.4.2 → 3.4.3 - Mend

@pasko70/pibo 3.4.2 → 3.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (21) hide show

package/dist/apps/chat-vscode-web/index.html CHANGED Viewed

@@ -5,7 +5,7 @@
 		<meta name="viewport" content="width=device-width, initial-scale=1.0" />
 		<meta name="theme-color" content="#101d22" />
 		<title>Pibo</title>
-		<script type="module" crossorigin src="/apps/chat-vscode/assets/index-BD3Ogz9Z.js"></script>
+		<script type="module" crossorigin src="/apps/chat-vscode/assets/index-0oTGFHni.js"></script>
 		<link rel="stylesheet" crossorigin href="/apps/chat-vscode/assets/index-b18ZkEo0.css">
 	</head>
 	<body>

package/dist/core/session-router.js CHANGED Viewed

@@ -1552,6 +1552,7 @@ export class PiboSessionRouter {
                 return result;
             },
             statusResources,
+            getToolMetricTokenCalculation: () => loadPiboUserSettings().toolMetrics.tokenCalculation,
         });
         this.sessions.set(piboSession.id, session);
         return session;

package/dist/core/user-settings.js CHANGED Viewed

@@ -4,6 +4,7 @@ import { piboHomePath } from "./pibo-home.js";
 import { sanitizePreviewServerSettings } from "./preview-server-settings.js";
 import { sanitizeTelemetryRetentionSettings } from "./telemetry-retention-settings.js";
 import { sanitizeTelemetryStaleThresholdSettings } from "./telemetry-staleness.js";
+import { DEFAULT_TOOL_METRIC_TOKEN_CALCULATION, parseToolMetricTokenCalculation, } from "../shared/tool-call-token-settings.js";
 export const DEFAULT_USER_TIMEZONE = "UTC";
 export const DEFAULT_WEB_ANNOTATIONS_TOGGLE_SHORTCUT = "Alt+Shift+A";
 export const DEFAULT_TRANSCRIPTION_PROVIDER_ID = "openai-chatgpt";
@@ -64,6 +65,15 @@ export function sanitizeSpeechSettings(value) {
         : {};
     return { providerId: sanitizeTranscriptionProviderId(raw.providerId) ?? DEFAULT_SPEECH_PROVIDER_ID };
 }
+export function sanitizeToolMetricSettings(value) {
+    const raw = value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : {};
+    return {
+        tokenCalculation: parseToolMetricTokenCalculation(raw.tokenCalculation)
+            ?? DEFAULT_TOOL_METRIC_TOKEN_CALCULATION,
+    };
+}
 export function sanitizeTimezone(value) {
     if (typeof value !== "string")
         return undefined;
@@ -87,6 +97,7 @@ function sanitizeUserSettings(value) {
         shortcuts: sanitizeShortcutSettings(raw.shortcuts),
         transcription: sanitizeTranscriptionSettings(raw.transcription),
         speech: sanitizeSpeechSettings(raw.speech),
+        toolMetrics: sanitizeToolMetricSettings(raw.toolMetrics),
         previewServers: sanitizePreviewServerSettings(raw.previewServers),
         telemetryStaleThresholds: sanitizeTelemetryStaleThresholdSettings(raw.telemetryStaleThresholds),
         telemetryRetention: sanitizeTelemetryRetentionSettings(raw.telemetryRetention),

package/dist/shared/tool-call-metrics.js CHANGED Viewed

@@ -1,5 +1,10 @@
-/** Bounded structural walk; never tokenize, copy or scan large strings. */
-export function estimateToolPayloadTokens(payload) {
+import { createRequire } from "node:module";
+import { DEFAULT_TOOL_METRIC_TOKEN_CALCULATION, sanitizeToolMetricTokenCalculation, toolMetricTokenBasis, } from "./tool-call-token-settings.js";
+export const MAX_TIKTOKEN_PAYLOAD_CHARACTERS = 4_000_000;
+const require = createRequire(import.meta.url);
+let activeTokenizer;
+/** Bounded structural walk; character mode never tokenizes, copies, or scans large strings. */
+export function measureToolPayloadCharacters(payload) {
     let budget = 10_000;
     const seen = new WeakSet();
     function size(value, depth) {
@@ -26,7 +31,6 @@ export function estimateToolPayloadTokens(payload) {
         }
         else {
             const record = value;
-            // Binary/media payloads have no meaningful character-based token count.
             if (["image", "audio", "document", "resource", "image_url"].includes(String(record.type)))
                 return NaN;
             for (const key in record) {
@@ -44,7 +48,26 @@ export function estimateToolPayloadTokens(payload) {
     }
     try {
         const chars = size(payload, 0);
-        return Number.isFinite(chars) ? Math.ceil(chars / 4) : undefined;
+        return Number.isFinite(chars) ? chars : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+export function estimateToolPayloadTokens(payload, calculation = DEFAULT_TOOL_METRIC_TOKEN_CALCULATION) {
+    try {
+        const normalized = sanitizeToolMetricTokenCalculation(calculation);
+        if (normalized.method === "characters") {
+            const chars = measureToolPayloadCharacters(payload);
+            if (chars === undefined)
+                return undefined;
+            const tokens = Math.ceil(chars / normalized.factor);
+            return Number.isSafeInteger(tokens) ? tokens : undefined;
+        }
+        const text = serializableToolPayloadText(payload);
+        if (text === undefined)
+            return undefined;
+        return tiktokenPayloadTokens(text, normalized.encoding);
     }
     catch {
         // Diagnostics must not turn an otherwise successful tool into a failure.
@@ -52,26 +75,64 @@ export function estimateToolPayloadTokens(payload) {
     }
 }
 export class ToolCallMetricsCollector {
+    getCalculation;
     active = new Map();
+    constructor(getCalculation = () => DEFAULT_TOOL_METRIC_TOKEN_CALCULATION) {
+        this.getCalculation = getCalculation;
+    }
     start(id, args, now = performance.now()) {
         if (this.active.has(id))
             return;
-        this.active.set(id, { startedAt: now, inputTokens: estimateToolPayloadTokens(args) });
+        const calculation = this.currentCalculation();
+        this.active.set(id, {
+            startedAt: now,
+            inputTokens: estimateToolPayloadTokens(args, calculation),
+            calculation,
+        });
     }
     finish(id, result, now = performance.now()) {
         const started = this.active.get(id);
         this.active.delete(id);
+        const calculation = started?.calculation ?? this.currentCalculation();
         // Harness result metadata is not model-visible tool output.
         const output = result && typeof result === "object" && "content" in result
             ? result.content : result;
         return {
-            tokenBasis: "chars/4",
+            tokenBasis: toolMetricTokenBasis(calculation),
             durationMs: started ? Math.max(0, now - started.startedAt) : undefined,
             inputTokens: started?.inputTokens,
-            outputTokens: estimateToolPayloadTokens(output),
+            outputTokens: estimateToolPayloadTokens(output, calculation),
         };
     }
     clear() {
         this.active.clear();
     }
+    currentCalculation() {
+        try {
+            return sanitizeToolMetricTokenCalculation(this.getCalculation());
+        }
+        catch {
+            return DEFAULT_TOOL_METRIC_TOKEN_CALCULATION;
+        }
+    }
+}
+function serializableToolPayloadText(payload) {
+    const measuredCharacters = measureToolPayloadCharacters(payload);
+    if (measuredCharacters === undefined || measuredCharacters > MAX_TIKTOKEN_PAYLOAD_CHARACTERS)
+        return undefined;
+    if (typeof payload === "string")
+        return payload;
+    const serialized = JSON.stringify(payload);
+    return typeof serialized === "string" && serialized.length <= MAX_TIKTOKEN_PAYLOAD_CHARACTERS
+        ? serialized
+        : undefined;
+}
+function tiktokenPayloadTokens(text, encoding) {
+    if (activeTokenizer?.encoding !== encoding) {
+        activeTokenizer?.tokenizer.free();
+        const tiktoken = require("tiktoken");
+        activeTokenizer = { encoding, tokenizer: tiktoken.get_encoding(encoding) };
+    }
+    const tokens = activeTokenizer.tokenizer.encode_ordinary(text).length;
+    return Number.isSafeInteger(tokens) ? tokens : undefined;
 }

package/dist/shared/tool-call-token-settings.js ADDED Viewed

@@ -0,0 +1,45 @@
+export const TIKTOKEN_ENCODINGS = [
+    "o200k_base",
+    "cl100k_base",
+    "p50k_base",
+    "r50k_base",
+    "p50k_edit",
+    "gpt2",
+];
+export const DEFAULT_TOOL_METRIC_TOKEN_CALCULATION = {
+    method: "characters",
+    factor: 4,
+};
+export function parseToolMetricTokenCalculation(value) {
+    if (!isRecord(value))
+        return undefined;
+    if (value.method === "characters") {
+        return typeof value.factor === "number" && Number.isFinite(value.factor) && value.factor > 0
+            ? { method: "characters", factor: value.factor }
+            : undefined;
+    }
+    if (value.method === "tiktoken" && typeof value.encoding === "string" && TIKTOKEN_ENCODINGS.includes(value.encoding)) {
+        return { method: "tiktoken", encoding: value.encoding };
+    }
+    return undefined;
+}
+export function sanitizeToolMetricTokenCalculation(value) {
+    return parseToolMetricTokenCalculation(value) ?? DEFAULT_TOOL_METRIC_TOKEN_CALCULATION;
+}
+export function toolMetricTokenBasis(calculation) {
+    return calculation.method === "characters"
+        ? `characters/${calculation.factor}`
+        : `tiktoken/${calculation.encoding}`;
+}
+export function toolMetricTokenBasisLabel(basis) {
+    if (!basis)
+        return "—";
+    if (basis === "chars/4")
+        return "chars ÷ 4";
+    if (basis.startsWith("characters/"))
+        return `chars ÷ ${basis.slice("characters/".length)}`;
+    return `tiktoken · ${basis.slice("tiktoken/".length)}`;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/npm-shrinkwrap.json CHANGED Viewed

@@ -1,12 +1,12 @@
 {
   "name": "@pasko70/pibo",
-  "version": "3.4.2",
+  "version": "3.4.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@pasko70/pibo",
-      "version": "3.4.2",
+      "version": "3.4.3",
       "workspaces": [
         "packages/workflows"
       ],
@@ -39,6 +39,7 @@
         "react-virtuoso": "^4.18.6",
         "remark-gfm": "^4.0.1",
         "tailwindcss": "^4.2.4",
+        "tiktoken": "^1.0.22",
         "typebox": "1.1.38",
         "xstate": "5.31.1"
       },
@@ -11502,7 +11503,7 @@
       "license": "MIT",
       "peer": true,
       "funding": {
-        "type": "GitHub Sponsors \u2764",
+        "type": "GitHub Sponsors ❤",
         "url": "https://github.com/sponsors/dmonad"
       }
     },
@@ -11781,7 +11782,7 @@
         "node": ">=16"
       },
       "funding": {
-        "type": "GitHub Sponsors \u2764",
+        "type": "GitHub Sponsors ❤",
         "url": "https://github.com/sponsors/dmonad"
       }
     },
@@ -15811,6 +15812,12 @@
         "url": "https://bevry.me/fund"
       }
     },
+    "node_modules/tiktoken": {
+      "version": "1.0.22",
+      "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.22.tgz",
+      "integrity": "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA==",
+      "license": "MIT"
+    },
     "node_modules/tinyglobby": {
       "version": "0.2.17",
       "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz",
@@ -17073,6 +17080,16 @@
         "node": ">=20.0"
       }
     },
+    "node_modules/xstate": {
+      "version": "5.31.1",
+      "resolved": "https://registry.npmjs.org/xstate/-/xstate-5.31.1.tgz",
+      "integrity": "sha512-3P7t7GQ61BvLu+8Cj6Zq7rcS34vecL9pvfN2ucUWmIFIUG+rAREviOs4Xy4OO3BuJHSz6RLU8eqDXxSbVotjDQ==",
+      "license": "MIT",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/xstate"
+      }
+    },
     "node_modules/yallist": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
@@ -17132,7 +17149,7 @@
         "npm": ">=8.0.0"
       },
       "funding": {
-        "type": "GitHub Sponsors \u2764",
+        "type": "GitHub Sponsors ❤",
         "url": "https://github.com/sponsors/dmonad"
       }
     },
@@ -17204,14 +17221,6 @@
       "dependencies": {
         "xstate": "5.31.1"
       }
-    },
-    "packages/workflows/node_modules/xstate": {
-      "version": "5.31.1",
-      "license": "MIT",
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/xstate"
-      }
     }
   }
 }

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@pasko70/pibo",
-  "version": "3.4.2",
+  "version": "3.4.3",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -109,6 +109,7 @@
     "react-virtuoso": "^4.18.6",
     "remark-gfm": "^4.0.1",
     "tailwindcss": "^4.2.4",
+    "tiktoken": "^1.0.22",
     "typebox": "1.1.38",
     "xstate": "5.31.1"
   },