@hibanacloud/core 0.3.15

package/dist/src/utils/request-tokenizer/requestTokenizer.js

@@ -0,0 +1,263 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { TextTokenizer } from './textTokenizer.js';
+import { ImageTokenizer } from './imageTokenizer.js';
+/**
+ * Simple request tokenizer that handles text and image content serially
+ */
+export class DefaultRequestTokenizer {
+    textTokenizer;
+    imageTokenizer;
+    constructor() {
+        this.textTokenizer = new TextTokenizer();
+        this.imageTokenizer = new ImageTokenizer();
+    }
+    /**
+     * Calculate tokens for a request using serial processing
+     */
+    async calculateTokens(request, config = {}) {
+        const startTime = performance.now();
+        // Apply configuration
+        if (config.textEncoding) {
+            this.textTokenizer = new TextTokenizer(config.textEncoding);
+        }
+        try {
+            // Process request content and group by type
+            const { textContents, imageContents, audioContents, otherContents } = this.processAndGroupContents(request);
+            if (textContents.length === 0 &&
+                imageContents.length === 0 &&
+                audioContents.length === 0 &&
+                otherContents.length === 0) {
+                return {
+                    totalTokens: 0,
+                    breakdown: {
+                        textTokens: 0,
+                        imageTokens: 0,
+                        audioTokens: 0,
+                        otherTokens: 0,
+                    },
+                    processingTime: performance.now() - startTime,
+                };
+            }
+            // Calculate tokens for each content type serially
+            const textTokens = await this.calculateTextTokens(textContents);
+            const imageTokens = await this.calculateImageTokens(imageContents);
+            const audioTokens = await this.calculateAudioTokens(audioContents);
+            const otherTokens = await this.calculateOtherTokens(otherContents);
+            const totalTokens = textTokens + imageTokens + audioTokens + otherTokens;
+            const processingTime = performance.now() - startTime;
+            return {
+                totalTokens,
+                breakdown: {
+                    textTokens,
+                    imageTokens,
+                    audioTokens,
+                    otherTokens,
+                },
+                processingTime,
+            };
+        }
+        catch (error) {
+            console.error('Error calculating tokens:', error);
+            // Fallback calculation
+            const fallbackTokens = this.calculateFallbackTokens(request);
+            return {
+                totalTokens: fallbackTokens,
+                breakdown: {
+                    textTokens: fallbackTokens,
+                    imageTokens: 0,
+                    audioTokens: 0,
+                    otherTokens: 0,
+                },
+                processingTime: performance.now() - startTime,
+            };
+        }
+    }
+    /**
+     * Calculate tokens for text contents
+     */
+    async calculateTextTokens(textContents) {
+        if (textContents.length === 0)
+            return 0;
+        try {
+            const tokenCounts = await this.textTokenizer.calculateTokensBatch(textContents);
+            return tokenCounts.reduce((sum, count) => sum + count, 0);
+        }
+        catch (error) {
+            console.warn('Error calculating text tokens:', error);
+            // Fallback: character-based estimation
+            const totalChars = textContents.join('').length;
+            return Math.ceil(totalChars / 4);
+        }
+    }
+    /**
+     * Calculate tokens for image contents using serial processing
+     */
+    async calculateImageTokens(imageContents) {
+        if (imageContents.length === 0)
+            return 0;
+        try {
+            const tokenCounts = await this.imageTokenizer.calculateTokensBatch(imageContents);
+            return tokenCounts.reduce((sum, count) => sum + count, 0);
+        }
+        catch (error) {
+            console.warn('Error calculating image tokens:', error);
+            // Fallback: minimum tokens per image
+            return imageContents.length * 6; // 4 image tokens + 2 special tokens as minimum
+        }
+    }
+    /**
+     * Calculate tokens for audio contents
+     * TODO: Implement proper audio token calculation
+     */
+    async calculateAudioTokens(audioContents) {
+        if (audioContents.length === 0)
+            return 0;
+        // Placeholder implementation - audio token calculation would depend on
+        // the specific model's audio processing capabilities
+        // For now, estimate based on data size
+        let totalTokens = 0;
+        for (const audioContent of audioContents) {
+            try {
+                const dataSize = Math.floor(audioContent.data.length * 0.75); // Approximate binary size
+                // Rough estimate: 1 token per 100 bytes of audio data
+                totalTokens += Math.max(Math.ceil(dataSize / 100), 10); // Minimum 10 tokens per audio
+            }
+            catch (error) {
+                console.warn('Error calculating audio tokens:', error);
+                totalTokens += 10; // Fallback minimum
+            }
+        }
+        return totalTokens;
+    }
+    /**
+     * Calculate tokens for other content types (functions, files, etc.)
+     */
+    async calculateOtherTokens(otherContents) {
+        if (otherContents.length === 0)
+            return 0;
+        try {
+            // Treat other content as text for token calculation
+            const tokenCounts = await this.textTokenizer.calculateTokensBatch(otherContents);
+            return tokenCounts.reduce((sum, count) => sum + count, 0);
+        }
+        catch (error) {
+            console.warn('Error calculating other content tokens:', error);
+            // Fallback: character-based estimation
+            const totalChars = otherContents.join('').length;
+            return Math.ceil(totalChars / 4);
+        }
+    }
+    /**
+     * Fallback token calculation using simple string serialization
+     */
+    calculateFallbackTokens(request) {
+        try {
+            const content = JSON.stringify(request.contents);
+            return Math.ceil(content.length / 4); // Rough estimate: 1 token ≈ 4 characters
+        }
+        catch (error) {
+            console.warn('Error in fallback token calculation:', error);
+            return 100; // Conservative fallback
+        }
+    }
+    /**
+     * Process request contents and group by type
+     */
+    processAndGroupContents(request) {
+        const textContents = [];
+        const imageContents = [];
+        const audioContents = [];
+        const otherContents = [];
+        if (!request.contents) {
+            return { textContents, imageContents, audioContents, otherContents };
+        }
+        const contents = Array.isArray(request.contents)
+            ? request.contents
+            : [request.contents];
+        for (const content of contents) {
+            this.processContent(content, textContents, imageContents, audioContents, otherContents);
+        }
+        return { textContents, imageContents, audioContents, otherContents };
+    }
+    /**
+     * Process a single content item and add to appropriate arrays
+     */
+    processContent(content, textContents, imageContents, audioContents, otherContents) {
+        if (typeof content === 'string') {
+            if (content.trim()) {
+                textContents.push(content);
+            }
+            return;
+        }
+        if ('parts' in content && content.parts) {
+            for (const part of content.parts) {
+                this.processPart(part, textContents, imageContents, audioContents, otherContents);
+            }
+        }
+    }
+    /**
+     * Process a single part and add to appropriate arrays
+     */
+    processPart(part, textContents, imageContents, audioContents, otherContents) {
+        if (typeof part === 'string') {
+            if (part.trim()) {
+                textContents.push(part);
+            }
+            return;
+        }
+        if ('text' in part && part.text) {
+            textContents.push(part.text);
+            return;
+        }
+        if ('inlineData' in part && part.inlineData) {
+            const { data, mimeType } = part.inlineData;
+            if (mimeType && mimeType.startsWith('image/')) {
+                imageContents.push({ data: data || '', mimeType });
+                return;
+            }
+            if (mimeType && mimeType.startsWith('audio/')) {
+                audioContents.push({ data: data || '', mimeType });
+                return;
+            }
+        }
+        if ('fileData' in part && part.fileData) {
+            otherContents.push(JSON.stringify(part.fileData));
+            return;
+        }
+        if ('functionCall' in part && part.functionCall) {
+            otherContents.push(JSON.stringify(part.functionCall));
+            return;
+        }
+        if ('functionResponse' in part && part.functionResponse) {
+            otherContents.push(JSON.stringify(part.functionResponse));
+            return;
+        }
+        // Unknown part type - try to serialize
+        try {
+            const serialized = JSON.stringify(part);
+            if (serialized && serialized !== '{}') {
+                otherContents.push(serialized);
+            }
+        }
+        catch (error) {
+            console.warn('Failed to serialize unknown part type:', error);
+        }
+    }
+    /**
+     * Dispose of resources
+     */
+    async dispose() {
+        try {
+            // Dispose of tokenizers
+            this.textTokenizer.dispose();
+        }
+        catch (error) {
+            console.warn('Error disposing request tokenizer:', error);
+        }
+    }
+}
+//# sourceMappingURL=requestTokenizer.js.map

package/dist/src/utils/request-tokenizer/requestTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"requestTokenizer.js","sourceRoot":"","sources":["../../../../src/utils/request-tokenizer/requestTokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAaH,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD;;GAEG;AACH,MAAM,OAAO,uBAAuB;IAC1B,aAAa,CAAgB;IAC7B,cAAc,CAAiB;IAEvC;QACE,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;QACzC,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CACnB,OAA8B,EAC9B,SAA0B,EAAE;QAE5B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,sBAAsB;QACtB,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACxB,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,CAAC;YACH,4CAA4C;YAC5C,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,GACjE,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAExC,IACE,YAAY,CAAC,MAAM,KAAK,CAAC;gBACzB,aAAa,CAAC,MAAM,KAAK,CAAC;gBAC1B,aAAa,CAAC,MAAM,KAAK,CAAC;gBAC1B,aAAa,CAAC,MAAM,KAAK,CAAC,EAC1B,CAAC;gBACD,OAAO;oBACL,WAAW,EAAE,CAAC;oBACd,SAAS,EAAE;wBACT,UAAU,EAAE,CAAC;wBACb,WAAW,EAAE,CAAC;wBACd,WAAW,EAAE,CAAC;wBACd,WAAW,EAAE,CAAC;qBACf;oBACD,cAAc,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;iBAC9C,CAAC;YACJ,CAAC;YAED,kDAAkD;YAClD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;YAChE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YACnE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YACnE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YAEnE,MAAM,WAAW,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,GAAG,WAAW,CAAC;YACzE,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAErD,OAAO;gBACL,WAAW;gBACX,SAAS,EAAE;oBACT,UAAU;oBACV,WAAW;oBACX,WAAW;oBACX,WAAW;iBACZ;gBACD,cAAc;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;YAElD,uBAAuB;YACvB,MAAM,cAAc,GAAG,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAE7D,OAAO;gBACL,WAAW,EAAE,cAAc;gBAC3B,SAAS,EAAE;oBACT,UAAU,EAAE,cAAc;oBAC1B,WAAW,EAAE,CAAC;oBACd,WAAW,EAAE,CAAC;oBACd,WAAW,EAAE,CAAC;iBACf;gBACD,cAAc,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;aAC9C,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,YAAsB;QACtD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAExC,IAAI,CAAC;YACH,MAAM,WAAW,GACf,MAAM,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAC9D,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YACtD,uCAAuC;YACvC,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;YAChD,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAChC,aAAwD;QAExD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,WAAW,GACf,MAAM,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YAChE,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAC;YACvD,qCAAqC;YACrC,OAAO,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,+CAA+C;QAClF,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,oBAAoB,CAChC,aAAwD;QAExD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEzC,uEAAuE;QACvE,qDAAqD;QACrD,uCAAuC;QACvC,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,0BAA0B;gBACxF,sDAAsD;gBACtD,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,8BAA8B;YACxF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAC;gBACvD,WAAW,IAAI,EAAE,CAAC,CAAC,mBAAmB;YACxC,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAAC,aAAuB;QACxD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEzC,IAAI,CAAC;YACH,oDAAoD;YACpD,MAAM,WAAW,GACf,MAAM,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YAC/D,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;YAC/D,uCAAuC;YACvC,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;YACjD,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,uBAAuB,CAAC,OAA8B;QAC5D,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACjD,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,yCAAyC;QACjF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,KAAK,CAAC,CAAC;YAC5D,OAAO,GAAG,CAAC,CAAC,wBAAwB;QACtC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,uBAAuB,CAAC,OAA8B;QAM5D,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,MAAM,aAAa,GAA8C,EAAE,CAAC;QACpE,MAAM,aAAa,GAA8C,EAAE,CAAC;QACpE,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC;QACvE,CAAC;QAED,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;YAC9C,CAAC,CAAC,OAAO,CAAC,QAAQ;YAClB,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CACjB,OAAO,EACP,YAAY,EACZ,aAAa,EACb,aAAa,EACb,aAAa,CACd,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC;IACvE,CAAC;IAED;;OAEG;IACK,cAAc,CACpB,OAAqC,EACrC,YAAsB,EACtB,aAAwD,EACxD,aAAwD,EACxD,aAAuB;QAEvB,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC7B,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,OAAO,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACxC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,WAAW,CACd,IAAI,EACJ,YAAY,EACZ,aAAa,EACb,aAAa,EACb,aAAa,CACd,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,WAAW,CACjB,IAAmB,EACnB,YAAsB,EACtB,aAAwD,EACxD,aAAwD,EACxD,aAAuB;QAEvB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBAChB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAChC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO;QACT,CAAC;QAED,IAAI,YAAY,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC;YAC3C,IAAI,QAAQ,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9C,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACnD,OAAO;YACT,CAAC;YACD,IAAI,QAAQ,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9C,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACnD,OAAO;YACT,CAAC;QACH,CAAC;QAED,IAAI,UAAU,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACxC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAClD,OAAO;QACT,CAAC;QAED,IAAI,cAAc,IAAI,IAAI,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAChD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YACtD,OAAO;QACT,CAAC;QAED,IAAI,kBAAkB,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAC1D,OAAO;QACT,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,UAAU,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;gBACtC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,CAAC;YACH,wBAAwB;YACxB,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;CACF"}

package/dist/src/utils/request-tokenizer/requestTokenizer.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};

package/dist/src/utils/request-tokenizer/requestTokenizer.test.js

@@ -0,0 +1,245 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { DefaultRequestTokenizer } from './requestTokenizer.js';
+describe('DefaultRequestTokenizer', () => {
+    let tokenizer;
+    beforeEach(() => {
+        tokenizer = new DefaultRequestTokenizer();
+    });
+    afterEach(async () => {
+        await tokenizer.dispose();
+    });
+    describe('text token calculation', () => {
+        it('should calculate tokens for simple text content', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [{ text: 'Hello, world!' }],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThan(0);
+            expect(result.breakdown.textTokens).toBeGreaterThan(0);
+            expect(result.breakdown.imageTokens).toBe(0);
+            expect(result.processingTime).toBeGreaterThan(0);
+        });
+        it('should handle multiple text parts', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            { text: 'First part' },
+                            { text: 'Second part' },
+                            { text: 'Third part' },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThan(0);
+            expect(result.breakdown.textTokens).toBeGreaterThan(0);
+        });
+        it('should handle string content', async () => {
+            const request = {
+                model: 'test-model',
+                contents: ['Simple string content'],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThan(0);
+            expect(result.breakdown.textTokens).toBeGreaterThan(0);
+        });
+    });
+    describe('image token calculation', () => {
+        it('should calculate tokens for image content', async () => {
+            // Create a simple 1x1 PNG image in base64
+            const pngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChAI9jU77yQAAAABJRU5ErkJggg==';
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            {
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: pngBase64,
+                                },
+                            },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThanOrEqual(4); // Minimum 4 tokens per image
+            expect(result.breakdown.imageTokens).toBeGreaterThanOrEqual(4);
+            expect(result.breakdown.textTokens).toBe(0);
+        });
+        it('should handle multiple images', async () => {
+            const pngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChAI9jU77yQAAAABJRU5ErkJggg==';
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            {
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: pngBase64,
+                                },
+                            },
+                            {
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: pngBase64,
+                                },
+                            },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThanOrEqual(8); // At least 4 tokens per image
+            expect(result.breakdown.imageTokens).toBeGreaterThanOrEqual(8);
+        });
+    });
+    describe('mixed content', () => {
+        it('should handle text and image content together', async () => {
+            const pngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChAI9jU77yQAAAABJRU5ErkJggg==';
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            { text: 'Here is an image:' },
+                            {
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: pngBase64,
+                                },
+                            },
+                            { text: 'What do you see?' },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThan(4);
+            expect(result.breakdown.textTokens).toBeGreaterThan(0);
+            expect(result.breakdown.imageTokens).toBeGreaterThanOrEqual(4);
+        });
+    });
+    describe('function content', () => {
+        it('should handle function calls', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            {
+                                functionCall: {
+                                    name: 'test_function',
+                                    args: { param1: 'value1', param2: 42 },
+                                },
+                            },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThan(0);
+            expect(result.breakdown.otherTokens).toBeGreaterThan(0);
+        });
+    });
+    describe('empty content', () => {
+        it('should handle empty request', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBe(0);
+            expect(result.breakdown.textTokens).toBe(0);
+            expect(result.breakdown.imageTokens).toBe(0);
+        });
+        it('should handle undefined contents', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBe(0);
+        });
+    });
+    describe('configuration', () => {
+        it('should use custom text encoding', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [{ text: 'Test text for encoding' }],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request, {
+                textEncoding: 'cl100k_base',
+            });
+            expect(result.totalTokens).toBeGreaterThan(0);
+        });
+        it('should process multiple images serially', async () => {
+            const pngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChAI9jU77yQAAAABJRU5ErkJggg==';
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: Array(10).fill({
+                            inlineData: {
+                                mimeType: 'image/png',
+                                data: pngBase64,
+                            },
+                        }),
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            expect(result.totalTokens).toBeGreaterThanOrEqual(60); // At least 6 tokens per image * 10 images
+        });
+    });
+    describe('error handling', () => {
+        it('should handle malformed image data gracefully', async () => {
+            const request = {
+                model: 'test-model',
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            {
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: 'invalid-base64-data',
+                                },
+                            },
+                        ],
+                    },
+                ],
+            };
+            const result = await tokenizer.calculateTokens(request);
+            // Should still return some tokens (fallback to minimum)
+            expect(result.totalTokens).toBeGreaterThanOrEqual(4);
+        });
+    });
+});
+//# sourceMappingURL=requestTokenizer.test.js.map

package/dist/src/utils/request-tokenizer/requestTokenizer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"requestTokenizer.test.js","sourceRoot":"","sources":["../../../../src/utils/request-tokenizer/requestTokenizer.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAGhE,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,IAAI,SAAkC,CAAC;IAEvC,UAAU,CAAC,GAAG,EAAE;QACd,SAAS,GAAG,IAAI,uBAAuB,EAAE,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;qBACnC;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL,EAAE,IAAI,EAAE,YAAY,EAAE;4BACtB,EAAE,IAAI,EAAE,aAAa,EAAE;4BACvB,EAAE,IAAI,EAAE,YAAY,EAAE;yBACvB;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE,CAAC,uBAAuB,CAAC;aACpC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,0CAA0C;YAC1C,MAAM,SAAS,GACb,kGAAkG,CAAC;YAErG,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL;gCACE,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,SAAS;iCAChB;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,6BAA6B;YACnF,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YAC/D,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,SAAS,GACb,kGAAkG,CAAC;YAErG,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL;gCACE,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,SAAS;iCAChB;6BACF;4BACD;gCACE,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,SAAS;iCAChB;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,8BAA8B;YACpF,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,SAAS,GACb,kGAAkG,CAAC;YAErG,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL,EAAE,IAAI,EAAE,mBAAmB,EAAE;4BAC7B;gCACE,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,SAAS;iCAChB;6BACF;4BACD,EAAE,IAAI,EAAE,kBAAkB,EAAE;yBAC7B;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL;gCACE,YAAY,EAAE;oCACZ,IAAI,EAAE,eAAe;oCACrB,IAAI,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;iCACvC;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE,EAAE;aACb,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE,EAAE;aACb,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC;qBAC5C;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,EAAE;gBACtD,YAAY,EAAE,aAAa;aAC5B,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,SAAS,GACb,kGAAkG,CAAC;YAErG,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC;4BACpB,UAAU,EAAE;gCACV,QAAQ,EAAE,WAAW;gCACrB,IAAI,EAAE,SAAS;6BAChB;yBACF,CAAC;qBACH;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,CAAC,0CAA0C;QACnG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,OAAO,GAA0B;gBACrC,KAAK,EAAE,YAAY;gBACnB,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL;gCACE,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,qBAAqB;iCAC5B;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAExD,wDAAwD;YACxD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/utils/request-tokenizer/supportedImageFormats.d.ts

@@ -0,0 +1,30 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Supported image MIME types for vision models
+ * These formats are supported by the vision model and can be processed by the image tokenizer
+ */
+export declare const SUPPORTED_IMAGE_MIME_TYPES: readonly ["image/bmp", "image/jpeg", "image/jpg", "image/png", "image/tiff", "image/webp", "image/heic"];
+/**
+ * Type for supported image MIME types
+ */
+export type SupportedImageMimeType = (typeof SUPPORTED_IMAGE_MIME_TYPES)[number];
+/**
+ * Check if a MIME type is supported for vision processing
+ * @param mimeType The MIME type to check
+ * @returns True if the MIME type is supported
+ */
+export declare function isSupportedImageMimeType(mimeType: string): mimeType is SupportedImageMimeType;
+/**
+ * Get a human-readable list of supported image formats
+ * @returns Comma-separated string of supported formats
+ */
+export declare function getSupportedImageFormatsString(): string;
+/**
+ * Get warning message for unsupported image formats
+ * @returns Warning message string
+ */
+export declare function getUnsupportedImageFormatWarning(): string;

package/dist/src/utils/request-tokenizer/supportedImageFormats.js

@@ -0,0 +1,41 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Supported image MIME types for vision models
+ * These formats are supported by the vision model and can be processed by the image tokenizer
+ */
+export const SUPPORTED_IMAGE_MIME_TYPES = [
+    'image/bmp',
+    'image/jpeg',
+    'image/jpg', // Alternative MIME type for JPEG
+    'image/png',
+    'image/tiff',
+    'image/webp',
+    'image/heic',
+];
+/**
+ * Check if a MIME type is supported for vision processing
+ * @param mimeType The MIME type to check
+ * @returns True if the MIME type is supported
+ */
+export function isSupportedImageMimeType(mimeType) {
+    return SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType);
+}
+/**
+ * Get a human-readable list of supported image formats
+ * @returns Comma-separated string of supported formats
+ */
+export function getSupportedImageFormatsString() {
+    return SUPPORTED_IMAGE_MIME_TYPES.map((type) => type.replace('image/', '').toUpperCase()).join(', ');
+}
+/**
+ * Get warning message for unsupported image formats
+ * @returns Warning message string
+ */
+export function getUnsupportedImageFormatWarning() {
+    return `Only the following image formats are supported: ${getSupportedImageFormatsString()}. Other formats may not work as expected.`;
+}
+//# sourceMappingURL=supportedImageFormats.js.map

package/dist/src/utils/request-tokenizer/supportedImageFormats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"supportedImageFormats.js","sourceRoot":"","sources":["../../../../src/utils/request-tokenizer/supportedImageFormats.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,WAAW;IACX,YAAY;IACZ,WAAW,EAAE,iCAAiC;IAC9C,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,YAAY;CACJ,CAAC;AAQX;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CACtC,QAAgB;IAEhB,OAAO,0BAA0B,CAAC,QAAQ,CACxC,QAAkC,CACnC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,8BAA8B;IAC5C,OAAO,0BAA0B,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CACzC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gCAAgC;IAC9C,OAAO,mDAAmD,8BAA8B,EAAE,2CAA2C,CAAC;AACxI,CAAC"}

package/dist/src/utils/request-tokenizer/textTokenizer.d.ts

@@ -0,0 +1,29 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Text tokenizer for calculating text tokens using tiktoken
+ */
+export declare class TextTokenizer {
+    private encoding;
+    private encodingName;
+    constructor(encodingName?: string);
+    /**
+     * Initialize the tokenizer (lazy loading)
+     */
+    private ensureEncoding;
+    /**
+     * Calculate tokens for text content
+     */
+    calculateTokens(text: string): Promise<number>;
+    /**
+     * Calculate tokens for multiple text strings in parallel
+     */
+    calculateTokensBatch(texts: string[]): Promise<number[]>;
+    /**
+     * Dispose of resources
+     */
+    dispose(): void;
+}