@aetherai/aether 0.0.7 → 0.0.9

package/dist/CLI/packages/core/src/utils/request-tokenizer/requestTokenizer.js

@@ -1,252 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-import { TextTokenizer } from './textTokenizer.js';
-import { ImageTokenizer } from './imageTokenizer.js';
-import { createDebugLogger } from '../debugLogger.js';
-const debugLogger = createDebugLogger('TOKENIZER');
-/**
- * Simple request token estimator that handles text and image content serially
- */
-export class RequestTokenizer {
-    textTokenizer;
-    imageTokenizer;
-    constructor() {
-        this.textTokenizer = new TextTokenizer();
-        this.imageTokenizer = new ImageTokenizer();
-    }
-    /**
-     * Calculate tokens for a request using serial processing
-     */
-    async calculateTokens(request) {
-        const startTime = performance.now();
-        try {
-            // Process request content and group by type
-            const { textContents, imageContents, audioContents, otherContents } = this.processAndGroupContents(request);
-            if (textContents.length === 0 &&
-                imageContents.length === 0 &&
-                audioContents.length === 0 &&
-                otherContents.length === 0) {
-                return {
-                    totalTokens: 0,
-                    breakdown: {
-                        textTokens: 0,
-                        imageTokens: 0,
-                        audioTokens: 0,
-                        otherTokens: 0,
-                    },
-                    processingTime: performance.now() - startTime,
-                };
-            }
-            // Calculate tokens for each content type serially
-            const textTokens = await this.calculateTextTokens(textContents);
-            const imageTokens = await this.calculateImageTokens(imageContents);
-            const audioTokens = await this.calculateAudioTokens(audioContents);
-            const otherTokens = await this.calculateOtherTokens(otherContents);
-            const totalTokens = textTokens + imageTokens + audioTokens + otherTokens;
-            const processingTime = performance.now() - startTime;
-            return {
-                totalTokens,
-                breakdown: {
-                    textTokens,
-                    imageTokens,
-                    audioTokens,
-                    otherTokens,
-                },
-                processingTime,
-            };
-        }
-        catch (error) {
-            debugLogger.error('Error calculating tokens:', error);
-            // Fallback calculation
-            const fallbackTokens = this.calculateFallbackTokens(request);
-            return {
-                totalTokens: fallbackTokens,
-                breakdown: {
-                    textTokens: fallbackTokens,
-                    imageTokens: 0,
-                    audioTokens: 0,
-                    otherTokens: 0,
-                },
-                processingTime: performance.now() - startTime,
-            };
-        }
-    }
-    /**
-     * Calculate tokens for text contents
-     */
-    async calculateTextTokens(textContents) {
-        if (textContents.length === 0)
-            return 0;
-        try {
-            // Avoid per-part rounding inflation by estimating once on the combined text.
-            return await this.textTokenizer.calculateTokens(textContents.join(''));
-        }
-        catch (error) {
-            debugLogger.warn('Error calculating text tokens:', error);
-            // Fallback: character-based estimation
-            const totalChars = textContents.join('').length;
-            return Math.ceil(totalChars / 4);
-        }
-    }
-    /**
-     * Calculate tokens for image contents using serial processing
-     */
-    async calculateImageTokens(imageContents) {
-        if (imageContents.length === 0)
-            return 0;
-        try {
-            const tokenCounts = await this.imageTokenizer.calculateTokensBatch(imageContents);
-            return tokenCounts.reduce((sum, count) => sum + count, 0);
-        }
-        catch (error) {
-            debugLogger.warn('Error calculating image tokens:', error);
-            // Fallback: minimum tokens per image
-            return imageContents.length * 6; // 4 image tokens + 2 special tokens as minimum
-        }
-    }
-    /**
-     * Calculate tokens for audio contents
-     * TODO: Implement proper audio token calculation
-     */
-    async calculateAudioTokens(audioContents) {
-        if (audioContents.length === 0)
-            return 0;
-        // Placeholder implementation - audio token calculation would depend on
-        // the specific model's audio processing capabilities
-        // For now, estimate based on data size
-        let totalTokens = 0;
-        for (const audioContent of audioContents) {
-            try {
-                const dataSize = Math.floor(audioContent.data.length * 0.75); // Approximate binary size
-                // Rough estimate: 1 token per 100 bytes of audio data
-                totalTokens += Math.max(Math.ceil(dataSize / 100), 10); // Minimum 10 tokens per audio
-            }
-            catch (error) {
-                debugLogger.warn('Error calculating audio tokens:', error);
-                totalTokens += 10; // Fallback minimum
-            }
-        }
-        return totalTokens;
-    }
-    /**
-     * Calculate tokens for other content types (functions, files, etc.)
-     */
-    async calculateOtherTokens(otherContents) {
-        if (otherContents.length === 0)
-            return 0;
-        try {
-            // Treat other content as text, and avoid per-item rounding inflation.
-            return await this.textTokenizer.calculateTokens(otherContents.join(''));
-        }
-        catch (error) {
-            debugLogger.warn('Error calculating other content tokens:', error);
-            // Fallback: character-based estimation
-            const totalChars = otherContents.join('').length;
-            return Math.ceil(totalChars / 4);
-        }
-    }
-    /**
-     * Fallback token calculation using simple string serialization
-     */
-    calculateFallbackTokens(request) {
-        try {
-            const content = JSON.stringify(request.contents);
-            return Math.ceil(content.length / 4); // Rough estimate: 1 token ≈ 4 characters
-        }
-        catch (error) {
-            debugLogger.warn('Error in fallback token calculation:', error);
-            return 100; // Conservative fallback
-        }
-    }
-    /**
-     * Process request contents and group by type
-     */
-    processAndGroupContents(request) {
-        const textContents = [];
-        const imageContents = [];
-        const audioContents = [];
-        const otherContents = [];
-        if (!request.contents) {
-            return { textContents, imageContents, audioContents, otherContents };
-        }
-        const contents = Array.isArray(request.contents)
-            ? request.contents
-            : [request.contents];
-        for (const content of contents) {
-            this.processContent(content, textContents, imageContents, audioContents, otherContents);
-        }
-        return { textContents, imageContents, audioContents, otherContents };
-    }
-    /**
-     * Process a single content item and add to appropriate arrays
-     */
-    processContent(content, textContents, imageContents, audioContents, otherContents) {
-        if (typeof content === 'string') {
-            if (content.trim()) {
-                textContents.push(content);
-            }
-            return;
-        }
-        if ('parts' in content && content.parts) {
-            for (const part of content.parts) {
-                this.processPart(part, textContents, imageContents, audioContents, otherContents);
-            }
-            return;
-        }
-        // Some request shapes (e.g. CountTokensParameters) allow passing parts directly
-        // instead of wrapping them in a { parts: [...] } Content object.
-        this.processPart(content, textContents, imageContents, audioContents, otherContents);
-    }
-    /**
-     * Process a single part and add to appropriate arrays
-     */
-    processPart(part, textContents, imageContents, audioContents, otherContents) {
-        if (typeof part === 'string') {
-            if (part.trim()) {
-                textContents.push(part);
-            }
-            return;
-        }
-        if ('text' in part && part.text) {
-            textContents.push(part.text);
-            return;
-        }
-        if ('inlineData' in part && part.inlineData) {
-            const { data, mimeType } = part.inlineData;
-            if (mimeType && mimeType.startsWith('image/')) {
-                imageContents.push({ data: data || '', mimeType });
-                return;
-            }
-            if (mimeType && mimeType.startsWith('audio/')) {
-                audioContents.push({ data: data || '', mimeType });
-                return;
-            }
-        }
-        if ('fileData' in part && part.fileData) {
-            otherContents.push(JSON.stringify(part.fileData));
-            return;
-        }
-        if ('functionCall' in part && part.functionCall) {
-            otherContents.push(JSON.stringify(part.functionCall));
-            return;
-        }
-        if ('functionResponse' in part && part.functionResponse) {
-            otherContents.push(JSON.stringify(part.functionResponse));
-            return;
-        }
-        // Unknown part type - try to serialize
-        try {
-            const serialized = JSON.stringify(part);
-            if (serialized && serialized !== '{}') {
-                otherContents.push(serialized);
-            }
-        }
-        catch (error) {
-            debugLogger.warn('Failed to serialize unknown part type:', error);
-        }
-    }
-}
-//# sourceMappingURL=requestTokenizer.js.map

package/dist/CLI/packages/core/src/utils/request-tokenizer/requestTokenizer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"requestTokenizer.js","sourceRoot":"","sources":["../../../../../../../../core/src/utils/request-tokenizer/requestTokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AASH,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,WAAW,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;AAEnD;;GAEG;AACH,MAAM,OAAO,gBAAgB;IACnB,aAAa,CAAgB;IAC7B,cAAc,CAAiB;IAEvC;QACE,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;QACzC,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CACnB,OAA8B;QAE9B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,IAAI,CAAC;YACH,4CAA4C;YAC5C,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,GACjE,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAExC,IACE,YAAY,CAAC,MAAM,KAAK,CAAC;gBACzB,aAAa,CAAC,MAAM,KAAK,CAAC;gBAC1B,aAAa,CAAC,MAAM,KAAK,CAAC;gBAC1B,aAAa,CAAC,MAAM,KAAK,CAAC,EAC1B,CAAC;gBACD,OAAO;oBACL,WAAW,EAAE,CAAC;oBACd,SAAS,EAAE;wBACT,UAAU,EAAE,CAAC;wBACb,WAAW,EAAE,CAAC;wBACd,WAAW,EAAE,CAAC;wBACd,WAAW,EAAE,CAAC;qBACf;oBACD,cAAc,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;iBAC9C,CAAC;YACJ,CAAC;YAED,kDAAkD;YAClD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;YAChE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YACnE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YACnE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YAEnE,MAAM,WAAW,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,GAAG,WAAW,CAAC;YACzE,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAErD,OAAO;gBACL,WAAW;gBACX,SAAS,EAAE;oBACT,UAAU;oBACV,WAAW;oBACX,WAAW;oBACX,WAAW;iBACZ;gBACD,cAAc;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,WAAW,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;YAEtD,uBAAuB;YACvB,MAAM,cAAc,GAAG,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAE7D,OAAO;gBACL,WAAW,EAAE,cAAc;gBAC3B,SAAS,EAAE;oBACT,UAAU,EAAE,cAAc;oBAC1B,WAAW,EAAE,CAAC;oBACd,WAAW,EAAE,CAAC;oBACd,WAAW,EAAE,CAAC;iBACf;gBACD,cAAc,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;aAC9C,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,YAAsB;QACtD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAExC,IAAI,CAAC;YACH,6EAA6E;YAC7E,OAAO,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACzE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,WAAW,CAAC,IAAI,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YAC1D,uCAAuC;YACvC,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;YAChD,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAChC,aAAwD;QAExD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,WAAW,GACf,MAAM,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YAChE,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,WAAW,CAAC,IAAI,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAC;YAC3D,qCAAqC;YACrC,OAAO,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,+CAA+C;QAClF,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,oBAAoB,CAChC,aAAwD;QAExD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEzC,uEAAuE;QACvE,qDAAqD;QACrD,uCAAuC;QACvC,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,0BAA0B;gBACxF,sDAAsD;gBACtD,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,8BAA8B;YACxF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,WAAW,CAAC,IAAI,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAC;gBAC3D,WAAW,IAAI,EAAE,CAAC,CAAC,mBAAmB;YACxC,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAAC,aAAuB;QACxD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEzC,IAAI,CAAC;YACH,sEAAsE;YACtE,OAAO,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC1E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,WAAW,CAAC,IAAI,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;YACnE,uCAAuC;YACvC,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;YACjD,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,uBAAuB,CAAC,OAA8B;QAC5D,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACjD,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,yCAAyC;QACjF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,WAAW,CAAC,IAAI,CAAC,sCAAsC,EAAE,KAAK,CAAC,CAAC;YAChE,OAAO,GAAG,CAAC,CAAC,wBAAwB;QACtC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,uBAAuB,CAAC,OAA8B;QAM5D,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,MAAM,aAAa,GAA8C,EAAE,CAAC;QACpE,MAAM,aAAa,GAA8C,EAAE,CAAC;QACpE,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC;QACvE,CAAC;QAED,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;YAC9C,CAAC,CAAC,OAAO,CAAC,QAAQ;YAClB,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CACjB,OAAO,EACP,YAAY,EACZ,aAAa,EACb,aAAa,EACb,aAAa,CACd,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC;IACvE,CAAC;IAED;;OAEG;IACK,cAAc,CACpB,OAAqC,EACrC,YAAsB,EACtB,aAAwD,EACxD,aAAwD,EACxD,aAAuB;QAEvB,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC7B,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,OAAO,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACxC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,WAAW,CACd,IAAI,EACJ,YAAY,EACZ,aAAa,EACb,aAAa,EACb,aAAa,CACd,CAAC;YACJ,CAAC;YACD,OAAO;QACT,CAAC;QAED,gFAAgF;QAChF,iEAAiE;QACjE,IAAI,CAAC,WAAW,CACd,OAAwB,EACxB,YAAY,EACZ,aAAa,EACb,aAAa,EACb,aAAa,CACd,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,WAAW,CACjB,IAAmB,EACnB,YAAsB,EACtB,aAAwD,EACxD,aAAwD,EACxD,aAAuB;QAEvB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBAChB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAChC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO;QACT,CAAC;QAED,IAAI,YAAY,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC;YAC3C,IAAI,QAAQ,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9C,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACnD,OAAO;YACT,CAAC;YACD,IAAI,QAAQ,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9C,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACnD,OAAO;YACT,CAAC;QACH,CAAC;QAED,IAAI,UAAU,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACxC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAClD,OAAO;QACT,CAAC;QAED,IAAI,cAAc,IAAI,IAAI,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAChD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YACtD,OAAO;QACT,CAAC;QAED,IAAI,kBAAkB,IAAI,IAAI,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAC1D,OAAO;QACT,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,UAAU,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;gBACtC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,WAAW,CAAC,IAAI,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;CACF"}

package/dist/CLI/packages/core/src/utils/request-tokenizer/supportedImageFormats.d.ts

@@ -1,30 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-/**
- * Supported image MIME types for vision models
- * These formats are supported by the vision model and can be processed by the image tokenizer
- */
-export declare const SUPPORTED_IMAGE_MIME_TYPES: readonly ["image/bmp", "image/jpeg", "image/jpg", "image/png", "image/tiff", "image/webp", "image/heic"];
-/**
- * Type for supported image MIME types
- */
-export type SupportedImageMimeType = (typeof SUPPORTED_IMAGE_MIME_TYPES)[number];
-/**
- * Check if a MIME type is supported for vision processing
- * @param mimeType The MIME type to check
- * @returns True if the MIME type is supported
- */
-export declare function isSupportedImageMimeType(mimeType: string): mimeType is SupportedImageMimeType;
-/**
- * Get a human-readable list of supported image formats
- * @returns Comma-separated string of supported formats
- */
-export declare function getSupportedImageFormatsString(): string;
-/**
- * Get warning message for unsupported image formats
- * @returns Warning message string
- */
-export declare function getUnsupportedImageFormatWarning(): string;

package/dist/CLI/packages/core/src/utils/request-tokenizer/supportedImageFormats.js

@@ -1,41 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-/**
- * Supported image MIME types for vision models
- * These formats are supported by the vision model and can be processed by the image tokenizer
- */
-export const SUPPORTED_IMAGE_MIME_TYPES = [
-    'image/bmp',
-    'image/jpeg',
-    'image/jpg', // Alternative MIME type for JPEG
-    'image/png',
-    'image/tiff',
-    'image/webp',
-    'image/heic',
-];
-/**
- * Check if a MIME type is supported for vision processing
- * @param mimeType The MIME type to check
- * @returns True if the MIME type is supported
- */
-export function isSupportedImageMimeType(mimeType) {
-    return SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType);
-}
-/**
- * Get a human-readable list of supported image formats
- * @returns Comma-separated string of supported formats
- */
-export function getSupportedImageFormatsString() {
-    return SUPPORTED_IMAGE_MIME_TYPES.map((type) => type.replace('image/', '').toUpperCase()).join(', ');
-}
-/**
- * Get warning message for unsupported image formats
- * @returns Warning message string
- */
-export function getUnsupportedImageFormatWarning() {
-    return `Only the following image formats are supported: ${getSupportedImageFormatsString()}. Other formats may not work as expected.`;
-}
-//# sourceMappingURL=supportedImageFormats.js.map

package/dist/CLI/packages/core/src/utils/request-tokenizer/supportedImageFormats.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"supportedImageFormats.js","sourceRoot":"","sources":["../../../../../../../../core/src/utils/request-tokenizer/supportedImageFormats.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,WAAW;IACX,YAAY;IACZ,WAAW,EAAE,iCAAiC;IAC9C,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,YAAY;CACJ,CAAC;AAQX;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CACtC,QAAgB;IAEhB,OAAO,0BAA0B,CAAC,QAAQ,CACxC,QAAkC,CACnC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,8BAA8B;IAC5C,OAAO,0BAA0B,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CACzC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gCAAgC;IAC9C,OAAO,mDAAmD,8BAA8B,EAAE,2CAA2C,CAAC;AACxI,CAAC"}

package/dist/CLI/packages/core/src/utils/request-tokenizer/textTokenizer.d.ts

@@ -1,32 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-/**
- * Text tokenizer for calculating text tokens using character-based estimation.
- *
- * Uses a lightweight character-based approach that is "good enough" for
- * guardrail features like sessionTokenLimit.
- *
- * Algorithm:
- * - ASCII characters: 0.25 tokens per char (4 chars = 1 token)
- * - Non-ASCII characters: 1.1 tokens per char (conservative for CJK, emoji, etc.)
- */
-export declare class TextTokenizer {
-    /**
-     * Calculate tokens for text content
-     *
-     * @param text - The text to estimate tokens for
-     * @returns The estimated token count
-     */
-    calculateTokens(text: string): Promise<number>;
-    /**
-     * Calculate tokens for multiple text strings
-     *
-     * @param texts - Array of text strings to estimate tokens for
-     * @returns Array of token counts corresponding to each input text
-     */
-    calculateTokensBatch(texts: string[]): Promise<number[]>;
-    private calculateTokensSync;
-}

package/dist/CLI/packages/core/src/utils/request-tokenizer/textTokenizer.js

@@ -1,54 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-/**
- * Text tokenizer for calculating text tokens using character-based estimation.
- *
- * Uses a lightweight character-based approach that is "good enough" for
- * guardrail features like sessionTokenLimit.
- *
- * Algorithm:
- * - ASCII characters: 0.25 tokens per char (4 chars = 1 token)
- * - Non-ASCII characters: 1.1 tokens per char (conservative for CJK, emoji, etc.)
- */
-export class TextTokenizer {
-    /**
-     * Calculate tokens for text content
-     *
-     * @param text - The text to estimate tokens for
-     * @returns The estimated token count
-     */
-    async calculateTokens(text) {
-        return this.calculateTokensSync(text);
-    }
-    /**
-     * Calculate tokens for multiple text strings
-     *
-     * @param texts - Array of text strings to estimate tokens for
-     * @returns Array of token counts corresponding to each input text
-     */
-    async calculateTokensBatch(texts) {
-        return texts.map((text) => this.calculateTokensSync(text));
-    }
-    calculateTokensSync(text) {
-        if (!text || text.length === 0) {
-            return 0;
-        }
-        let asciiChars = 0;
-        let nonAsciiChars = 0;
-        for (let i = 0; i < text.length; i++) {
-            const charCode = text.charCodeAt(i);
-            if (charCode < 128) {
-                asciiChars++;
-            }
-            else {
-                nonAsciiChars++;
-            }
-        }
-        const tokens = asciiChars / 4 + nonAsciiChars * 1.1;
-        return Math.ceil(tokens);
-    }
-}
-//# sourceMappingURL=textTokenizer.js.map

package/dist/CLI/packages/core/src/utils/request-tokenizer/textTokenizer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"textTokenizer.js","sourceRoot":"","sources":["../../../../../../../../core/src/utils/request-tokenizer/textTokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;GASG;AACH,MAAM,OAAO,aAAa;IACxB;;;;;OAKG;IACH,KAAK,CAAC,eAAe,CAAC,IAAY;QAChC,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,oBAAoB,CAAC,KAAe;QACxC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,CAAC;IAEO,mBAAmB,CAAC,IAAY;QACtC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,CAAC;QACX,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;gBACnB,UAAU,EAAE,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,aAAa,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,UAAU,GAAG,CAAC,GAAG,aAAa,GAAG,GAAG,CAAC;QACpD,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;CACF"}

package/dist/CLI/packages/core/src/utils/request-tokenizer/types.d.ts

@@ -1,34 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-/**
- * Token calculation result for different content types
- */
-export interface TokenCalculationResult {
-    /** Total tokens calculated */
-    totalTokens: number;
-    /** Breakdown by content type */
-    breakdown: {
-        textTokens: number;
-        imageTokens: number;
-        audioTokens: number;
-        otherTokens: number;
-    };
-    /** Processing time in milliseconds */
-    processingTime: number;
-}
-/**
- * Image metadata extracted from base64 data
- */
-export interface ImageMetadata {
-    /** Image width in pixels */
-    width: number;
-    /** Image height in pixels */
-    height: number;
-    /** MIME type of the image */
-    mimeType: string;
-    /** Size of the base64 data in bytes */
-    dataSize: number;
-}

package/dist/CLI/packages/core/src/utils/request-tokenizer/types.js

@@ -1,7 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-export {};
-//# sourceMappingURL=types.js.map

package/dist/CLI/packages/core/src/utils/request-tokenizer/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../../../../../core/src/utils/request-tokenizer/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}

package/dist/CLI/packages/core/src/utils/retry.d.ts

@@ -1,25 +0,0 @@
-/**
- * @license
- * Copyright 2025 Google LLC
- * SPDX-License-Identifier: Apache-2.0
- */
-import type { GenerateContentResponse } from '@google/genai';
-export interface HttpError extends Error {
-    status?: number;
-}
-export interface RetryOptions {
-    maxAttempts: number;
-    initialDelayMs: number;
-    maxDelayMs: number;
-    shouldRetryOnError: (error: Error) => boolean;
-    shouldRetryOnContent?: (content: GenerateContentResponse) => boolean;
-    authType?: string;
-}
-/**
- * Retries a function with exponential backoff and jitter.
- * @param fn The asynchronous function to retry.
- * @param options Optional retry configuration.
- * @returns A promise that resolves with the result of the function if successful.
- * @throws The last error encountered if all attempts fail.
- */
-export declare function retryWithBackoff<T>(fn: () => Promise<T>, options?: Partial<RetryOptions>): Promise<T>;

package/dist/CLI/packages/core/src/utils/retry.js

@@ -1,159 +0,0 @@
-/**
- * @license
- * Copyright 2025 Google LLC
- * SPDX-License-Identifier: Apache-2.0
- */
-import { AuthType } from '../core/contentGenerator.js';
-import { isQwenQuotaExceededError } from './quotaErrorDetection.js';
-import { createDebugLogger } from './debugLogger.js';
-import { getErrorStatus } from './errors.js';
-const debugLogger = createDebugLogger('RETRY');
-const DEFAULT_RETRY_OPTIONS = {
-    maxAttempts: 7,
-    initialDelayMs: 1500,
-    maxDelayMs: 30000, // 30 seconds
-    shouldRetryOnError: defaultShouldRetry,
-};
-/**
- * Default predicate function to determine if a retry should be attempted.
- * Retries on 429 (Too Many Requests) and 5xx server errors.
- * @param error The error object.
- * @returns True if the error is a transient error, false otherwise.
- */
-function defaultShouldRetry(error) {
-    const status = getErrorStatus(error);
-    return (status === 429 || (status !== undefined && status >= 500 && status < 600));
-}
-/**
- * Delays execution for a specified number of milliseconds.
- * @param ms The number of milliseconds to delay.
- * @returns A promise that resolves after the delay.
- */
-function delay(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-}
-/**
- * Retries a function with exponential backoff and jitter.
- * @param fn The asynchronous function to retry.
- * @param options Optional retry configuration.
- * @returns A promise that resolves with the result of the function if successful.
- * @throws The last error encountered if all attempts fail.
- */
-export async function retryWithBackoff(fn, options) {
-    if (options?.maxAttempts !== undefined && options.maxAttempts <= 0) {
-        throw new Error('maxAttempts must be a positive number.');
-    }
-    const cleanOptions = options
-        ? Object.fromEntries(Object.entries(options).filter(([_, v]) => v != null))
-        : {};
-    const { maxAttempts, initialDelayMs, maxDelayMs, authType, shouldRetryOnError, shouldRetryOnContent, } = {
-        ...DEFAULT_RETRY_OPTIONS,
-        ...cleanOptions,
-    };
-    let attempt = 0;
-    let currentDelay = initialDelayMs;
-    while (attempt < maxAttempts) {
-        attempt++;
-        try {
-            const result = await fn();
-            if (shouldRetryOnContent &&
-                shouldRetryOnContent(result)) {
-                const jitter = currentDelay * 0.3 * (Math.random() * 2 - 1);
-                const delayWithJitter = Math.max(0, currentDelay + jitter);
-                await delay(delayWithJitter);
-                currentDelay = Math.min(maxDelayMs, currentDelay * 2);
-                continue;
-            }
-            return result;
-        }
-        catch (error) {
-            const errorStatus = getErrorStatus(error);
-            // Check for Aether OAuth quota exceeded error - throw immediately without retry
-            if (authType === AuthType.AETHER_OAUTH && isQwenQuotaExceededError(error)) {
-                throw new Error(`Aether OAuth quota exceeded: Your free daily quota has been reached.\n\n` +
-                    `To continue using Aether without waiting, upgrade to the Alibaba Cloud Coding Plan:\n` +
-                    `  China:       https://help.aliyun.com/zh/model-studio/coding-plan\n` +
-                    `  Global/Intl: https://www.alibabacloud.com/help/en/model-studio/coding-plan\n\n` +
-                    `After subscribing, run /auth to configure your Coding Plan API key.`);
-            }
-            // Check if we've exhausted retries or shouldn't retry
-            if (attempt >= maxAttempts || !shouldRetryOnError(error)) {
-                throw error;
-            }
-            const retryAfterMs = errorStatus === 429 ? getRetryAfterDelayMs(error) : 0;
-            if (retryAfterMs > 0) {
-                // Respect Retry-After header if present and parsed
-                debugLogger.warn(`Attempt ${attempt} failed with status ${errorStatus ?? 'unknown'}. Retrying after explicit delay of ${retryAfterMs}ms...`, error);
-                await delay(retryAfterMs);
-                // Reset currentDelay for next potential non-429 error, or if Retry-After is not present next time
-                currentDelay = initialDelayMs;
-            }
-            else {
-                // Fallback to exponential backoff with jitter
-                logRetryAttempt(attempt, error, errorStatus);
-                // Add jitter: +/- 30% of currentDelay
-                const jitter = currentDelay * 0.3 * (Math.random() * 2 - 1);
-                const delayWithJitter = Math.max(0, currentDelay + jitter);
-                await delay(delayWithJitter);
-                currentDelay = Math.min(maxDelayMs, currentDelay * 2);
-            }
-        }
-    }
-    // This line should theoretically be unreachable due to the throw in the catch block.
-    // Added for type safety and to satisfy the compiler that a promise is always returned.
-    throw new Error('Retry attempts exhausted');
-}
-/**
- * Extracts the Retry-After delay from an error object's headers.
- * @param error The error object.
- * @returns The delay in milliseconds, or 0 if not found or invalid.
- */
-function getRetryAfterDelayMs(error) {
-    if (typeof error === 'object' && error !== null) {
-        // Check for error.response.headers (common in axios errors)
-        if ('response' in error &&
-            typeof error.response === 'object' &&
-            error.response !== null) {
-            const response = error.response;
-            if ('headers' in response &&
-                typeof response.headers === 'object' &&
-                response.headers !== null) {
-                const headers = response.headers;
-                const retryAfterHeader = headers['retry-after'];
-                if (typeof retryAfterHeader === 'string') {
-                    const retryAfterSeconds = parseInt(retryAfterHeader, 10);
-                    if (!isNaN(retryAfterSeconds)) {
-                        return retryAfterSeconds * 1000;
-                    }
-                    // It might be an HTTP date
-                    const retryAfterDate = new Date(retryAfterHeader);
-                    if (!isNaN(retryAfterDate.getTime())) {
-                        return Math.max(0, retryAfterDate.getTime() - Date.now());
-                    }
-                }
-            }
-        }
-    }
-    return 0;
-}
-/**
- * Logs a message for a retry attempt when using exponential backoff.
- * @param attempt The current attempt number.
- * @param error The error that caused the retry.
- * @param errorStatus The HTTP status code of the error, if available.
- */
-function logRetryAttempt(attempt, error, errorStatus) {
-    const message = errorStatus
-        ? `Attempt ${attempt} failed with status ${errorStatus}. Retrying with backoff...`
-        : `Attempt ${attempt} failed. Retrying with backoff...`;
-    if (errorStatus === 429) {
-        debugLogger.warn(message, error);
-    }
-    else if (errorStatus && errorStatus >= 500 && errorStatus < 600) {
-        debugLogger.error(message, error);
-    }
-    else {
-        debugLogger.warn(message, error);
-    }
-}
-//# sourceMappingURL=retry.js.map