@aetherai/aether 0.0.7 → 0.0.9

package/dist/CLI/packages/core/src/utils/request-tokenizer/imageTokenizer.js

@@ -1,403 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-import { isSupportedImageMimeType } from './supportedImageFormats.js';
-import { createDebugLogger } from '../debugLogger.js';
-const debugLogger = createDebugLogger('IMAGE_TOKENIZER');
-/**
- * Image tokenizer for calculating image tokens based on dimensions
- *
- * Key rules:
- * - 28x28 pixels = 1 token
- * - Minimum: 4 tokens per image
- * - Maximum: 16384 tokens per image
- * - Additional: 2 special tokens (vision_bos + vision_eos)
- * - Supports: PNG, JPEG, WebP, GIF, BMP, TIFF, HEIC formats
- */
-export class ImageTokenizer {
-    /** 28x28 pixels = 1 token */
-    static PIXELS_PER_TOKEN = 28 * 28;
-    /** Minimum tokens per image */
-    static MIN_TOKENS_PER_IMAGE = 4;
-    /** Maximum tokens per image */
-    static MAX_TOKENS_PER_IMAGE = 16384;
-    /** Special tokens for vision markers */
-    static VISION_SPECIAL_TOKENS = 2;
-    /**
-     * Extract image metadata from base64 data
-     *
-     * @param base64Data Base64-encoded image data (with or without data URL prefix)
-     * @param mimeType MIME type of the image
-     * @returns Promise resolving to ImageMetadata with dimensions and format info
-     */
-    async extractImageMetadata(base64Data, mimeType) {
-        try {
-            // Check if the MIME type is supported
-            if (!isSupportedImageMimeType(mimeType)) {
-                debugLogger.warn(`Unsupported image format: ${mimeType}`);
-                // Return default metadata for unsupported formats
-                return {
-                    width: 512,
-                    height: 512,
-                    mimeType,
-                    dataSize: Math.floor(base64Data.length * 0.75),
-                };
-            }
-            const cleanBase64 = base64Data.replace(/^data:[^;]+;base64,/, '');
-            const buffer = Buffer.from(cleanBase64, 'base64');
-            const dimensions = await this.extractDimensions(buffer, mimeType);
-            return {
-                width: dimensions.width,
-                height: dimensions.height,
-                mimeType,
-                dataSize: buffer.length,
-            };
-        }
-        catch (error) {
-            debugLogger.warn('Failed to extract image metadata:', error);
-            // Return default metadata for fallback
-            return {
-                width: 512,
-                height: 512,
-                mimeType,
-                dataSize: Math.floor(base64Data.length * 0.75),
-            };
-        }
-    }
-    /**
-     * Extract image dimensions from buffer based on format
-     *
-     * @param buffer Binary image data buffer
-     * @param mimeType MIME type to determine parsing strategy
-     * @returns Promise resolving to width and height dimensions
-     */
-    async extractDimensions(buffer, mimeType) {
-        if (mimeType.includes('png')) {
-            return this.extractPngDimensions(buffer);
-        }
-        if (mimeType.includes('jpeg') || mimeType.includes('jpg')) {
-            return this.extractJpegDimensions(buffer);
-        }
-        if (mimeType.includes('webp')) {
-            return this.extractWebpDimensions(buffer);
-        }
-        if (mimeType.includes('gif')) {
-            return this.extractGifDimensions(buffer);
-        }
-        if (mimeType.includes('bmp')) {
-            return this.extractBmpDimensions(buffer);
-        }
-        if (mimeType.includes('tiff')) {
-            return this.extractTiffDimensions(buffer);
-        }
-        if (mimeType.includes('heic')) {
-            return this.extractHeicDimensions(buffer);
-        }
-        return { width: 512, height: 512 };
-    }
-    /**
-     * Extract PNG dimensions from IHDR chunk
-     * PNG signature: 89 50 4E 47 0D 0A 1A 0A
-     * Width/height at bytes 16-19 and 20-23 (big-endian)
-     */
-    extractPngDimensions(buffer) {
-        if (buffer.length < 24) {
-            throw new Error('Invalid PNG: buffer too short');
-        }
-        // Verify PNG signature
-        const signature = buffer.subarray(0, 8);
-        const expectedSignature = Buffer.from([
-            0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-        ]);
-        if (!signature.equals(expectedSignature)) {
-            throw new Error('Invalid PNG signature');
-        }
-        const width = buffer.readUInt32BE(16);
-        const height = buffer.readUInt32BE(20);
-        return { width, height };
-    }
-    /**
-     * Extract JPEG dimensions from SOF (Start of Frame) markers
-     * JPEG starts with FF D8, SOF markers: 0xC0-0xC3, 0xC5-0xC7, 0xC9-0xCB, 0xCD-0xCF
-     * Dimensions at offset +5 (height) and +7 (width) from SOF marker
-     */
-    extractJpegDimensions(buffer) {
-        if (buffer.length < 4 || buffer[0] !== 0xff || buffer[1] !== 0xd8) {
-            throw new Error('Invalid JPEG signature');
-        }
-        let offset = 2;
-        while (offset < buffer.length - 8) {
-            if (buffer[offset] !== 0xff) {
-                offset++;
-                continue;
-            }
-            const marker = buffer[offset + 1];
-            // SOF markers
-            if ((marker >= 0xc0 && marker <= 0xc3) ||
-                (marker >= 0xc5 && marker <= 0xc7) ||
-                (marker >= 0xc9 && marker <= 0xcb) ||
-                (marker >= 0xcd && marker <= 0xcf)) {
-                const height = buffer.readUInt16BE(offset + 5);
-                const width = buffer.readUInt16BE(offset + 7);
-                return { width, height };
-            }
-            const segmentLength = buffer.readUInt16BE(offset + 2);
-            offset += 2 + segmentLength;
-        }
-        throw new Error('Could not find JPEG dimensions');
-    }
-    /**
-     * Extract WebP dimensions from RIFF container
-     * Supports VP8, VP8L, and VP8X formats
-     */
-    extractWebpDimensions(buffer) {
-        if (buffer.length < 30) {
-            throw new Error('Invalid WebP: too short');
-        }
-        const riffSignature = buffer.subarray(0, 4).toString('ascii');
-        const webpSignature = buffer.subarray(8, 12).toString('ascii');
-        if (riffSignature !== 'RIFF' || webpSignature !== 'WEBP') {
-            throw new Error('Invalid WebP signature');
-        }
-        const format = buffer.subarray(12, 16).toString('ascii');
-        if (format === 'VP8 ') {
-            const width = buffer.readUInt16LE(26) & 0x3fff;
-            const height = buffer.readUInt16LE(28) & 0x3fff;
-            return { width, height };
-        }
-        else if (format === 'VP8L') {
-            const bits = buffer.readUInt32LE(21);
-            const width = (bits & 0x3fff) + 1;
-            const height = ((bits >> 14) & 0x3fff) + 1;
-            return { width, height };
-        }
-        else if (format === 'VP8X') {
-            const width = (buffer.readUInt32LE(24) & 0xffffff) + 1;
-            const height = (buffer.readUInt32LE(26) & 0xffffff) + 1;
-            return { width, height };
-        }
-        throw new Error('Unsupported WebP format');
-    }
-    /**
-     * Extract GIF dimensions from header
-     * Supports GIF87a and GIF89a formats
-     */
-    extractGifDimensions(buffer) {
-        if (buffer.length < 10) {
-            throw new Error('Invalid GIF: too short');
-        }
-        const signature = buffer.subarray(0, 6).toString('ascii');
-        if (signature !== 'GIF87a' && signature !== 'GIF89a') {
-            throw new Error('Invalid GIF signature');
-        }
-        const width = buffer.readUInt16LE(6);
-        const height = buffer.readUInt16LE(8);
-        return { width, height };
-    }
-    /**
-     * Calculate tokens for an image based on its metadata
-     *
-     * @param metadata Image metadata containing width, height, and format info
-     * @returns Total token count including base image tokens and special tokens
-     */
-    calculateTokens(metadata) {
-        return this.calculateTokensWithScaling(metadata.width, metadata.height);
-    }
-    /**
-     * Calculate tokens with scaling logic
-     *
-     * Steps:
-     * 1. Normalize to 28-pixel multiples
-     * 2. Scale large images down, small images up
-     * 3. Calculate tokens: pixels / 784 + 2 special tokens
-     *
-     * @param width Original image width in pixels
-     * @param height Original image height in pixels
-     * @returns Total token count for the image
-     */
-    calculateTokensWithScaling(width, height) {
-        // Normalize to 28-pixel multiples
-        let hBar = Math.round(height / 28) * 28;
-        let wBar = Math.round(width / 28) * 28;
-        // Define pixel boundaries
-        const minPixels = ImageTokenizer.MIN_TOKENS_PER_IMAGE * ImageTokenizer.PIXELS_PER_TOKEN;
-        const maxPixels = ImageTokenizer.MAX_TOKENS_PER_IMAGE * ImageTokenizer.PIXELS_PER_TOKEN;
-        // Apply scaling
-        if (hBar * wBar > maxPixels) {
-            // Scale down large images
-            const beta = Math.sqrt((height * width) / maxPixels);
-            hBar = Math.floor(height / beta / 28) * 28;
-            wBar = Math.floor(width / beta / 28) * 28;
-        }
-        else if (hBar * wBar < minPixels) {
-            // Scale up small images
-            const beta = Math.sqrt(minPixels / (height * width));
-            hBar = Math.ceil((height * beta) / 28) * 28;
-            wBar = Math.ceil((width * beta) / 28) * 28;
-        }
-        // Calculate tokens
-        const imageTokens = Math.floor((hBar * wBar) / ImageTokenizer.PIXELS_PER_TOKEN);
-        return imageTokens + ImageTokenizer.VISION_SPECIAL_TOKENS;
-    }
-    /**
-     * Calculate tokens for multiple images serially
-     *
-     * @param base64DataArray Array of image data with MIME type information
-     * @returns Promise resolving to array of token counts in same order as input
-     */
-    async calculateTokensBatch(base64DataArray) {
-        const results = [];
-        for (const { data, mimeType } of base64DataArray) {
-            try {
-                const metadata = await this.extractImageMetadata(data, mimeType);
-                results.push(this.calculateTokens(metadata));
-            }
-            catch (error) {
-                debugLogger.warn('Error calculating tokens for image:', error);
-                // Return minimum tokens as fallback
-                results.push(ImageTokenizer.MIN_TOKENS_PER_IMAGE +
-                    ImageTokenizer.VISION_SPECIAL_TOKENS);
-            }
-        }
-        return results;
-    }
-    /**
-     * Extract BMP dimensions from header
-     * BMP signature: 42 4D (BM)
-     * Width/height at bytes 18-21 and 22-25 (little-endian)
-     */
-    extractBmpDimensions(buffer) {
-        if (buffer.length < 26) {
-            throw new Error('Invalid BMP: buffer too short');
-        }
-        // Verify BMP signature
-        if (buffer[0] !== 0x42 || buffer[1] !== 0x4d) {
-            throw new Error('Invalid BMP signature');
-        }
-        const width = buffer.readUInt32LE(18);
-        const height = buffer.readUInt32LE(22);
-        return { width, height: Math.abs(height) }; // Height can be negative for top-down BMPs
-    }
-    /**
-     * Extract TIFF dimensions from IFD (Image File Directory)
-     * TIFF can be little-endian (II) or big-endian (MM)
-     * Width/height are stored in IFD entries with tags 0x0100 and 0x0101
-     */
-    extractTiffDimensions(buffer) {
-        if (buffer.length < 8) {
-            throw new Error('Invalid TIFF: buffer too short');
-        }
-        // Check byte order
-        const byteOrder = buffer.subarray(0, 2).toString('ascii');
-        const isLittleEndian = byteOrder === 'II';
-        const isBigEndian = byteOrder === 'MM';
-        if (!isLittleEndian && !isBigEndian) {
-            throw new Error('Invalid TIFF byte order');
-        }
-        // Read magic number (should be 42)
-        const magic = isLittleEndian
-            ? buffer.readUInt16LE(2)
-            : buffer.readUInt16BE(2);
-        if (magic !== 42) {
-            throw new Error('Invalid TIFF magic number');
-        }
-        // Read IFD offset
-        const ifdOffset = isLittleEndian
-            ? buffer.readUInt32LE(4)
-            : buffer.readUInt32BE(4);
-        if (ifdOffset >= buffer.length) {
-            throw new Error('Invalid TIFF IFD offset');
-        }
-        // Read number of directory entries
-        const numEntries = isLittleEndian
-            ? buffer.readUInt16LE(ifdOffset)
-            : buffer.readUInt16BE(ifdOffset);
-        let width = 0;
-        let height = 0;
-        // Parse IFD entries
-        for (let i = 0; i < numEntries; i++) {
-            const entryOffset = ifdOffset + 2 + i * 12;
-            if (entryOffset + 12 > buffer.length)
-                break;
-            const tag = isLittleEndian
-                ? buffer.readUInt16LE(entryOffset)
-                : buffer.readUInt16BE(entryOffset);
-            const type = isLittleEndian
-                ? buffer.readUInt16LE(entryOffset + 2)
-                : buffer.readUInt16BE(entryOffset + 2);
-            const value = isLittleEndian
-                ? buffer.readUInt32LE(entryOffset + 8)
-                : buffer.readUInt32BE(entryOffset + 8);
-            if (tag === 0x0100) {
-                // ImageWidth
-                width = type === 3 ? value : value; // SHORT or LONG
-            }
-            else if (tag === 0x0101) {
-                // ImageLength (height)
-                height = type === 3 ? value : value; // SHORT or LONG
-            }
-            if (width > 0 && height > 0)
-                break;
-        }
-        if (width === 0 || height === 0) {
-            throw new Error('Could not find TIFF dimensions');
-        }
-        return { width, height };
-    }
-    /**
-     * Extract HEIC dimensions from meta box
-     * HEIC is based on ISO Base Media File Format
-     * This is a simplified implementation that looks for 'ispe' (Image Spatial Extents) box
-     */
-    extractHeicDimensions(buffer) {
-        if (buffer.length < 12) {
-            throw new Error('Invalid HEIC: buffer too short');
-        }
-        // Check for ftyp box with HEIC brand
-        const ftypBox = buffer.subarray(4, 8).toString('ascii');
-        if (ftypBox !== 'ftyp') {
-            throw new Error('Invalid HEIC: missing ftyp box');
-        }
-        const brand = buffer.subarray(8, 12).toString('ascii');
-        if (!['heic', 'heix', 'hevc', 'hevx'].includes(brand)) {
-            throw new Error('Invalid HEIC brand');
-        }
-        // Look for meta box and then ispe box
-        let offset = 0;
-        while (offset < buffer.length - 8) {
-            const boxSize = buffer.readUInt32BE(offset);
-            const boxType = buffer.subarray(offset + 4, offset + 8).toString('ascii');
-            if (boxType === 'meta') {
-                // Look for ispe box inside meta box
-                const metaOffset = offset + 8;
-                let innerOffset = metaOffset + 4; // Skip version and flags
-                while (innerOffset < offset + boxSize - 8) {
-                    const innerBoxSize = buffer.readUInt32BE(innerOffset);
-                    const innerBoxType = buffer
-                        .subarray(innerOffset + 4, innerOffset + 8)
-                        .toString('ascii');
-                    if (innerBoxType === 'ispe') {
-                        // Found Image Spatial Extents box
-                        if (innerOffset + 20 <= buffer.length) {
-                            const width = buffer.readUInt32BE(innerOffset + 12);
-                            const height = buffer.readUInt32BE(innerOffset + 16);
-                            return { width, height };
-                        }
-                    }
-                    if (innerBoxSize === 0)
-                        break;
-                    innerOffset += innerBoxSize;
-                }
-            }
-            if (boxSize === 0)
-                break;
-            offset += boxSize;
-        }
-        // Fallback: return default dimensions if we can't parse the structure
-        debugLogger.warn('Could not extract HEIC dimensions, using default');
-        return { width: 512, height: 512 };
-    }
-}
-//# sourceMappingURL=imageTokenizer.js.map

package/dist/CLI/packages/core/src/utils/request-tokenizer/imageTokenizer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"imageTokenizer.js","sourceRoot":"","sources":["../../../../../../../../core/src/utils/request-tokenizer/imageTokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,wBAAwB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,WAAW,GAAG,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;AAEzD;;;;;;;;;GASG;AACH,MAAM,OAAO,cAAc;IACzB,6BAA6B;IACrB,MAAM,CAAU,gBAAgB,GAAG,EAAE,GAAG,EAAE,CAAC;IAEnD,+BAA+B;IACvB,MAAM,CAAU,oBAAoB,GAAG,CAAC,CAAC;IAEjD,+BAA+B;IACvB,MAAM,CAAU,oBAAoB,GAAG,KAAK,CAAC;IAErD,wCAAwC;IAChC,MAAM,CAAU,qBAAqB,GAAG,CAAC,CAAC;IAElD;;;;;;OAMG;IACH,KAAK,CAAC,oBAAoB,CACxB,UAAkB,EAClB,QAAgB;QAEhB,IAAI,CAAC;YACH,sCAAsC;YACtC,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxC,WAAW,CAAC,IAAI,CAAC,6BAA6B,QAAQ,EAAE,CAAC,CAAC;gBAC1D,kDAAkD;gBAClD,OAAO;oBACL,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,GAAG;oBACX,QAAQ;oBACR,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC;iBAC/C,CAAC;YACJ,CAAC;YAED,MAAM,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;YAClE,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YAClD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAElE,OAAO;gBACL,KAAK,EAAE,UAAU,CAAC,KAAK;gBACvB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ;gBACR,QAAQ,EAAE,MAAM,CAAC,MAAM;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,WAAW,CAAC,IAAI,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;YAC7D,uCAAuC;YACvC,OAAO;gBACL,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,QAAQ;gBACR,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC;aAC/C,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,iBAAiB,CAC7B,MAAc,EACd,QAAgB;QAEhB,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1D,OAAO,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACK,oBAAoB,CAAC,MAAc;QAIzC,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,uBAAuB;QACvB,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxC,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC;YACpC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;SAC/C,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAEvC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,MAAc;QAI1C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,OAAO,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC5B,MAAM,EAAE,CAAC;gBACT,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAElC,cAAc;YACd,IACE,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,CAAC;gBAClC,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,CAAC;gBAClC,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,CAAC;gBAClC,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,CAAC,EAClC,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;YAC3B,CAAC;YAED,MAAM,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACtD,MAAM,IAAI,CAAC,GAAG,aAAa,CAAC;QAC9B,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED;;;OAGG;IACK,qBAAqB,CAAC,MAAc;QAI1C,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC9D,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAE/D,IAAI,aAAa,KAAK,MAAM,IAAI,aAAa,KAAK,MAAM,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAEzD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;YAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;YAChD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAC3B,CAAC;aAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YACrC,MAAM,KAAK,GAAG,CAAC,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAC3B,CAAC;aAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YACxD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAC3B,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACK,oBAAoB,CAAC,MAAc;QAIzC,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1D,IAAI,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAEtC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,QAAuB;QACrC,OAAO,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC;IAED;;;;;;;;;;;OAWG;IACK,0BAA0B,CAAC,KAAa,EAAE,MAAc;QAC9D,kCAAkC;QAClC,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QACxC,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAEvC,0BAA0B;QAC1B,MAAM,SAAS,GACb,cAAc,CAAC,oBAAoB,GAAG,cAAc,CAAC,gBAAgB,CAAC;QACxE,MAAM,SAAS,GACb,cAAc,CAAC,oBAAoB,GAAG,cAAc,CAAC,gBAAgB,CAAC;QAExE,gBAAgB;QAChB,IAAI,IAAI,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC;YAC5B,0BAA0B;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;YACrD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;YAC3C,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAC5C,CAAC;aAAM,IAAI,IAAI,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC;YACnC,wBAAwB;YACxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;YACrD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;YAC5C,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAC7C,CAAC;QAED,mBAAmB;QACnB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAC5B,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,cAAc,CAAC,gBAAgB,CAChD,CAAC;QAEF,OAAO,WAAW,GAAG,cAAc,CAAC,qBAAqB,CAAC;IAC5D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,oBAAoB,CACxB,eAA0D;QAE1D,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,eAAe,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBACjE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC/C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,WAAW,CAAC,IAAI,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAC;gBAC/D,oCAAoC;gBACpC,OAAO,CAAC,IAAI,CACV,cAAc,CAAC,oBAAoB;oBACjC,cAAc,CAAC,qBAAqB,CACvC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACK,oBAAoB,CAAC,MAAc;QAIzC,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,uBAAuB;QACvB,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAEvC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,2CAA2C;IACzF,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,MAAc;QAI1C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QAED,mBAAmB;QACnB,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1D,MAAM,cAAc,GAAG,SAAS,KAAK,IAAI,CAAC;QAC1C,MAAM,WAAW,GAAG,SAAS,KAAK,IAAI,CAAC;QAEvC,IAAI,CAAC,cAAc,IAAI,CAAC,WAAW,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,mCAAmC;QACnC,MAAM,KAAK,GAAG,cAAc;YAC1B,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;YACxB,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,kBAAkB;QAClB,MAAM,SAAS,GAAG,cAAc;YAC9B,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;YACxB,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAE3B,IAAI,SAAS,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,mCAAmC;QACnC,MAAM,UAAU,GAAG,cAAc;YAC/B,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC;YAChC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAEnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,oBAAoB;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,WAAW,GAAG,SAAS,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;YAE3C,IAAI,WAAW,GAAG,EAAE,GAAG,MAAM,CAAC,MAAM;gBAAE,MAAM;YAE5C,MAAM,GAAG,GAAG,cAAc;gBACxB,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,CAAC;gBAClC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;YAErC,MAAM,IAAI,GAAG,cAAc;gBACzB,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBACtC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;YAEzC,MAAM,KAAK,GAAG,cAAc;gBAC1B,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBACtC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;YAEzC,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;gBACnB,aAAa;gBACb,KAAK,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,gBAAgB;YACtD,CAAC;iBAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;gBAC1B,uBAAuB;gBACvB,MAAM,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,gBAAgB;YACvD,CAAC;YAED,IAAI,KAAK,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC;gBAAE,MAAM;QACrC,CAAC;QAED,IAAI,KAAK,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,MAAc;QAI1C,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QAED,qCAAqC;QACrC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACxD,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QAED,sCAAsC;QACtC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,OAAO,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE1E,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;gBACvB,oCAAoC;gBACpC,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;gBAC9B,IAAI,WAAW,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,yBAAyB;gBAE3D,OAAO,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,CAAC,EAAE,CAAC;oBAC1C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;oBACtD,MAAM,YAAY,GAAG,MAAM;yBACxB,QAAQ,CAAC,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC;yBAC1C,QAAQ,CAAC,OAAO,CAAC,CAAC;oBAErB,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;wBAC5B,kCAAkC;wBAClC,IAAI,WAAW,GAAG,EAAE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;4BACtC,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC;4BACpD,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC;4BACrD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;wBAC3B,CAAC;oBACH,CAAC;oBAED,IAAI,YAAY,KAAK,CAAC;wBAAE,MAAM;oBAC9B,WAAW,IAAI,YAAY,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,IAAI,OAAO,KAAK,CAAC;gBAAE,MAAM;YACzB,MAAM,IAAI,OAAO,CAAC;QACpB,CAAC;QAED,sEAAsE;QACtE,WAAW,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QACrE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IACrC,CAAC"}

package/dist/CLI/packages/core/src/utils/request-tokenizer/index.d.ts

@@ -1,9 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-export { RequestTokenizer as RequestTokenEstimator } from './requestTokenizer.js';
-export { TextTokenizer } from './textTokenizer.js';
-export { ImageTokenizer } from './imageTokenizer.js';
-export type { TokenCalculationResult, ImageMetadata } from './types.js';

package/dist/CLI/packages/core/src/utils/request-tokenizer/index.js

@@ -1,9 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-export { RequestTokenizer as RequestTokenEstimator } from './requestTokenizer.js';
-export { TextTokenizer } from './textTokenizer.js';
-export { ImageTokenizer } from './imageTokenizer.js';
-//# sourceMappingURL=index.js.map

package/dist/CLI/packages/core/src/utils/request-tokenizer/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../../../core/src/utils/request-tokenizer/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,gBAAgB,IAAI,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/CLI/packages/core/src/utils/request-tokenizer/requestTokenizer.d.ts

@@ -1,52 +0,0 @@
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */
-import type { CountTokensParameters } from '@google/genai';
-import type { TokenCalculationResult } from './types.js';
-/**
- * Simple request token estimator that handles text and image content serially
- */
-export declare class RequestTokenizer {
-    private textTokenizer;
-    private imageTokenizer;
-    constructor();
-    /**
-     * Calculate tokens for a request using serial processing
-     */
-    calculateTokens(request: CountTokensParameters): Promise<TokenCalculationResult>;
-    /**
-     * Calculate tokens for text contents
-     */
-    private calculateTextTokens;
-    /**
-     * Calculate tokens for image contents using serial processing
-     */
-    private calculateImageTokens;
-    /**
-     * Calculate tokens for audio contents
-     * TODO: Implement proper audio token calculation
-     */
-    private calculateAudioTokens;
-    /**
-     * Calculate tokens for other content types (functions, files, etc.)
-     */
-    private calculateOtherTokens;
-    /**
-     * Fallback token calculation using simple string serialization
-     */
-    private calculateFallbackTokens;
-    /**
-     * Process request contents and group by type
-     */
-    private processAndGroupContents;
-    /**
-     * Process a single content item and add to appropriate arrays
-     */
-    private processContent;
-    /**
-     * Process a single part and add to appropriate arrays
-     */
-    private processPart;
-}