@qwen-code/qwen-code 0.22.0 → 0.22.1

package/chunks/chunk-IO257NUD.js

@@ -1,623 +0,0 @@
-// Force strict mode and setup for ESM
-"use strict";
-import {
-  TextTokenizer,
-  isSupportedImageMimeType
-} from "./chunk-YHC2EYYG.js";
-import {
-  createDebugLogger
-} from "./chunk-A5F2YNO6.js";
-import {
-  init_esbuild_shims
-} from "./chunk-5O2XNYP6.js";
-import {
-  __name
-} from "./chunk-J2S4EL5Y.js";
-
-// packages/core/src/utils/request-tokenizer/requestTokenizer.ts
-init_esbuild_shims();
-
-// packages/core/src/utils/request-tokenizer/imageTokenizer.ts
-init_esbuild_shims();
-var debugLogger = createDebugLogger("IMAGE_TOKENIZER");
-var ImageTokenizer = class _ImageTokenizer {
-  static {
-    __name(this, "ImageTokenizer");
-  }
-  /** 28x28 pixels = 1 token */
-  static PIXELS_PER_TOKEN = 28 * 28;
-  /** Minimum tokens per image */
-  static MIN_TOKENS_PER_IMAGE = 4;
-  /** Maximum tokens per image */
-  static MAX_TOKENS_PER_IMAGE = 16384;
-  /** Special tokens for vision markers */
-  static VISION_SPECIAL_TOKENS = 2;
-  /**
-   * Extract image metadata from base64 data
-   *
-   * @param base64Data Base64-encoded image data (with or without data URL prefix)
-   * @param mimeType MIME type of the image
-   * @returns Promise resolving to ImageMetadata with dimensions and format info
-   */
-  async extractImageMetadata(base64Data, mimeType) {
-    try {
-      if (!isSupportedImageMimeType(mimeType)) {
-        debugLogger.warn(`Unsupported image format: ${mimeType}`);
-        return {
-          width: 512,
-          height: 512,
-          mimeType,
-          dataSize: Math.floor(base64Data.length * 0.75)
-        };
-      }
-      const cleanBase64 = base64Data.replace(/^data:[^;]+;base64,/, "");
-      const buffer = Buffer.from(cleanBase64, "base64");
-      const dimensions = await this.extractDimensions(buffer, mimeType);
-      return {
-        width: dimensions.width,
-        height: dimensions.height,
-        mimeType,
-        dataSize: buffer.length
-      };
-    } catch (error) {
-      debugLogger.warn("Failed to extract image metadata:", error);
-      return {
-        width: 512,
-        height: 512,
-        mimeType,
-        dataSize: Math.floor(base64Data.length * 0.75)
-      };
-    }
-  }
-  /**
-   * Extract image dimensions from buffer based on format
-   *
-   * @param buffer Binary image data buffer
-   * @param mimeType MIME type to determine parsing strategy
-   * @returns Promise resolving to width and height dimensions
-   */
-  async extractDimensions(buffer, mimeType) {
-    if (mimeType.includes("png")) {
-      return this.extractPngDimensions(buffer);
-    }
-    if (mimeType.includes("jpeg") || mimeType.includes("jpg")) {
-      return this.extractJpegDimensions(buffer);
-    }
-    if (mimeType.includes("webp")) {
-      return this.extractWebpDimensions(buffer);
-    }
-    if (mimeType.includes("gif")) {
-      return this.extractGifDimensions(buffer);
-    }
-    if (mimeType.includes("bmp")) {
-      return this.extractBmpDimensions(buffer);
-    }
-    if (mimeType.includes("tiff")) {
-      return this.extractTiffDimensions(buffer);
-    }
-    if (mimeType.includes("heic")) {
-      return this.extractHeicDimensions(buffer);
-    }
-    return { width: 512, height: 512 };
-  }
-  /**
-   * Extract PNG dimensions from IHDR chunk
-   * PNG signature: 89 50 4E 47 0D 0A 1A 0A
-   * Width/height at bytes 16-19 and 20-23 (big-endian)
-   */
-  extractPngDimensions(buffer) {
-    if (buffer.length < 24) {
-      throw new Error("Invalid PNG: buffer too short");
-    }
-    const signature = buffer.subarray(0, 8);
-    const expectedSignature = Buffer.from([
-      137,
-      80,
-      78,
-      71,
-      13,
-      10,
-      26,
-      10
-    ]);
-    if (!signature.equals(expectedSignature)) {
-      throw new Error("Invalid PNG signature");
-    }
-    const width = buffer.readUInt32BE(16);
-    const height = buffer.readUInt32BE(20);
-    return { width, height };
-  }
-  /**
-   * Extract JPEG dimensions from SOF (Start of Frame) markers
-   * JPEG starts with FF D8, SOF markers: 0xC0-0xC3, 0xC5-0xC7, 0xC9-0xCB, 0xCD-0xCF
-   * Dimensions at offset +5 (height) and +7 (width) from SOF marker
-   */
-  extractJpegDimensions(buffer) {
-    if (buffer.length < 4 || buffer[0] !== 255 || buffer[1] !== 216) {
-      throw new Error("Invalid JPEG signature");
-    }
-    let offset = 2;
-    while (offset < buffer.length - 8) {
-      if (buffer[offset] !== 255) {
-        offset++;
-        continue;
-      }
-      const marker = buffer[offset + 1];
-      if (marker >= 192 && marker <= 195 || marker >= 197 && marker <= 199 || marker >= 201 && marker <= 203 || marker >= 205 && marker <= 207) {
-        const height = buffer.readUInt16BE(offset + 5);
-        const width = buffer.readUInt16BE(offset + 7);
-        return { width, height };
-      }
-      const segmentLength = buffer.readUInt16BE(offset + 2);
-      offset += 2 + segmentLength;
-    }
-    throw new Error("Could not find JPEG dimensions");
-  }
-  /**
-   * Extract WebP dimensions from RIFF container
-   * Supports VP8, VP8L, and VP8X formats
-   */
-  extractWebpDimensions(buffer) {
-    if (buffer.length < 16) {
-      throw new Error("Invalid WebP: too short");
-    }
-    const riffSignature = buffer.subarray(0, 4).toString("ascii");
-    const webpSignature = buffer.subarray(8, 12).toString("ascii");
-    if (riffSignature !== "RIFF" || webpSignature !== "WEBP") {
-      throw new Error("Invalid WebP signature");
-    }
-    const format = buffer.subarray(12, 16).toString("ascii");
-    if (format === "VP8 ") {
-      if (buffer.length < 30) {
-        throw new Error("Invalid VP8 WebP: too short");
-      }
-      const width = buffer.readUInt16LE(26) & 16383;
-      const height = buffer.readUInt16LE(28) & 16383;
-      return { width, height };
-    } else if (format === "VP8L") {
-      if (buffer.length < 25) {
-        throw new Error("Invalid VP8L WebP: too short");
-      }
-      if (buffer[20] !== 47) {
-        throw new Error("Invalid VP8L WebP signature");
-      }
-      const bits = buffer.readUInt32LE(21);
-      const width = (bits & 16383) + 1;
-      const height = (bits >> 14 & 16383) + 1;
-      return { width, height };
-    } else if (format === "VP8X") {
-      if (buffer.length < 30) {
-        throw new Error("Invalid VP8X WebP: too short");
-      }
-      const width = buffer.readUIntLE(24, 3) + 1;
-      const height = buffer.readUIntLE(27, 3) + 1;
-      return { width, height };
-    }
-    throw new Error("Unsupported WebP format");
-  }
-  /**
-   * Extract GIF dimensions from header
-   * Supports GIF87a and GIF89a formats
-   */
-  extractGifDimensions(buffer) {
-    if (buffer.length < 10) {
-      throw new Error("Invalid GIF: too short");
-    }
-    const signature = buffer.subarray(0, 6).toString("ascii");
-    if (signature !== "GIF87a" && signature !== "GIF89a") {
-      throw new Error("Invalid GIF signature");
-    }
-    const width = buffer.readUInt16LE(6);
-    const height = buffer.readUInt16LE(8);
-    return { width, height };
-  }
-  /**
-   * Calculate tokens for an image based on its metadata
-   *
-   * @param metadata Image metadata containing width, height, and format info
-   * @returns Total token count including base image tokens and special tokens
-   */
-  calculateTokens(metadata) {
-    return this.calculateTokensWithScaling(metadata.width, metadata.height);
-  }
-  /**
-   * Calculate tokens with scaling logic
-   *
-   * Steps:
-   * 1. Normalize to 28-pixel multiples
-   * 2. Scale large images down, small images up
-   * 3. Calculate tokens: pixels / 784 + 2 special tokens
-   *
-   * @param width Original image width in pixels
-   * @param height Original image height in pixels
-   * @returns Total token count for the image
-   */
-  calculateTokensWithScaling(width, height) {
-    let hBar = Math.round(height / 28) * 28;
-    let wBar = Math.round(width / 28) * 28;
-    const minPixels = _ImageTokenizer.MIN_TOKENS_PER_IMAGE * _ImageTokenizer.PIXELS_PER_TOKEN;
-    const maxPixels = _ImageTokenizer.MAX_TOKENS_PER_IMAGE * _ImageTokenizer.PIXELS_PER_TOKEN;
-    if (hBar * wBar > maxPixels) {
-      const beta = Math.sqrt(height * width / maxPixels);
-      hBar = Math.floor(height / beta / 28) * 28;
-      wBar = Math.floor(width / beta / 28) * 28;
-    } else if (hBar * wBar < minPixels) {
-      const beta = Math.sqrt(minPixels / (height * width));
-      hBar = Math.ceil(height * beta / 28) * 28;
-      wBar = Math.ceil(width * beta / 28) * 28;
-    }
-    const imageTokens = Math.floor(
-      hBar * wBar / _ImageTokenizer.PIXELS_PER_TOKEN
-    );
-    return imageTokens + _ImageTokenizer.VISION_SPECIAL_TOKENS;
-  }
-  /**
-   * Calculate tokens for multiple images serially
-   *
-   * @param base64DataArray Array of image data with MIME type information
-   * @returns Promise resolving to array of token counts in same order as input
-   */
-  async calculateTokensBatch(base64DataArray) {
-    const results = [];
-    for (const { data, mimeType } of base64DataArray) {
-      try {
-        const metadata = await this.extractImageMetadata(data, mimeType);
-        results.push(this.calculateTokens(metadata));
-      } catch (error) {
-        debugLogger.warn("Error calculating tokens for image:", error);
-        results.push(
-          _ImageTokenizer.MIN_TOKENS_PER_IMAGE + _ImageTokenizer.VISION_SPECIAL_TOKENS
-        );
-      }
-    }
-    return results;
-  }
-  /**
-   * Extract BMP dimensions from header
-   * BMP signature: 42 4D (BM)
-   * Width/height at bytes 18-21 and 22-25 (little-endian)
-   */
-  extractBmpDimensions(buffer) {
-    if (buffer.length < 26) {
-      throw new Error("Invalid BMP: buffer too short");
-    }
-    if (buffer[0] !== 66 || buffer[1] !== 77) {
-      throw new Error("Invalid BMP signature");
-    }
-    const width = buffer.readUInt32LE(18);
-    const height = buffer.readInt32LE(22);
-    return { width, height: Math.abs(height) };
-  }
-  /**
-   * Extract TIFF dimensions from IFD (Image File Directory)
-   * TIFF can be little-endian (II) or big-endian (MM)
-   * Width/height are stored in IFD entries with tags 0x0100 and 0x0101
-   */
-  extractTiffDimensions(buffer) {
-    if (buffer.length < 8) {
-      throw new Error("Invalid TIFF: buffer too short");
-    }
-    const byteOrder = buffer.subarray(0, 2).toString("ascii");
-    const isLittleEndian = byteOrder === "II";
-    const isBigEndian = byteOrder === "MM";
-    if (!isLittleEndian && !isBigEndian) {
-      throw new Error("Invalid TIFF byte order");
-    }
-    const magic = isLittleEndian ? buffer.readUInt16LE(2) : buffer.readUInt16BE(2);
-    if (magic !== 42) {
-      throw new Error("Invalid TIFF magic number");
-    }
-    const ifdOffset = isLittleEndian ? buffer.readUInt32LE(4) : buffer.readUInt32BE(4);
-    if (ifdOffset >= buffer.length) {
-      throw new Error("Invalid TIFF IFD offset");
-    }
-    const numEntries = isLittleEndian ? buffer.readUInt16LE(ifdOffset) : buffer.readUInt16BE(ifdOffset);
-    let width = 0;
-    let height = 0;
-    for (let i = 0; i < numEntries; i++) {
-      const entryOffset = ifdOffset + 2 + i * 12;
-      if (entryOffset + 12 > buffer.length) break;
-      const tag = isLittleEndian ? buffer.readUInt16LE(entryOffset) : buffer.readUInt16BE(entryOffset);
-      const type = isLittleEndian ? buffer.readUInt16LE(entryOffset + 2) : buffer.readUInt16BE(entryOffset + 2);
-      const value = type === 3 ? isLittleEndian ? buffer.readUInt16LE(entryOffset + 8) : buffer.readUInt16BE(entryOffset + 8) : isLittleEndian ? buffer.readUInt32LE(entryOffset + 8) : buffer.readUInt32BE(entryOffset + 8);
-      if (tag === 256) {
-        width = value;
-      } else if (tag === 257) {
-        height = value;
-      }
-      if (width > 0 && height > 0) break;
-    }
-    if (width === 0 || height === 0) {
-      throw new Error("Could not find TIFF dimensions");
-    }
-    return { width, height };
-  }
-  /**
-   * Extract HEIC dimensions from meta box
-   * HEIC is based on ISO Base Media File Format
-   * This is a simplified implementation that looks for 'ispe' (Image Spatial Extents) box
-   */
-  extractHeicDimensions(buffer) {
-    if (buffer.length < 12) {
-      throw new Error("Invalid HEIC: buffer too short");
-    }
-    const ftypBox = buffer.subarray(4, 8).toString("ascii");
-    if (ftypBox !== "ftyp") {
-      throw new Error("Invalid HEIC: missing ftyp box");
-    }
-    const brand = buffer.subarray(8, 12).toString("ascii");
-    if (!["heic", "heix", "hevc", "hevx"].includes(brand)) {
-      throw new Error("Invalid HEIC brand");
-    }
-    let offset = 0;
-    while (offset < buffer.length - 8) {
-      const boxSize = buffer.readUInt32BE(offset);
-      const boxType = buffer.subarray(offset + 4, offset + 8).toString("ascii");
-      if (boxType === "meta") {
-        const metaOffset = offset + 8;
-        let innerOffset = metaOffset + 4;
-        while (innerOffset < offset + boxSize - 8) {
-          const innerBoxSize = buffer.readUInt32BE(innerOffset);
-          const innerBoxType = buffer.subarray(innerOffset + 4, innerOffset + 8).toString("ascii");
-          if (innerBoxType === "ispe") {
-            if (innerOffset + 20 <= buffer.length) {
-              const width = buffer.readUInt32BE(innerOffset + 12);
-              const height = buffer.readUInt32BE(innerOffset + 16);
-              return { width, height };
-            }
-          }
-          if (innerBoxSize === 0) break;
-          innerOffset += innerBoxSize;
-        }
-      }
-      if (boxSize === 0) break;
-      offset += boxSize;
-    }
-    debugLogger.warn("Could not extract HEIC dimensions, using default");
-    return { width: 512, height: 512 };
-  }
-};
-
-// packages/core/src/utils/request-tokenizer/requestTokenizer.ts
-var debugLogger2 = createDebugLogger("TOKENIZER");
-var RequestTokenizer = class {
-  static {
-    __name(this, "RequestTokenizer");
-  }
-  textTokenizer;
-  imageTokenizer;
-  constructor() {
-    this.textTokenizer = new TextTokenizer();
-    this.imageTokenizer = new ImageTokenizer();
-  }
-  /**
-   * Calculate tokens for a request using serial processing
-   */
-  async calculateTokens(request) {
-    const startTime = performance.now();
-    try {
-      const { textContents, imageContents, audioContents, otherContents } = this.processAndGroupContents(request);
-      if (textContents.length === 0 && imageContents.length === 0 && audioContents.length === 0 && otherContents.length === 0) {
-        return {
-          totalTokens: 0,
-          breakdown: {
-            textTokens: 0,
-            imageTokens: 0,
-            audioTokens: 0,
-            otherTokens: 0
-          },
-          processingTime: performance.now() - startTime
-        };
-      }
-      const textTokens = await this.calculateTextTokens(textContents);
-      const imageTokens = await this.calculateImageTokens(imageContents);
-      const audioTokens = await this.calculateAudioTokens(audioContents);
-      const otherTokens = await this.calculateOtherTokens(otherContents);
-      const totalTokens = textTokens + imageTokens + audioTokens + otherTokens;
-      const processingTime = performance.now() - startTime;
-      return {
-        totalTokens,
-        breakdown: {
-          textTokens,
-          imageTokens,
-          audioTokens,
-          otherTokens
-        },
-        processingTime
-      };
-    } catch (error) {
-      debugLogger2.error("Error calculating tokens:", error);
-      const fallbackTokens = this.calculateFallbackTokens(request);
-      return {
-        totalTokens: fallbackTokens,
-        breakdown: {
-          textTokens: fallbackTokens,
-          imageTokens: 0,
-          audioTokens: 0,
-          otherTokens: 0
-        },
-        processingTime: performance.now() - startTime
-      };
-    }
-  }
-  /**
-   * Calculate tokens for text contents
-   */
-  async calculateTextTokens(textContents) {
-    if (textContents.length === 0) return 0;
-    try {
-      return await this.textTokenizer.calculateTokens(textContents.join(""));
-    } catch (error) {
-      debugLogger2.warn("Error calculating text tokens:", error);
-      const totalChars = textContents.join("").length;
-      return Math.ceil(totalChars / 4);
-    }
-  }
-  /**
-   * Calculate tokens for image contents using serial processing
-   */
-  async calculateImageTokens(imageContents) {
-    if (imageContents.length === 0) return 0;
-    try {
-      const tokenCounts = await this.imageTokenizer.calculateTokensBatch(imageContents);
-      return tokenCounts.reduce((sum, count) => sum + count, 0);
-    } catch (error) {
-      debugLogger2.warn("Error calculating image tokens:", error);
-      return imageContents.length * 6;
-    }
-  }
-  /**
-   * Calculate tokens for audio contents
-   * TODO: Implement proper audio token calculation
-   */
-  async calculateAudioTokens(audioContents) {
-    if (audioContents.length === 0) return 0;
-    let totalTokens = 0;
-    for (const audioContent of audioContents) {
-      try {
-        const dataSize = Math.floor(audioContent.data.length * 0.75);
-        totalTokens += Math.max(Math.ceil(dataSize / 100), 10);
-      } catch (error) {
-        debugLogger2.warn("Error calculating audio tokens:", error);
-        totalTokens += 10;
-      }
-    }
-    return totalTokens;
-  }
-  /**
-   * Calculate tokens for other content types (functions, files, etc.)
-   */
-  async calculateOtherTokens(otherContents) {
-    if (otherContents.length === 0) return 0;
-    try {
-      return await this.textTokenizer.calculateTokens(otherContents.join(""));
-    } catch (error) {
-      debugLogger2.warn("Error calculating other content tokens:", error);
-      const totalChars = otherContents.join("").length;
-      return Math.ceil(totalChars / 4);
-    }
-  }
-  /**
-   * Fallback token calculation using simple string serialization
-   */
-  calculateFallbackTokens(request) {
-    try {
-      const content = JSON.stringify(request.contents);
-      return Math.ceil(content.length / 4);
-    } catch (error) {
-      debugLogger2.warn("Error in fallback token calculation:", error);
-      return 100;
-    }
-  }
-  /**
-   * Process request contents and group by type
-   */
-  processAndGroupContents(request) {
-    const textContents = [];
-    const imageContents = [];
-    const audioContents = [];
-    const otherContents = [];
-    if (!request.contents) {
-      return { textContents, imageContents, audioContents, otherContents };
-    }
-    const contents = Array.isArray(request.contents) ? request.contents : [request.contents];
-    for (const content of contents) {
-      this.processContent(
-        content,
-        textContents,
-        imageContents,
-        audioContents,
-        otherContents
-      );
-    }
-    return { textContents, imageContents, audioContents, otherContents };
-  }
-  /**
-   * Process a single content item and add to appropriate arrays
-   */
-  processContent(content, textContents, imageContents, audioContents, otherContents) {
-    if (typeof content === "string") {
-      if (content.trim()) {
-        textContents.push(content);
-      }
-      return;
-    }
-    if ("parts" in content && content.parts) {
-      for (const part of content.parts) {
-        this.processPart(
-          part,
-          textContents,
-          imageContents,
-          audioContents,
-          otherContents
-        );
-      }
-      return;
-    }
-    this.processPart(
-      content,
-      textContents,
-      imageContents,
-      audioContents,
-      otherContents
-    );
-  }
-  /**
-   * Process a single part and add to appropriate arrays
-   */
-  processPart(part, textContents, imageContents, audioContents, otherContents) {
-    if (typeof part === "string") {
-      if (part.trim()) {
-        textContents.push(part);
-      }
-      return;
-    }
-    if ("text" in part && part.text) {
-      textContents.push(part.text);
-      return;
-    }
-    if ("inlineData" in part && part.inlineData) {
-      const { data, mimeType } = part.inlineData;
-      if (mimeType && mimeType.startsWith("image/")) {
-        imageContents.push({ data: data || "", mimeType });
-        return;
-      }
-      if (mimeType && mimeType.startsWith("audio/")) {
-        audioContents.push({ data: data || "", mimeType });
-        return;
-      }
-    }
-    if ("fileData" in part && part.fileData) {
-      otherContents.push(JSON.stringify(part.fileData));
-      return;
-    }
-    if ("functionCall" in part && part.functionCall) {
-      otherContents.push(JSON.stringify(part.functionCall));
-      return;
-    }
-    if ("functionResponse" in part && part.functionResponse) {
-      otherContents.push(JSON.stringify(part.functionResponse));
-      return;
-    }
-    try {
-      const serialized = JSON.stringify(part);
-      if (serialized && serialized !== "{}") {
-        otherContents.push(serialized);
-      }
-    } catch (error) {
-      debugLogger2.warn("Failed to serialize unknown part type:", error);
-    }
-  }
-};
-
-// packages/core/src/utils/request-tokenizer/index.ts
-init_esbuild_shims();
-
-export {
-  RequestTokenizer
-};
-/**
- * @license
- * Copyright 2025 Qwen
- * SPDX-License-Identifier: Apache-2.0
- */