@aiscene/shared 8.0.3 → 8.0.5

package/src/env/types.ts

@@ -10,6 +10,7 @@ export const MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG =
 export const MIDSCENE_DEBUG_MODE = 'MIDSCENE_DEBUG_MODE';
 export const MIDSCENE_MCP_USE_PUPPETEER_MODE =
   'MIDSCENE_MCP_USE_PUPPETEER_MODE';
+export const MIDSCENE_CHROME_PATH = 'MIDSCENE_CHROME_PATH';
 export const MIDSCENE_MCP_CHROME_PATH = 'MIDSCENE_MCP_CHROME_PATH';
 export const MIDSCENE_MCP_ANDROID_MODE = 'MIDSCENE_MCP_ANDROID_MODE';
 export const DOCKER_CONTAINER = 'DOCKER_CONTAINER';
@@ -35,6 +36,10 @@ export const MIDSCENE_MODEL_REASONING_ENABLED =
   'MIDSCENE_MODEL_REASONING_ENABLED';
 export const MIDSCENE_MODEL_REASONING_BUDGET =
   'MIDSCENE_MODEL_REASONING_BUDGET';
+export const MIDSCENE_MODEL_RESPONSE_FORMAT = 'MIDSCENE_MODEL_RESPONSE_FORMAT';
+
+export type TModelReasoningEnabled = boolean | 'default';
+export type TModelResponseFormat = 'none' | 'auto';
 
 /**
  * @deprecated Use MIDSCENE_MODEL_API_KEY instead. This is kept for backward compatibility.
@@ -92,6 +97,7 @@ export const MIDSCENE_REPLANNING_CYCLE_LIMIT =
   'MIDSCENE_REPLANNING_CYCLE_LIMIT';
 
 export const MIDSCENE_RUN_DIR = 'MIDSCENE_RUN_DIR';
+export const MIDSCENE_RECORD_MODEL_CALL = 'MIDSCENE_RECORD_MODEL_CALL';
 
 // INSIGHT (unified VQA and Grounding)
 export const MIDSCENE_INSIGHT_MODEL_NAME = 'MIDSCENE_INSIGHT_MODEL_NAME';
@@ -120,6 +126,8 @@ export const MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED =
   'MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED';
 export const MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET =
   'MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET';
+export const MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT =
+  'MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT';
 
 // PLANNING
 export const MIDSCENE_PLANNING_MODEL_NAME = 'MIDSCENE_PLANNING_MODEL_NAME';
@@ -150,6 +158,8 @@ export const MIDSCENE_PLANNING_MODEL_REASONING_ENABLED =
   'MIDSCENE_PLANNING_MODEL_REASONING_ENABLED';
 export const MIDSCENE_PLANNING_MODEL_REASONING_BUDGET =
   'MIDSCENE_PLANNING_MODEL_REASONING_BUDGET';
+export const MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT =
+  'MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT';
 export const MIDSCENE_MODEL_FAMILY = 'MIDSCENE_MODEL_FAMILY';
 
 /**
@@ -166,6 +176,7 @@ export const BASIC_ENV_KEYS = [
   MIDSCENE_DEBUG_MODEL_PROFILE,
   MIDSCENE_DEBUG_MODEL_RESPONSE,
   MIDSCENE_RUN_DIR,
+  MIDSCENE_RECORD_MODEL_CALL,
 ] as const;
 
 export const BOOLEAN_ENV_KEYS = [
@@ -195,6 +206,7 @@ export const STRING_ENV_KEYS = [
   MIDSCENE_REPORT_TAG_NAME,
   MIDSCENE_PREFERRED_LANGUAGE,
   MATCH_BY_POSITION,
+  MIDSCENE_CHROME_PATH,
   MIDSCENE_MCP_CHROME_PATH,
   DOCKER_CONTAINER,
 ] as const;
@@ -231,6 +243,7 @@ export const MODEL_ENV_KEYS = [
   MIDSCENE_MODEL_REASONING_EFFORT,
   MIDSCENE_MODEL_REASONING_ENABLED,
   MIDSCENE_MODEL_REASONING_BUDGET,
+  MIDSCENE_MODEL_RESPONSE_FORMAT,
   MIDSCENE_USE_VLM_UI_TARS,
   MIDSCENE_USE_QWEN_VL,
   MIDSCENE_USE_QWEN3_VL,
@@ -259,6 +272,7 @@ export const MODEL_ENV_KEYS = [
   MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT,
   MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED,
   MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET,
+  MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT,
   // PLANNING
   MIDSCENE_PLANNING_MODEL_NAME,
   MIDSCENE_PLANNING_MODEL_SOCKS_PROXY,
@@ -275,6 +289,7 @@ export const MODEL_ENV_KEYS = [
   MIDSCENE_PLANNING_MODEL_REASONING_EFFORT,
   MIDSCENE_PLANNING_MODEL_REASONING_ENABLED,
   MIDSCENE_PLANNING_MODEL_REASONING_BUDGET,
+  MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT,
   MIDSCENE_MODEL_FAMILY,
 ] as const;
 
@@ -294,6 +309,7 @@ export type TGlobalConfig = Record<TEnvKeys, string | undefined>;
 export type TModelFamily =
   | 'qwen2.5-vl'
   | 'qwen3-vl'
+  | 'qwen3'
   | 'qwen3.5'
   | 'qwen3.6'
   | 'doubao-vision'
@@ -305,7 +321,11 @@ export type TModelFamily =
   | 'glm-v'
   | 'auto-glm'
   | 'auto-glm-multilingual'
-  | 'gpt-5';
+  | 'gpt-5'
+  | 'deepseek'
+  | 'kimi'
+  | 'kimi3'
+  | 'xiaomi-mimo';
 
 export const MODEL_FAMILY_VALUES: TModelFamily[] = [
   'doubao-vision',
@@ -313,6 +333,7 @@ export const MODEL_FAMILY_VALUES: TModelFamily[] = [
   'gemini',
   'qwen2.5-vl',
   'qwen3-vl',
+  'qwen3',
   'qwen3.5',
   'qwen3.6',
   'vlm-ui-tars',
@@ -322,6 +343,10 @@ export const MODEL_FAMILY_VALUES: TModelFamily[] = [
   'auto-glm',
   'auto-glm-multilingual',
   'gpt-5',
+  'deepseek',
+  'kimi',
+  'kimi3',
+  'xiaomi-mimo',
 ];
 
 export interface IModelConfigForInsight {
@@ -391,6 +416,8 @@ export interface IModelConfigForDefault {
   [MIDSCENE_MODEL_REASONING_ENABLED]?: string;
   // reasoning budget (number as string)
   [MIDSCENE_MODEL_REASONING_BUDGET]?: string;
+  // Response format strategy (none/auto)
+  [MIDSCENE_MODEL_RESPONSE_FORMAT]?: TModelResponseFormat;
 }
 
 export interface IModelConfigForDefaultLegacy {
@@ -507,12 +534,13 @@ export interface IModelConfig {
    * Enable/disable reasoning for the model.
    * Passed through to model-family-specific parameters (e.g., enable_thinking for qwen, thinking.type for doubao/glm-v).
    */
-  reasoningEnabled?: boolean;
+  reasoningEnabled?: TModelReasoningEnabled;
   /**
    * Reasoning token budget for the model.
    * Passed through to model-family-specific parameters (e.g., thinking_budget for qwen).
    */
   reasoningBudget?: number;
+  responseFormat?: TModelResponseFormat;
   /**
    * Model family - unified model configuration
    * Maps directly to model families like 'qwen2.5-vl', 'qwen3-vl', 'doubao-vision', 'doubao-seed', etc.

package/src/img/index.ts

@@ -1,4 +1,5 @@
 export {
+  encodedImageInfoOfBuffer,
   imageInfoOfBase64,
   isValidPNGImageBuffer,
   isValidJPEGImageBuffer,
@@ -8,6 +9,10 @@ export {
 } from './info';
 export {
   resizeAndConvertImgBuffer,
+  convertImgBufferToJpeg,
+  convertBase64ImageToJpeg,
+  constrainBase64ImageToMaxSize,
+  resizeBase64ImageToJpeg,
   resizeImgBase64,
   zoomForGPT4o,
   saveBase64Image,
@@ -19,6 +24,13 @@ export {
   preProcessImageUrl,
   parseBase64,
   createImgBase64ByFormat,
+  inferBase64ImageFormat,
+  normalizeBase64Image,
+  normalizeScreenshotBase64,
+  type NormalizeScreenshotBase64Options,
+  type ConstrainBase64ImageToMaxSizeOptions,
+  type JpegBase64DataUrl,
+  type ResizeBase64ImageToJpegOptions,
 } from './transform';
 export {
   processImageElementInfo,

package/src/img/info.ts

@@ -1,9 +1,70 @@
 import assert from 'node:assert';
+import { Buffer } from 'node:buffer';
 import type { Size } from '../types';
 import getPhoton from './get-photon';
 
 export interface ImageInfo extends Size {}
 
+const isJpegStartOfFrameMarker = (marker: number): boolean =>
+  marker >= 0xc0 &&
+  marker <= 0xcf &&
+  marker !== 0xc4 &&
+  marker !== 0xc8 &&
+  marker !== 0xcc;
+
+function jpegInfoFromBuffer(imageBuffer: Buffer): ImageInfo {
+  let offset = 2;
+  while (offset < imageBuffer.length) {
+    if (imageBuffer[offset] !== 0xff) {
+      offset += 1;
+      continue;
+    }
+
+    while (offset < imageBuffer.length && imageBuffer[offset] === 0xff) {
+      offset += 1;
+    }
+    if (offset >= imageBuffer.length) break;
+
+    const marker = imageBuffer[offset];
+    offset += 1;
+    if (marker === 0xd9 || marker === 0xda) break;
+    if (
+      marker === 0x01 ||
+      marker === 0xd8 ||
+      (marker >= 0xd0 && marker <= 0xd7)
+    ) {
+      continue;
+    }
+    if (offset + 2 > imageBuffer.length) break;
+
+    const segmentLength = imageBuffer.readUInt16BE(offset);
+    if (segmentLength < 2 || offset + segmentLength > imageBuffer.length) break;
+    if (isJpegStartOfFrameMarker(marker)) {
+      if (segmentLength < 7) break;
+      const height = imageBuffer.readUInt16BE(offset + 3);
+      const width = imageBuffer.readUInt16BE(offset + 5);
+      assert(width && height, 'Invalid image: cannot get width or height');
+      return { width, height };
+    }
+    offset += segmentLength;
+  }
+
+  throw new Error('Invalid image: cannot get JPEG width or height');
+}
+
+export function encodedImageInfoOfBuffer(imageBuffer: Buffer): ImageInfo {
+  if (isValidPNGImageBuffer(imageBuffer)) {
+    const width = imageBuffer.readUInt32BE(16);
+    const height = imageBuffer.readUInt32BE(20);
+    assert(width && height, 'Invalid image: cannot get width or height');
+    return { width, height };
+  }
+  if (isValidJPEGImageBuffer(imageBuffer)) {
+    return jpegInfoFromBuffer(imageBuffer);
+  }
+  throw new Error('Invalid image: unsupported format');
+}
+
 /**
  * Retrieves the dimensions of an image from a base64-encoded string
  *

package/src/img/photon-loader.ts

@@ -0,0 +1,5 @@
+export type PhotonModule = typeof import('@silvia-odwyer/photon');
+
+export async function loadPhotonModule(): Promise<PhotonModule> {
+  return import('@silvia-odwyer/photon');
+}

package/src/img/transform.ts

@@ -4,11 +4,12 @@ import { readFileSync } from 'node:fs';
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon-node';
-import type { Rect } from 'src/types';
+import type { Rect, Size } from '../types';
 import { getDebug } from '../logger';
 import { ifInNode } from '../utils';
 import getPhoton from './get-photon';
 import getSharp from './get-sharp';
+import { encodedImageInfoOfBuffer } from './info';
 
 const imgDebug = getDebug('img');
 
@@ -154,10 +155,268 @@ export async function resizeAndConvertImgBuffer(
   };
 }
 
+const rawBase64BodyPattern = /^[A-Za-z0-9+/]+={0,2}$/;
+const base64ImageDataUrlPattern =
+  /^data:image\/(?:png|jpe?g);base64,[A-Za-z0-9+/=\s]+$/i;
+const supportedScreenshotDataUriPattern =
+  /^data:image\/(png|jpe?g);base64,([A-Za-z0-9+/=\s]+)$/i;
+
+function normalizeBase64Body(body: string): string {
+  return body.replace(/\s/g, '');
+}
+
+function detectImageMimeTypeFromBuffer(buffer: Buffer): string | undefined {
+  if (
+    buffer.length >= 8 &&
+    buffer[0] === 0x89 &&
+    buffer[1] === 0x50 &&
+    buffer[2] === 0x4e &&
+    buffer[3] === 0x47 &&
+    buffer[4] === 0x0d &&
+    buffer[5] === 0x0a &&
+    buffer[6] === 0x1a &&
+    buffer[7] === 0x0a
+  ) {
+    return 'image/png';
+  }
+  if (
+    buffer.length >= 3 &&
+    buffer[0] === 0xff &&
+    buffer[1] === 0xd8 &&
+    buffer[2] === 0xff
+  ) {
+    return 'image/jpeg';
+  }
+  return undefined;
+}
+
+export function inferBase64ImageFormat(base64Body: string): string {
+  const buffer = Buffer.from(normalizeBase64Body(base64Body), 'base64');
+  const mimeType = detectImageMimeTypeFromBuffer(buffer);
+  if (mimeType === 'image/jpeg') return 'jpeg';
+  if (mimeType === 'image/png') return 'png';
+  return 'png';
+}
+
 export const createImgBase64ByFormat = (format: string, body: string) => {
-  return `data:image/${format};base64,${body}`;
+  return `data:image/${format};base64,${normalizeBase64Body(body)}`;
 };
 
+export interface NormalizeScreenshotBase64Options {
+  label?: string;
+}
+
+export const normalizeScreenshotBase64 = (
+  base64: string,
+  options?: NormalizeScreenshotBase64Options,
+) => {
+  const label = options?.label ?? 'screenshot base64';
+  const trimmedBase64 = base64.trim();
+  if (!trimmedBase64) {
+    throw new Error(`${label} cannot be empty`);
+  }
+
+  const dataUriMatch = trimmedBase64.match(supportedScreenshotDataUriPattern);
+  if (dataUriMatch) {
+    const imageFormat =
+      dataUriMatch[1].toLowerCase() === 'jpg'
+        ? 'jpeg'
+        : dataUriMatch[1].toLowerCase();
+    const body = normalizeBase64Body(dataUriMatch[2]);
+    if (!body) {
+      throw new Error(`${label} cannot be empty`);
+    }
+    return createImgBase64ByFormat(imageFormat, body);
+  }
+
+  if (trimmedBase64.startsWith('data:')) {
+    throw new Error(
+      `${label} must be a PNG/JPEG data URI or raw PNG base64 string`,
+    );
+  }
+
+  if (!rawBase64BodyPattern.test(normalizeBase64Body(trimmedBase64))) {
+    throw new Error(
+      `${label} must be a PNG/JPEG data URI or raw PNG base64 string`,
+    );
+  }
+
+  return createImgBase64ByFormat('png', trimmedBase64);
+};
+
+export const normalizeBase64Image = (base64: string) => {
+  const trimmedBase64 = base64.trim();
+  if (base64ImageDataUrlPattern.test(trimmedBase64)) {
+    return trimmedBase64.replace(/\s/g, '');
+  }
+
+  const base64Body = normalizeBase64Body(trimmedBase64);
+  assert(base64Body, 'base64 image must include image data');
+  return createImgBase64ByFormat(
+    inferBase64ImageFormat(base64Body),
+    base64Body,
+  );
+};
+
+export type JpegBase64DataUrl = `data:image/jpeg;base64,${string}`;
+
+export interface ResizeBase64ImageToJpegOptions {
+  sourceSize?: Size;
+  targetSize: Size;
+  jpegQuality?: number;
+}
+
+export interface ConstrainBase64ImageToMaxSizeOptions {
+  maxSize: number;
+  jpegQuality?: number;
+}
+
+function assertValidJpegQuality(jpegQuality: number): void {
+  if (!Number.isInteger(jpegQuality) || jpegQuality < 1 || jpegQuality > 100) {
+    throw new Error(
+      `jpegQuality must be an integer between 1 and 100. Received: ${jpegQuality}`,
+    );
+  }
+}
+
+function assertValidImageSize(size: Size, label: string): void {
+  if (
+    !Number.isInteger(size.width) ||
+    !Number.isInteger(size.height) ||
+    size.width <= 0 ||
+    size.height <= 0
+  ) {
+    throw new Error(
+      `${label} width and height must be positive integers. Received width: ${size.width}, height: ${size.height}`,
+    );
+  }
+}
+
+export async function convertImgBufferToJpeg(
+  inputData: Buffer,
+  jpegQuality = 90,
+): Promise<Buffer> {
+  assertValidJpegQuality(jpegQuality);
+
+  if (ifInNode) {
+    try {
+      const Sharp = await getSharp();
+      return await Sharp(inputData).jpeg({ quality: jpegQuality }).toBuffer();
+    } catch (error) {
+      imgDebug('Sharp failed, falling back to Photon:', error);
+    }
+  }
+
+  const { PhotonImage } = await getPhoton();
+  const inputBytes = new Uint8Array(inputData);
+  const bytesliceResult = PhotonImage.new_from_byteslice(inputBytes);
+  const image =
+    bytesliceResult instanceof Promise
+      ? await bytesliceResult
+      : bytesliceResult;
+  try {
+    return Buffer.from(image.get_bytes_jpeg(jpegQuality));
+  } finally {
+    image.free();
+  }
+}
+
+export async function convertBase64ImageToJpeg(
+  inputBase64: string,
+  jpegQuality = 90,
+): Promise<JpegBase64DataUrl> {
+  assertValidJpegQuality(jpegQuality);
+  const normalized = normalizeBase64Image(inputBase64);
+  const { body } = parseBase64(normalized);
+  const imageBuffer = Buffer.from(body, 'base64');
+  const detectedMimeType = detectImageMimeTypeFromBuffer(imageBuffer);
+  if (detectedMimeType === 'image/jpeg') {
+    return createImgBase64ByFormat('jpeg', body) as JpegBase64DataUrl;
+  }
+  if (detectedMimeType !== 'image/png') {
+    throw new Error(
+      `inputBase64 must contain a PNG or JPEG image. Detected: ${detectedMimeType ?? 'unsupported format'}`,
+    );
+  }
+
+  const jpegBuffer = await convertImgBufferToJpeg(imageBuffer, jpegQuality);
+  return createImgBase64ByFormat(
+    'jpeg',
+    jpegBuffer.toString('base64'),
+  ) as JpegBase64DataUrl;
+}
+
+export async function resizeBase64ImageToJpeg(
+  inputBase64: string,
+  options: ResizeBase64ImageToJpegOptions,
+): Promise<JpegBase64DataUrl> {
+  const jpegQuality = options.jpegQuality ?? 90;
+  assertValidJpegQuality(jpegQuality);
+  assertValidImageSize(options.targetSize, 'targetSize');
+
+  const normalized = normalizeBase64Image(inputBase64);
+  const { body, mimeType } = parseBase64(normalized);
+  const imageBuffer = Buffer.from(body, 'base64');
+  const sourceSize = encodedImageInfoOfBuffer(imageBuffer);
+  if (options.sourceSize) {
+    assertValidImageSize(options.sourceSize, 'sourceSize');
+    if (
+      options.sourceSize.width !== sourceSize.width ||
+      options.sourceSize.height !== sourceSize.height
+    ) {
+      throw new Error(
+        `sourceSize ${options.sourceSize.width}x${options.sourceSize.height} does not match encoded image dimensions ${sourceSize.width}x${sourceSize.height}`,
+      );
+    }
+  }
+
+  const dimensionsUnchanged =
+    sourceSize.width === options.targetSize.width &&
+    sourceSize.height === options.targetSize.height;
+  if (dimensionsUnchanged) {
+    return convertBase64ImageToJpeg(normalized, jpegQuality);
+  }
+
+  const { buffer } = await resizeAndConvertImgBuffer(
+    mimeType.split('/')[1],
+    imageBuffer,
+    options.targetSize,
+  );
+  return createImgBase64ByFormat(
+    'jpeg',
+    buffer.toString('base64'),
+  ) as JpegBase64DataUrl;
+}
+
+export async function constrainBase64ImageToMaxSize(
+  inputBase64: string,
+  options: ConstrainBase64ImageToMaxSizeOptions,
+): Promise<string> {
+  if (!Number.isInteger(options.maxSize) || options.maxSize <= 0) {
+    throw new Error(
+      `maxSize must be a positive integer. Received: ${options.maxSize}`,
+    );
+  }
+
+  const normalized = normalizeBase64Image(inputBase64);
+  const { body } = parseBase64(normalized);
+  const sourceSize = encodedImageInfoOfBuffer(Buffer.from(body, 'base64'));
+  const largestDimension = Math.max(sourceSize.width, sourceSize.height);
+  if (largestDimension <= options.maxSize) {
+    return normalized;
+  }
+
+  const scale = options.maxSize / largestDimension;
+  return resizeBase64ImageToJpeg(normalized, {
+    targetSize: {
+      width: Math.max(1, Math.round(sourceSize.width * scale)),
+      height: Math.max(1, Math.round(sourceSize.height * scale)),
+    },
+    sourceSize,
+    jpegQuality: options.jpegQuality,
+  });
+}
+
 export async function resizeImgBase64(
   inputBase64: string,
   newSize: {
@@ -297,7 +556,7 @@ export async function paddingToMatchBlockByBase64(
 export async function cropByRect(
   imageBase64: string,
   rect: Rect,
-  paddingImage: boolean,
+  paddingImage = false,
 ): Promise<{
   width: number;
   height: number;

package/src/mcp/types.ts

@@ -111,6 +111,8 @@ export interface BaseAgent {
     msg?: string,
     options?: Record<string, unknown>,
   ) => Promise<unknown>;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  aiQuery?: (demand: any, opt?: any) => Promise<any>;
 }
 
 /**