@aiscene/shared 8.0.2 → 8.0.4

package/src/constants/example-code.ts

@@ -28,8 +28,8 @@ aiQuery<T>(queryObject: Record<string, string>): Promise<T> // Extracts data fro
 // The following is test code for Midscene AI, for reference
 // The following is Playwright syntax, you can use Playwright to assist in test generation
 import { test as base } from '@playwright/test';
-import type { PlayWrightAiFixtureType } from '@midscene/web/playwright';
-import { PlaywrightAiFixture } from '@midscene/web/playwright';
+import type { PlayWrightAiFixtureType } from '@aiscene/web/playwright';
+import { PlaywrightAiFixture } from '@aiscene/web/playwright';
 
 const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture({
   waitForNetworkIdleTimeout: 2000, // optional, the timeout for waiting for network idle between each action, default is 2000ms

package/src/constants/index.ts

@@ -17,6 +17,16 @@ export enum NodeType {
 
 export const PLAYGROUND_SERVER_PORT = 5800;
 export const SCRCPY_SERVER_PORT = 5700;
+export const SCRCPY_ADB_CONNECT_TIMEOUT_MS = 10_000;
+export const SCRCPY_PUSH_TIMEOUT_MS = 10_000;
+export const SCRCPY_START_TIMEOUT_MS = 15_000;
+export const SCRCPY_VIDEO_STREAM_TIMEOUT_MS = 15_000;
+export const SCRCPY_PREVIEW_METADATA_TIMEOUT_MS =
+  SCRCPY_ADB_CONNECT_TIMEOUT_MS +
+  SCRCPY_PUSH_TIMEOUT_MS +
+  SCRCPY_START_TIMEOUT_MS +
+  SCRCPY_VIDEO_STREAM_TIMEOUT_MS +
+  5_000;
 
 // WebDriver constants
 export const WEBDRIVER_ELEMENT_ID_KEY = 'element-6066-11e4-a52e-4f735466cecf';

package/src/env/model-config-manager.ts

@@ -68,14 +68,17 @@ export class ModelConfigManager {
     this.modelConfigMap = {
       default: {
         ...defaultConfig,
+        intent: 'default',
         createOpenAIClient: this.createOpenAIClientFn,
       },
       insight: {
         ...(insightConfig || defaultConfig),
+        intent: 'insight',
         createOpenAIClient: this.createOpenAIClientFn,
       },
       planning: {
         ...(planningConfig || defaultConfig),
+        intent: 'planning',
         createOpenAIClient: this.createOpenAIClientFn,
       },
     };

package/src/env/parse-model-config.ts

@@ -24,10 +24,26 @@ import {
 } from './types';
 
 import { getDebug } from '../logger';
+
 import { assert } from '../utils';
 import { maskConfig, parseJson } from './helper';
 import { initDebugConfig } from './init-debug';
 
+declare const __VERSION__: string | undefined;
+
+const MODEL_CONFIG_DOC_URL = 'https://midscenejs.com/model-common-config.html';
+
+const getCurrentVersion = (): string => {
+  if (typeof __VERSION__ !== 'undefined' && __VERSION__) {
+    return __VERSION__;
+  }
+
+  return 'unknown';
+};
+
+const getInvalidModelFamilyMessage = (modelFamily: TModelFamily): string =>
+  `Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}. Current version v${getCurrentVersion()} accepts the following model families: ${MODEL_FAMILY_VALUES.join(', ')}. You can also visit ${MODEL_CONFIG_DOC_URL} for the latest configuration information.`;
+
 type TModelConfigKeys =
   | typeof INSIGHT_MODEL_CONFIG_KEYS
   | typeof PLANNING_MODEL_CONFIG_KEYS
@@ -70,7 +86,7 @@ export const getUITarsModelVersion = (
  */
 export const validateModelFamily = (modelFamily?: TModelFamily): void => {
   if (modelFamily && !MODEL_FAMILY_VALUES.includes(modelFamily as any)) {
-    throw new Error(`Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}`);
+    throw new Error(getInvalidModelFamilyMessage(modelFamily));
   }
 };
 
@@ -228,6 +244,7 @@ export const parseOpenaiSdkConfig = ({
     modelName: modelName!,
     modelDescription,
     intent: '-' as any,
+    slot: '-' as any,
     timeout: provider[keys.timeout]
       ? Number(provider[keys.timeout])
       : undefined,
@@ -292,6 +309,7 @@ export const decideModelConfigFromIntentConfig = (
     useLegacyLogic: intent === 'default',
   });
   finalResult.intent = intent;
+  finalResult.slot = intent;
 
   debugLog(
     'decideModelConfig result by agent.modelConfig() with intent',

package/src/env/types.ts

@@ -10,6 +10,7 @@ export const MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG =
 export const MIDSCENE_DEBUG_MODE = 'MIDSCENE_DEBUG_MODE';
 export const MIDSCENE_MCP_USE_PUPPETEER_MODE =
   'MIDSCENE_MCP_USE_PUPPETEER_MODE';
+export const MIDSCENE_CHROME_PATH = 'MIDSCENE_CHROME_PATH';
 export const MIDSCENE_MCP_CHROME_PATH = 'MIDSCENE_MCP_CHROME_PATH';
 export const MIDSCENE_MCP_ANDROID_MODE = 'MIDSCENE_MCP_ANDROID_MODE';
 export const DOCKER_CONTAINER = 'DOCKER_CONTAINER';
@@ -69,6 +70,8 @@ export const MIDSCENE_ANDROID_IME_STRATEGY = 'MIDSCENE_ANDROID_IME_STRATEGY';
 
 export const MIDSCENE_IOS_DEVICE_UDID = 'MIDSCENE_IOS_DEVICE_UDID';
 export const MIDSCENE_IOS_SIMULATOR_UDID = 'MIDSCENE_IOS_SIMULATOR_UDID';
+export const MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE =
+  'MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE';
 
 export const MIDSCENE_CACHE = 'MIDSCENE_CACHE';
 export const MIDSCENE_USE_VLM_UI_TARS = 'MIDSCENE_USE_VLM_UI_TARS';
@@ -193,6 +196,7 @@ export const STRING_ENV_KEYS = [
   MIDSCENE_REPORT_TAG_NAME,
   MIDSCENE_PREFERRED_LANGUAGE,
   MATCH_BY_POSITION,
+  MIDSCENE_CHROME_PATH,
   MIDSCENE_MCP_CHROME_PATH,
   DOCKER_CONTAINER,
 ] as const;
@@ -425,7 +429,7 @@ export enum UITarsModelVersion {
 
 /**
  * Callback to create custom OpenAI client instance
- * @param config - Resolved model configuration including apiKey, baseURL, modelName, intent, etc.
+ * @param config - Resolved model configuration including apiKey, baseURL, modelName, intent, slot, etc.
  * @returns OpenAI client instance (can be wrapped with langsmith, langfuse, etc.)
  *
  * Note: Wrapper functions like langsmith's wrapOpenAI() return the same OpenAI instance
@@ -519,9 +523,16 @@ export interface IModelConfig {
   uiTarsModelVersion?: UITarsModelVersion;
   modelDescription: string;
   /**
-   * original intent from the config
+   * The semantic intent this config is requested for.
+   * For example, getModelConfig('planning') always returns intent === 'planning'.
    */
   intent: TIntent;
+  /**
+   * The model-config slot this config was resolved from.
+   * For example, getModelConfig('planning') may resolve from slot === 'default'
+   * when MIDSCENE_PLANNING_MODEL_NAME is not configured.
+   */
+  slot: TIntent;
   /**
    * Custom OpenAI client factory function
    *

package/src/env/utils.ts

@@ -13,56 +13,6 @@ export const globalConfigManager = new GlobalConfigManager();
 globalConfigManager.registerModelConfigManager(globalModelConfigManager);
 globalModelConfigManager.registerGlobalConfigManager(globalConfigManager);
 
-/**
- * Interface for devices that support getTimestamp method.
- * This is a minimal interface to avoid circular dependencies with @midscene/core.
- */
-export interface DeviceWithTimestamp {
-  getTimestamp?: () => Promise<number>;
-}
-
-/**
- * Get the current timestamp, optionally from the target device.
- *
- * When useDeviceTimestamp is enabled and a device with getTimestamp is provided,
- * this function will return the device's time. Otherwise, it returns the system time.
- *
- * This is useful when:
- * - Testing on devices with different time zones
- * - Debugging time-sensitive features
- * - The system clock and device clock are not synchronized
- *
- * @param device Optional device interface that supports getTimestamp
- * @param useDeviceTimestamp Whether to use device timestamp (from agent config)
- * @returns Timestamp in milliseconds
- *
- * @example
- * // Without device - always returns system time
- * const systemTime = await getCurrentTime();
- *
- * @example
- * // With device and config enabled - returns device time
- * const deviceTime = await getCurrentTime(androidDevice, true);
- */
-export async function getCurrentTime(
-  device?: DeviceWithTimestamp,
-  useDeviceTimestamp?: boolean,
-): Promise<number> {
-  if (useDeviceTimestamp && device?.getTimestamp) {
-    try {
-      return await device.getTimestamp();
-    } catch (error) {
-      // Fall back to system time if device time retrieval fails
-      console.warn(
-        `Failed to get device time, falling back to system time: ${error}`,
-      );
-      return Date.now();
-    }
-  }
-
-  return Date.now();
-}
-
 export const getPreferredLanguage = () => {
   const prefer = globalConfigManager.getEnvConfigValue(
     MIDSCENE_PREFERRED_LANGUAGE,

package/src/extractor/index.ts

@@ -43,6 +43,6 @@ export {
   getElementXpath,
 } from './locator';
 
-export { generateElementByRect } from './dom-util';
+export { generateElementByPoint, generateElementByRect } from './dom-util';
 
 export { isNotContainerElement } from './dom-util';

package/src/img/index.ts

@@ -1,11 +1,18 @@
 export {
+  encodedImageInfoOfBuffer,
   imageInfoOfBase64,
   isValidPNGImageBuffer,
   isValidJPEGImageBuffer,
   isValidImageBuffer,
+  validateScreenshotBuffer,
+  type ValidateScreenshotBufferOptions,
 } from './info';
 export {
   resizeAndConvertImgBuffer,
+  convertImgBufferToJpeg,
+  convertBase64ImageToJpeg,
+  constrainBase64ImageToMaxSize,
+  resizeBase64ImageToJpeg,
   resizeImgBase64,
   zoomForGPT4o,
   saveBase64Image,
@@ -17,6 +24,13 @@ export {
   preProcessImageUrl,
   parseBase64,
   createImgBase64ByFormat,
+  inferBase64ImageFormat,
+  normalizeBase64Image,
+  normalizeScreenshotBase64,
+  type NormalizeScreenshotBase64Options,
+  type ConstrainBase64ImageToMaxSizeOptions,
+  type JpegBase64DataUrl,
+  type ResizeBase64ImageToJpegOptions,
 } from './transform';
 export {
   processImageElementInfo,

package/src/img/info.ts

@@ -1,9 +1,70 @@
 import assert from 'node:assert';
+import { Buffer } from 'node:buffer';
 import type { Size } from '../types';
 import getPhoton from './get-photon';
 
 export interface ImageInfo extends Size {}
 
+const isJpegStartOfFrameMarker = (marker: number): boolean =>
+  marker >= 0xc0 &&
+  marker <= 0xcf &&
+  marker !== 0xc4 &&
+  marker !== 0xc8 &&
+  marker !== 0xcc;
+
+function jpegInfoFromBuffer(imageBuffer: Buffer): ImageInfo {
+  let offset = 2;
+  while (offset < imageBuffer.length) {
+    if (imageBuffer[offset] !== 0xff) {
+      offset += 1;
+      continue;
+    }
+
+    while (offset < imageBuffer.length && imageBuffer[offset] === 0xff) {
+      offset += 1;
+    }
+    if (offset >= imageBuffer.length) break;
+
+    const marker = imageBuffer[offset];
+    offset += 1;
+    if (marker === 0xd9 || marker === 0xda) break;
+    if (
+      marker === 0x01 ||
+      marker === 0xd8 ||
+      (marker >= 0xd0 && marker <= 0xd7)
+    ) {
+      continue;
+    }
+    if (offset + 2 > imageBuffer.length) break;
+
+    const segmentLength = imageBuffer.readUInt16BE(offset);
+    if (segmentLength < 2 || offset + segmentLength > imageBuffer.length) break;
+    if (isJpegStartOfFrameMarker(marker)) {
+      if (segmentLength < 7) break;
+      const height = imageBuffer.readUInt16BE(offset + 3);
+      const width = imageBuffer.readUInt16BE(offset + 5);
+      assert(width && height, 'Invalid image: cannot get width or height');
+      return { width, height };
+    }
+    offset += segmentLength;
+  }
+
+  throw new Error('Invalid image: cannot get JPEG width or height');
+}
+
+export function encodedImageInfoOfBuffer(imageBuffer: Buffer): ImageInfo {
+  if (isValidPNGImageBuffer(imageBuffer)) {
+    const width = imageBuffer.readUInt32BE(16);
+    const height = imageBuffer.readUInt32BE(20);
+    assert(width && height, 'Invalid image: cannot get width or height');
+    return { width, height };
+  }
+  if (isValidJPEGImageBuffer(imageBuffer)) {
+    return jpegInfoFromBuffer(imageBuffer);
+  }
+  throw new Error('Invalid image: unsupported format');
+}
+
 /**
  * Retrieves the dimensions of an image from a base64-encoded string
  *
@@ -14,52 +75,16 @@ export interface ImageInfo extends Size {}
 export async function imageInfoOfBase64(
   imageBase64: string,
 ): Promise<ImageInfo> {
-  // Validate input before passing to Photon
-  if (!imageBase64 || typeof imageBase64 !== 'string') {
-    throw new Error('Invalid image base64: data is empty or not a string');
-  }
-
+  const { PhotonImage } = await getPhoton();
   const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '');
-
-  // Check if base64 data is valid after stripping data URL prefix
-  if (!base64Data || base64Data.length === 0) {
-    throw new Error('Invalid image base64: no data after stripping prefix');
-  }
-
-  // Basic validation: base64 should only contain valid characters
-  // (A-Z, a-z, 0-9, +, /, and = for padding)
-  if (!/^[A-Za-z0-9+/]*={0,2}$/.test(base64Data)) {
-    throw new Error(
-      'Invalid image base64: contains invalid characters',
-    );
-  }
-
-  // Check minimum length for a valid image (at least a few bytes)
-  if (base64Data.length < 4) {
-    throw new Error(
-      `Invalid image base64: data too short (${base64Data.length} characters)`,
-    );
-  }
-
-  try {
-    const { PhotonImage } = await getPhoton();
-    // Support both sync (Photon) and async (Canvas fallback) versions
-    const result = PhotonImage.new_from_base64(base64Data);
-    const image = result instanceof Promise ? await result : result;
-    const width = image.get_width();
-    const height = image.get_height();
-    image.free();
-    assert(width && height, 'Invalid image: cannot get width or height');
-    return { width, height };
-  } catch (error) {
-    // Provide more helpful error message for Photon failures
-    if (error instanceof Error && error.message.includes('unreachable')) {
-      throw new Error(
-        `Failed to decode image: invalid or corrupted image data. Original error: ${error.message}`,
-      );
-    }
-    throw error;
-  }
+  // Support both sync (Photon) and async (Canvas fallback) versions
+  const result = PhotonImage.new_from_base64(base64Data);
+  const image = result instanceof Promise ? await result : result;
+  const width = image.get_width();
+  const height = image.get_height();
+  image.free();
+  assert(width && height, 'Invalid image: cannot get width or height');
+  return { width, height };
 }
 
 /**
@@ -109,3 +134,30 @@ export function isValidJPEGImageBuffer(buffer: Buffer): boolean {
 export function isValidImageBuffer(buffer: Buffer): boolean {
   return isValidPNGImageBuffer(buffer) || isValidJPEGImageBuffer(buffer);
 }
+
+export interface ValidateScreenshotBufferOptions {
+  label: string;
+  minBufferSize?: number;
+}
+
+export function validateScreenshotBuffer(
+  screenshotBuffer: Buffer | undefined,
+  { label, minBufferSize = 0 }: ValidateScreenshotBufferOptions,
+): asserts screenshotBuffer is Buffer {
+  const bufferSize = screenshotBuffer?.length ?? 0;
+  if (!screenshotBuffer || bufferSize === 0) {
+    throw new Error(
+      `${label} validation failed: buffer size ${bufferSize} bytes`,
+    );
+  }
+
+  if (!isValidImageBuffer(screenshotBuffer)) {
+    throw new Error(`${label} buffer has invalid image format`);
+  }
+
+  if (minBufferSize > 0 && bufferSize < minBufferSize) {
+    throw new Error(
+      `${label} validation failed: buffer size ${bufferSize} bytes (minimum: ${minBufferSize})`,
+    );
+  }
+}

package/src/img/photon-loader.ts

@@ -0,0 +1,5 @@
+export type PhotonModule = typeof import('@silvia-odwyer/photon');
+
+export async function loadPhotonModule(): Promise<PhotonModule> {
+  return import('@silvia-odwyer/photon');
+}