@aiscene/core 1.6.2

package/dist/es/agent/utils.mjs

@@ -0,0 +1,198 @@
+import { ScreenshotItem } from "../screenshot-item.mjs";
+import { uploadTestInfoToServer } from "../utils.mjs";
+import { MIDSCENE_REPORT_QUIET, MIDSCENE_REPORT_TAG_NAME, globalConfigManager } from "@midscene/shared/env";
+import { generateElementByRect } from "@midscene/shared/extractor";
+import { imageInfoOfBase64, resizeImgBase64 } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+import { assert, logMsg, uuid } from "@midscene/shared/utils";
+import dayjs from "dayjs";
+import { debug as external_task_cache_mjs_debug } from "./task-cache.mjs";
+async function commonContextParser(interfaceInstance, _opt) {
+    const debug = getDebug('commonContextParser');
+    assert(interfaceInstance, 'interfaceInstance is required');
+    debug("Getting interface description");
+    const description = interfaceInstance.describe?.() || '';
+    debug("Interface description end");
+    debug('Uploading test info to server');
+    uploadTestInfoToServer({
+        testUrl: description,
+        serverUrl: _opt.uploadServerUrl
+    });
+    debug('UploadTestInfoToServer end');
+    debug('will get size');
+    const interfaceSize = await interfaceInstance.size();
+    const { width: logicalWidth, height: logicalHeight } = interfaceSize;
+    if (interfaceSize.dpr) console.warn('Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ');
+    if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) throw new Error(`Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`);
+    if (logicalWidth <= 0 || logicalHeight <= 0) throw new Error(`Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`);
+    debug(`size: ${logicalWidth}x${logicalHeight}`);
+    const screenshotBase64 = await interfaceInstance.screenshotBase64();
+    const screenshotCapturedAt = Date.now();
+    assert(screenshotBase64, 'screenshotBase64 is required');
+    debug('will get screenshot dimensions');
+    const { width: imgWidth, height: imgHeight } = await imageInfoOfBase64(screenshotBase64);
+    if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) throw new Error(`Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
+    if (imgWidth <= 0 || imgHeight <= 0) throw new Error(`Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
+    debug('screenshot dimensions', imgWidth, 'x', imgHeight);
+    const logicalIsPortrait = logicalWidth < logicalHeight;
+    const screenshotIsPortrait = imgWidth < imgHeight;
+    let finalLogicalWidth = logicalWidth;
+    if (logicalIsPortrait !== screenshotIsPortrait) {
+        debug(`Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`);
+        finalLogicalWidth = logicalHeight;
+    }
+    const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;
+    if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) throw new Error(`Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`);
+    const dpr = imgWidth / finalLogicalWidth;
+    debug('calculated dpr:', dpr);
+    const shrunkShotToLogicalRatio = dpr / userShrinkFactor;
+    debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);
+    if (1 !== userShrinkFactor) {
+        const targetWidth = Math.round(imgWidth / userShrinkFactor);
+        const targetHeight = Math.round(imgHeight / userShrinkFactor);
+        debug(`Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`);
+        const resizedBase64 = await resizeImgBase64(screenshotBase64, {
+            width: targetWidth,
+            height: targetHeight
+        });
+        return {
+            shotSize: {
+                width: targetWidth,
+                height: targetHeight
+            },
+            deprecatedDpr: dpr,
+            screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),
+            shrunkShotToLogicalRatio
+        };
+    }
+    return {
+        shotSize: {
+            width: imgWidth,
+            height: imgHeight
+        },
+        deprecatedDpr: dpr,
+        screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),
+        shrunkShotToLogicalRatio
+    };
+}
+function getReportFileName(tag = 'web') {
+    const reportTagName = globalConfigManager.getEnvConfigValue(MIDSCENE_REPORT_TAG_NAME);
+    const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');
+    const uniqueId = uuid().substring(0, 8);
+    return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;
+}
+function printReportMsg(filepath) {
+    if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) return;
+    logMsg(`Midscene - report file updated: ${filepath}`);
+}
+function getCurrentExecutionFile(trace) {
+    const error = new Error();
+    const stackTrace = trace || error.stack;
+    const pkgDir = process.cwd() || '';
+    if (stackTrace) {
+        const stackLines = stackTrace.split('\n');
+        for (const line of stackLines)if (line.includes('.spec.') || line.includes('.test.') || line.includes('.ts') || line.includes('.js')) {
+            const match = line.match(/(?:at\s+)?(.*?\.(?:spec|test)\.[jt]s)/);
+            if (match?.[1]) {
+                const targetFileName = match[1].replace(pkgDir, '').trim().replace('at ', '');
+                return targetFileName;
+            }
+        }
+    }
+    return false;
+}
+const testFileIndex = new Map();
+function generateCacheId(fileName) {
+    let taskFile = fileName || getCurrentExecutionFile();
+    if (!taskFile) {
+        taskFile = uuid();
+        console.warn('Midscene - using random UUID for cache id. Cache may be invalid.');
+    }
+    if (testFileIndex.has(taskFile)) {
+        const currentIndex = testFileIndex.get(taskFile);
+        if (void 0 !== currentIndex) testFileIndex.set(taskFile, currentIndex + 1);
+    } else testFileIndex.set(taskFile, 1);
+    return `${taskFile}-${testFileIndex.get(taskFile)}`;
+}
+function ifPlanLocateParamIsBbox(planLocateParam) {
+    return !!(planLocateParam.bbox && Array.isArray(planLocateParam.bbox) && 4 === planLocateParam.bbox.length);
+}
+function matchElementFromPlan(planLocateParam) {
+    if (!planLocateParam) return;
+    if (planLocateParam.bbox) {
+        const rect = {
+            left: planLocateParam.bbox[0],
+            top: planLocateParam.bbox[1],
+            width: planLocateParam.bbox[2] - planLocateParam.bbox[0] + 1,
+            height: planLocateParam.bbox[3] - planLocateParam.bbox[1] + 1
+        };
+        const element = generateElementByRect(rect, 'string' == typeof planLocateParam.prompt ? planLocateParam.prompt : planLocateParam.prompt?.prompt || '');
+        return element;
+    }
+}
+async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable) {
+    if (!cacheEntry) return;
+    if (false === cacheable) return void external_task_cache_mjs_debug('cache disabled for prompt: %s', cachePrompt);
+    if (!context.taskCache?.isCacheResultUsed) return;
+    if (!context.interfaceInstance.rectMatchesCacheFeature) return void external_task_cache_mjs_debug('interface does not implement rectMatchesCacheFeature, skip cache');
+    try {
+        const rect = await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);
+        const element = {
+            center: [
+                Math.round(rect.left + rect.width / 2),
+                Math.round(rect.top + rect.height / 2)
+            ],
+            rect,
+            description: 'string' == typeof cachePrompt ? cachePrompt : cachePrompt.prompt || ''
+        };
+        external_task_cache_mjs_debug('cache hit, prompt: %s', cachePrompt);
+        return element;
+    } catch (error) {
+        external_task_cache_mjs_debug('rectMatchesCacheFeature error: %s', error);
+        return;
+    }
+}
+const getMidsceneVersion = ()=>"1.6.2";
+const parsePrompt = (prompt)=>{
+    if ('string' == typeof prompt) return {
+        textPrompt: prompt,
+        multimodalPrompt: void 0
+    };
+    return {
+        textPrompt: prompt.prompt,
+        multimodalPrompt: prompt.images ? {
+            images: prompt.images,
+            convertHttpImage2Base64: !!prompt.convertHttpImage2Base64
+        } : void 0
+    };
+};
+const transformLogicalElementToScreenshot = (element, shrunkShotToLogicalRatio)=>{
+    if (1 === shrunkShotToLogicalRatio) return element;
+    return {
+        ...element,
+        center: [
+            Math.round(element.center[0] * shrunkShotToLogicalRatio),
+            Math.round(element.center[1] * shrunkShotToLogicalRatio)
+        ],
+        rect: {
+            ...element.rect,
+            left: Math.round(element.rect.left * shrunkShotToLogicalRatio),
+            top: Math.round(element.rect.top * shrunkShotToLogicalRatio),
+            width: Math.round(element.rect.width * shrunkShotToLogicalRatio),
+            height: Math.round(element.rect.height * shrunkShotToLogicalRatio)
+        }
+    };
+};
+const transformLogicalRectToScreenshotRect = (rect, shrunkShotToLogicalRatio)=>{
+    if (1 === shrunkShotToLogicalRatio) return rect;
+    return {
+        ...rect,
+        left: Math.round(rect.left * shrunkShotToLogicalRatio),
+        top: Math.round(rect.top * shrunkShotToLogicalRatio),
+        width: Math.round(rect.width * shrunkShotToLogicalRatio),
+        height: Math.round(rect.height * shrunkShotToLogicalRatio)
+    };
+};
+export { commonContextParser, generateCacheId, getCurrentExecutionFile, getMidsceneVersion, getReportFileName, ifPlanLocateParamIsBbox, matchElementFromCache, matchElementFromPlan, parsePrompt, printReportMsg, transformLogicalElementToScreenshot, transformLogicalRectToScreenshotRect };
+
+//# sourceMappingURL=utils.mjs.map

package/dist/es/agent/utils.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { imageInfoOfBase64, resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n    modelFamily?: TModelFamily;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    // Convert bbox [x1, y1, x2, y2] to rect {left, top, width, height}\n    const rect = {\n      left: planLocateParam.bbox[0],\n      top: planLocateParam.bbox[1],\n      width: planLocateParam.bbox[2] - planLocateParam.bbox[0] + 1,\n      height: planLocateParam.bbox[3] - planLocateParam.bbox[1] + 1,\n    };\n\n    const element = generateElementByRect(\n      rect,\n      typeof planLocateParam.prompt === 'string'\n        ? planLocateParam.prompt\n        : planLocateParam.prompt?.prompt || '',\n    );\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["commonContextParser","interfaceInstance","_opt","debug","getDebug","assert","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotBase64","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","getCurrentExecutionFile","trace","error","stackTrace","pkgDir","process","stackLines","line","match","targetFileName","testFileIndex","Map","generateCacheId","fileName","taskFile","currentIndex","undefined","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","rect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","getMidsceneVersion","__VERSION__","parsePrompt","prompt","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;;;;;;;AA0BO,eAAeA,oBACpBC,iBAAoC,EACpCC,IAIC;IAED,MAAMC,QAAQC,SAAS;IAEvBC,OAAOJ,mBAAmB;IAE1BE,MAAM;IACN,MAAMG,cAAcL,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNI,uBAAuB;QACrB,SAASD;QACT,WAAWJ,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMK,gBAAgB,MAAMP,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOQ,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIP,MAAM,CAAC,MAAM,EAAEM,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMI,mBAAmB,MAAMb,kBAAkB,gBAAgB;IACjE,MAAMc,uBAAuBC,KAAK,GAAG;IACrCX,OAAOS,kBAAmB;IAG1BX,MAAM;IACN,MAAM,EAAE,OAAOc,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,kBAAkBL;IAE1B,IAAI,CAACF,OAAO,QAAQ,CAACK,aAAa,CAACL,OAAO,QAAQ,CAACM,YACjD,MAAM,IAAIL,MACR,CAAC,wFAAwF,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIL,MACR,CAAC,0FAA0F,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAGjIf,MAAM,yBAAyBc,UAAU,KAAKC;IAM9C,MAAME,oBAAoBX,eAAeC;IACzC,MAAMW,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBb;IAExB,IAAIW,sBAAsBC,sBAAsB;QAC9ClB,MACE,CAAC,4CAA4C,EAAEM,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEU,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBZ;IAEtB;IAEA,MAAMa,mBAAmBrB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACU,OAAO,QAAQ,CAACW,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIV,MACR,CAAC,wEAAwE,EAAEU,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBnB,MAAM,mBAAmBqB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCpB,MAAM,4BAA4BsB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CpB,MACE,CAAC,mCAAmC,EAAEoB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,gBAAgBhB,kBAAkB;YAC5D,OAAOY;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,eAAe,MAAM,CAACF,eAAed;YACjDU;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOR;YACP,QAAQC;QACV;QACA,eAAeM;QACf,YAAYO,eAAe,MAAM,CAACjB,kBAAkBC;QACpDU;IACF;AACF;AAEO,SAASO,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAMO,SAASG,wBAAwBC,KAAc;IACpD,MAAMC,QAAQ,IAAIlC;IAClB,MAAMmC,aAAaF,SAASC,MAAM,KAAK;IACvC,MAAME,SAASC,QAAQ,GAAG,MAAM;IAChC,IAAIF,YAAY;QACd,MAAMG,aAAaH,WAAW,KAAK,CAAC;QACpC,KAAK,MAAMI,QAAQD,WACjB,IACEC,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,UACdA,KAAK,QAAQ,CAAC,QACd;YACA,MAAMC,QAAQD,KAAK,KAAK,CAAC;YACzB,IAAIC,OAAO,CAAC,EAAE,EAAE;gBACd,MAAMC,iBAAiBD,KAAK,CAAC,EAAE,CAC5B,OAAO,CAACJ,QAAQ,IAChB,IAAI,GACJ,OAAO,CAAC,OAAO;gBAClB,OAAOK;YACT;QACF;IAEJ;IACA,OAAO;AACT;AAEA,MAAMC,gBAAgB,IAAIC;AAEnB,SAASC,gBAAgBC,QAAiB;IAC/C,IAAIC,WAAWD,YAAYb;IAC3B,IAAI,CAACc,UAAU;QACbA,WAAWnB;QACX7B,QAAQ,IAAI,CACV;IAEJ;IAEA,IAAI4C,cAAc,GAAG,CAACI,WAAW;QAC/B,MAAMC,eAAeL,cAAc,GAAG,CAACI;QACvC,IAAIC,AAAiBC,WAAjBD,cACFL,cAAc,GAAG,CAACI,UAAUC,eAAe;IAE/C,OACEL,cAAc,GAAG,CAACI,UAAU;IAE9B,OAAO,GAAGA,SAAS,CAAC,EAAEJ,cAAc,GAAG,CAACI,WAAW;AACrD;AAEO,SAASG,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;QAExB,MAAMG,OAAO;YACX,MAAMH,gBAAgB,IAAI,CAAC,EAAE;YAC7B,KAAKA,gBAAgB,IAAI,CAAC,EAAE;YAC5B,OAAOA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;YAC3D,QAAQA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;QAC9D;QAEA,MAAMI,UAAUC,sBACdF,MACA,AAAkC,YAAlC,OAAOH,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QAExC,OAAOI;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMR,OACJ,MAAMI,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNxC,KAAK,KAAK,CAACuC,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCvC,KAAK,KAAK,CAACuC,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOM,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOpB,OAAO;QACd2B,8BAAW,qCAAqC3B;QAChD;IACF;AACF;AAIO,MAAM4B,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBjB;IACpB;IAEF,OAAO;QACL,YAAYiB,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAjB;IACN;AACF;AAEO,MAAMkB,sCAAsC,CACjDZ,SACA1C;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO0C;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNxC,KAAK,KAAK,CAACwC,QAAQ,MAAM,CAAC,EAAE,GAAG1C;YAC/BE,KAAK,KAAK,CAACwC,QAAQ,MAAM,CAAC,EAAE,GAAG1C;SAChC;QACD,MAAM;YACJ,GAAG0C,QAAQ,IAAI;YACf,MAAMxC,KAAK,KAAK,CAACwC,QAAQ,IAAI,CAAC,IAAI,GAAG1C;YACrC,KAAKE,KAAK,KAAK,CAACwC,QAAQ,IAAI,CAAC,GAAG,GAAG1C;YACnC,OAAOE,KAAK,KAAK,CAACwC,QAAQ,IAAI,CAAC,KAAK,GAAG1C;YACvC,QAAQE,KAAK,KAAK,CAACwC,QAAQ,IAAI,CAAC,MAAM,GAAG1C;QAC3C;IACF;AACF;AAEO,MAAMuD,uCAAuC,CAClDd,MACAzC;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOyC;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMvC,KAAK,KAAK,CAACuC,KAAK,IAAI,GAAGzC;QAC7B,KAAKE,KAAK,KAAK,CAACuC,KAAK,GAAG,GAAGzC;QAC3B,OAAOE,KAAK,KAAK,CAACuC,KAAK,KAAK,GAAGzC;QAC/B,QAAQE,KAAK,KAAK,CAACuC,KAAK,MAAM,GAAGzC;IACnC;AACF"}

package/dist/es/ai-model/auto-glm/actions.mjs

@@ -0,0 +1,237 @@
+import { adaptBbox, pointToBbox } from "../../common.mjs";
+import { getDebug } from "@midscene/shared/logger";
+const debug = getDebug('auto-glm-actions');
+const AUTO_GLM_COORDINATE_MAX = 1000;
+function autoGLMCoordinateToBbox(x, y, width, height) {
+    const bbox = pointToBbox(x, y, 10);
+    return adaptBbox(bbox, width, height, 'auto-glm');
+}
+const BACK_BUTTON_NAMES = [
+    'AndroidBackButton',
+    'HarmonyBackButton'
+];
+const HOME_BUTTON_NAMES = [
+    'AndroidHomeButton',
+    'HarmonyHomeButton'
+];
+function findActionName(actionSpace, knownNames, defaultName) {
+    if (!actionSpace) return defaultName;
+    const match = actionSpace.find((a)=>knownNames.includes(a.name));
+    return match ? match.name : defaultName;
+}
+function transformAutoGLMAction(action, size, actionSpace) {
+    try {
+        switch(action._metadata){
+            case 'finish':
+                {
+                    const finishAction = action;
+                    debug('Transform finish action:', finishAction);
+                    return [
+                        {
+                            type: 'Finished',
+                            param: {},
+                            thought: finishAction.message
+                        }
+                    ];
+                }
+            case 'do':
+                {
+                    const doAction = action;
+                    switch(doAction.action){
+                        case 'Tap':
+                            {
+                                const tapAction = doAction;
+                                debug('Transform Tap action:', tapAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(tapAction.element[0], tapAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                return [
+                                    {
+                                        type: 'Tap',
+                                        param: {
+                                            locate
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Double Tap':
+                            {
+                                const doubleTapAction = doAction;
+                                debug('Transform Double Tap action:', doubleTapAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(doubleTapAction.element[0], doubleTapAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                return [
+                                    {
+                                        type: 'DoubleClick',
+                                        param: {
+                                            locate
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Type':
+                            {
+                                const typeAction = doAction;
+                                debug('Transform Type action:', typeAction);
+                                return [
+                                    {
+                                        type: 'Input',
+                                        param: {
+                                            value: typeAction.text
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Swipe':
+                            {
+                                const swipeAction = doAction;
+                                debug('Transform Swipe action:', swipeAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(swipeAction.start[0], swipeAction.start[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                const deltaX = swipeAction.end[0] - swipeAction.start[0];
+                                const deltaY = swipeAction.end[1] - swipeAction.start[1];
+                                let direction;
+                                let distance;
+                                const absDeltaX = Math.abs(deltaX);
+                                const absDeltaY = Math.abs(deltaY);
+                                if (absDeltaY > absDeltaX) {
+                                    distance = Math.round(absDeltaY * size.height / AUTO_GLM_COORDINATE_MAX);
+                                    direction = deltaY > 0 ? 'up' : 'down';
+                                } else {
+                                    distance = Math.round(absDeltaX * size.width / AUTO_GLM_COORDINATE_MAX);
+                                    direction = deltaX > 0 ? 'left' : 'right';
+                                }
+                                debug(`Calculate swipe direction: ${direction}, distance: ${distance}`);
+                                return [
+                                    {
+                                        type: 'Scroll',
+                                        param: {
+                                            locate,
+                                            distance,
+                                            direction
+                                        },
+                                        thought: swipeAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Long Press':
+                            {
+                                const longPressAction = doAction;
+                                debug('Transform Long Press action:', longPressAction);
+                                const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(longPressAction.element[0], longPressAction.element[1], size.width, size.height);
+                                const locate = {
+                                    prompt: '',
+                                    bbox: [
+                                        x1,
+                                        y1,
+                                        x2,
+                                        y2
+                                    ]
+                                };
+                                return [
+                                    {
+                                        type: 'LongPress',
+                                        param: {
+                                            locate
+                                        },
+                                        thought: longPressAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Back':
+                            {
+                                const backAction = doAction;
+                                debug('Transform Back action:', backAction);
+                                return [
+                                    {
+                                        type: findActionName(actionSpace, BACK_BUTTON_NAMES, 'AndroidBackButton'),
+                                        param: {},
+                                        thought: backAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Home':
+                            {
+                                const homeAction = doAction;
+                                debug('Transform Home action:', homeAction);
+                                return [
+                                    {
+                                        type: findActionName(actionSpace, HOME_BUTTON_NAMES, 'AndroidHomeButton'),
+                                        param: {},
+                                        thought: homeAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Wait':
+                            {
+                                const waitAction = doAction;
+                                debug('Transform Wait action:', waitAction);
+                                return [
+                                    {
+                                        type: 'Sleep',
+                                        param: {
+                                            timeMs: waitAction.durationMs
+                                        },
+                                        thought: waitAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Launch':
+                            {
+                                const launchAction = doAction;
+                                debug('Transform Launch action:', launchAction);
+                                return [
+                                    {
+                                        type: 'Launch',
+                                        param: launchAction.app,
+                                        thought: launchAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Interact':
+                            throw new Error('Action "Interact" from auto-glm is not supported in the current implementation.');
+                        case 'Call_API':
+                            throw new Error('Action "Call_API" from auto-glm is not supported in the current implementation.');
+                        case 'Take_over':
+                            throw new Error('Action "Take_over" from auto-glm is not supported in the current implementation.');
+                        case 'Note':
+                            throw new Error('Action "Note" from auto-glm is not supported in the current implementation.');
+                        default:
+                            throw new Error(`Unknown do() action type: ${doAction.action}`);
+                    }
+                }
+            default:
+                throw new Error(`Unknown action metadata: ${action._metadata}`);
+        }
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        debug('Transform error:', errorMessage);
+        throw new Error(`Failed to transform action: ${errorMessage}`);
+    }
+}
+export { transformAutoGLMAction };
+
+//# sourceMappingURL=actions.mjs.map

package/dist/es/ai-model/auto-glm/actions.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/auto-glm/actions.mjs","sources":["../../../../src/ai-model/auto-glm/actions.ts"],"sourcesContent":["import { adaptBbox, pointToBbox } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\n\nconst debug = getDebug('auto-glm-actions');\n\n/**\n * Auto-GLM coordinate system range: [0, AUTO_GLM_COORDINATE_MAX]\n */\nconst AUTO_GLM_COORDINATE_MAX = 1000;\n\n/**\n * Convert auto-glm coordinate [0,1000] to bbox in pixel coordinates\n */\nfunction autoGLMCoordinateToBbox(\n  x: number,\n  y: number,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const bbox = pointToBbox(x, y, 10);\n  return adaptBbox(bbox, width, height, 'auto-glm');\n}\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type ParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nconst BACK_BUTTON_NAMES = ['AndroidBackButton', 'HarmonyBackButton'];\nconst HOME_BUTTON_NAMES = ['AndroidHomeButton', 'HarmonyHomeButton'];\n\n/**\n * Find the action name in actionSpace that matches one of the known names.\n * Falls back to defaultName if no match found or actionSpace is not provided.\n */\nfunction findActionName(\n  actionSpace: DeviceAction[] | undefined,\n  knownNames: string[],\n  defaultName: string,\n): string {\n  if (!actionSpace) return defaultName;\n  const match = actionSpace.find((a) => knownNames.includes(a.name));\n  return match ? match.name : defaultName;\n}\n\nexport function transformAutoGLMAction(\n  action: ParsedAction,\n  size: { width: number; height: number },\n  actionSpace?: DeviceAction[],\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              tapAction.element[0],\n              tapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              doubleTapAction.element[0],\n              doubleTapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            // Calculate locate using start coordinate\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              swipeAction.start[0],\n              swipeAction.start[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            // Calculate horizontal and vertical delta in [0,AUTO_GLM_COORDINATE_MAX] coordinate system\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = Math.round(\n                (absDeltaY * size.height) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = Math.round(\n                (absDeltaX * size.width) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate,\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              },\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              longPressAction.element[0],\n              longPressAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'LongPress',\n                param: {\n                  locate,\n                },\n                thought: longPressAction.think || '',\n              },\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  BACK_BUTTON_NAMES,\n                  'AndroidBackButton',\n                ),\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  HOME_BUTTON_NAMES,\n                  'AndroidHomeButton',\n                ),\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: launchAction.app,\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["debug","getDebug","AUTO_GLM_COORDINATE_MAX","autoGLMCoordinateToBbox","x","y","width","height","bbox","pointToBbox","adaptBbox","BACK_BUTTON_NAMES","HOME_BUTTON_NAMES","findActionName","actionSpace","knownNames","defaultName","match","a","transformAutoGLMAction","action","size","finishAction","doAction","tapAction","x1","y1","x2","y2","locate","doubleTapAction","typeAction","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";;AAKA,MAAMA,QAAQC,SAAS;AAKvB,MAAMC,0BAA0B;AAKhC,SAASC,wBACPC,CAAS,EACTC,CAAS,EACTC,KAAa,EACbC,MAAc;IAEd,MAAMC,OAAOC,YAAYL,GAAGC,GAAG;IAC/B,OAAOK,UAAUF,MAAMF,OAAOC,QAAQ;AACxC;AAwGA,MAAMI,oBAAoB;IAAC;IAAqB;CAAoB;AACpE,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AAMpE,SAASC,eACPC,WAAuC,EACvCC,UAAoB,EACpBC,WAAmB;IAEnB,IAAI,CAACF,aAAa,OAAOE;IACzB,MAAMC,QAAQH,YAAY,IAAI,CAAC,CAACI,IAAMH,WAAW,QAAQ,CAACG,EAAE,IAAI;IAChE,OAAOD,QAAQA,MAAM,IAAI,GAAGD;AAC9B;AAEO,SAASG,uBACdC,MAAoB,EACpBC,IAAuC,EACvCP,WAA4B;IAE5B,IAAI;QACF,OAAQM,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAME,eAAeF;oBACrBpB,MAAM,4BAA4BsB;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWH;oBAejB,OAASG,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBvB,MAAM,yBAAyBwB;gCAC/B,MAAM,CAACC,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvBqB,UAAU,OAAO,CAAC,EAAE,EACpBA,UAAU,OAAO,CAAC,EAAE,EACpBH,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBP;gCACxBvB,MAAM,gCAAgC8B;gCACtC,MAAM,CAACL,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvB2B,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BT,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAME,aAAaR;gCACnBvB,MAAM,0BAA0B+B;gCAEhC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOA,WAAW,IAAI;wCACxB;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAMC,cAAcT;gCACpBvB,MAAM,2BAA2BgC;gCAGjC,MAAM,CAACP,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvB6B,YAAY,KAAK,CAAC,EAAE,EACpBA,YAAY,KAAK,CAAC,EAAE,EACpBX,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAGA,MAAMK,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWE,KAAK,KAAK,CAClBC,YAAYlB,KAAK,MAAM,GAAInB;oCAE9BiC,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWE,KAAK,KAAK,CAClBD,YAAYhB,KAAK,KAAK,GAAInB;oCAE7BiC,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEAjC,MACE,CAAC,2BAA2B,EAAEmC,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLP;4CAEAO;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMQ,kBAAkBjB;gCACxBvB,MAAM,gCAAgCwC;gCACtC,MAAM,CAACf,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvBqC,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BnB,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;wCACA,SAASW,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAalB;gCACnBvB,MAAM,0BAA0ByC;gCAChC,OAAO;oCACL;wCACE,MAAM5B,eACJC,aACAH,mBACA;wCAEF,OAAO,CAAC;wCACR,SAAS8B,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAanB;gCACnBvB,MAAM,0BAA0B0C;gCAChC,OAAO;oCACL;wCACE,MAAM7B,eACJC,aACAF,mBACA;wCAEF,OAAO,CAAC;wCACR,SAAS8B,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAapB;gCACnBvB,MAAM,0BAA0B2C;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAerB;gCACrBvB,MAAM,4BAA4B4C;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAOA,aAAa,GAAG;wCACvB,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGtB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIsB,MACR,CAAC,yBAAyB,EAAGzB,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAO0B,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrE9C,MAAM,oBAAoB+C;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}

package/dist/es/ai-model/auto-glm/index.mjs

@@ -0,0 +1,6 @@
+import { getAutoGLMLocatePrompt, getAutoGLMPlanPrompt } from "./prompt.mjs";
+import { parseAction, parseAutoGLMLocateResponse, parseAutoGLMResponse } from "./parser.mjs";
+import { autoGLMPlanning } from "./planning.mjs";
+import { transformAutoGLMAction } from "./actions.mjs";
+import { isAutoGLM, isUITars } from "./util.mjs";
+export { autoGLMPlanning, getAutoGLMLocatePrompt, getAutoGLMPlanPrompt, isAutoGLM, isUITars, parseAction, parseAutoGLMLocateResponse, parseAutoGLMResponse, transformAutoGLMAction };