npm - @midscene/core - Versions diffs - 1.8.10 → 1.9.0 - Mend

@midscene/core 1.8.10 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (304) hide show

package/dist/es/common.mjs CHANGED Viewed

@@ -1,180 +1,8 @@
 import { assert, isPlainObject } from "@midscene/shared/utils";
-import { isUITars } from "./ai-model/auto-glm/util.mjs";
 import { NodeType } from "@midscene/shared/constants";
 import { treeToList } from "@midscene/shared/extractor";
 import { compositeElementInfoImg } from "@midscene/shared/img";
-import { getDebug } from "@midscene/shared/logger";
 import { z } from "zod";
-const defaultBboxSize = 20;
-const debugInspectUtils = getDebug('ai:common');
-function pointToBbox(x, y, bboxSize = defaultBboxSize) {
-    const halfSize = bboxSize / 2;
-    const x1 = Math.max(x - halfSize, 0);
-    const y1 = Math.max(y - halfSize, 0);
-    const x2 = Math.min(x + halfSize, 1000);
-    const y2 = Math.min(y + halfSize, 1000);
-    return [
-        x1,
-        y1,
-        x2,
-        y2
-    ];
-}
-function fillBboxParam(locate, width, height, modelFamily) {
-    if (locate.bbox_2d && !locate?.bbox) {
-        locate.bbox = locate.bbox_2d;
-        delete locate.bbox_2d;
-    }
-    if (locate?.bbox) locate.bbox = adaptBbox(locate.bbox, width, height, modelFamily);
-    return locate;
-}
-function adaptQwen2_5Bbox(bbox) {
-    if (bbox.length < 2) {
-        const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;
-        throw new Error(msg);
-    }
-    const result = [
-        Math.round(bbox[0]),
-        Math.round(bbox[1]),
-        'number' == typeof bbox[2] ? Math.round(bbox[2]) : Math.round(bbox[0] + defaultBboxSize),
-        'number' == typeof bbox[3] ? Math.round(bbox[3]) : Math.round(bbox[1] + defaultBboxSize)
-    ];
-    return result;
-}
-function adaptGpt5Bbox(bbox) {
-    if (!Array.isArray(bbox) || 4 !== bbox.length || !bbox.every((value)=>'number' == typeof value && Number.isFinite(value))) {
-        const msg = `invalid bbox data for gpt-5 mode: ${JSON.stringify(bbox)} `;
-        throw new Error(msg);
-    }
-    const numericBbox = bbox;
-    return [
-        numericBbox[0],
-        numericBbox[1],
-        numericBbox[2],
-        numericBbox[3]
-    ];
-}
-function adaptDoubaoBbox(bbox, width, height) {
-    assert(width > 0 && height > 0, 'width and height must be greater than 0 in doubao mode');
-    if ('string' == typeof bbox) {
-        assert(/^(\d+)\s(\d+)\s(\d+)\s(\d+)$/.test(bbox.trim()), `invalid bbox data string for doubao-vision mode: ${bbox}`);
-        const splitted = bbox.split(' ');
-        if (4 === splitted.length) return [
-            Math.round(Number(splitted[0]) * width / 1000),
-            Math.round(Number(splitted[1]) * height / 1000),
-            Math.round(Number(splitted[2]) * width / 1000),
-            Math.round(Number(splitted[3]) * height / 1000)
-        ];
-        throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);
-    }
-    let bboxList = [];
-    if (Array.isArray(bbox) && 'string' == typeof bbox[0]) bbox.forEach((item)=>{
-        if ('string' == typeof item && item.includes(',')) {
-            const [x, y] = item.split(',');
-            bboxList.push(Number(x.trim()), Number(y.trim()));
-        } else if ('string' == typeof item && item.includes(' ')) {
-            const [x, y] = item.split(' ');
-            bboxList.push(Number(x.trim()), Number(y.trim()));
-        } else bboxList.push(Number(item));
-    });
-    else bboxList = bbox;
-    if (4 === bboxList.length || 5 === bboxList.length) return [
-        Math.round(bboxList[0] * width / 1000),
-        Math.round(bboxList[1] * height / 1000),
-        Math.round(bboxList[2] * width / 1000),
-        Math.round(bboxList[3] * height / 1000)
-    ];
-    if (6 === bboxList.length || 2 === bboxList.length || 3 === bboxList.length || 7 === bboxList.length) return [
-        Math.max(0, Math.round(bboxList[0] * width / 1000) - defaultBboxSize / 2),
-        Math.max(0, Math.round(bboxList[1] * height / 1000) - defaultBboxSize / 2),
-        Math.min(width, Math.round(bboxList[0] * width / 1000) + defaultBboxSize / 2),
-        Math.min(height, Math.round(bboxList[1] * height / 1000) + defaultBboxSize / 2)
-    ];
-    if (8 === bbox.length) return [
-        Math.round(bboxList[0] * width / 1000),
-        Math.round(bboxList[1] * height / 1000),
-        Math.round(bboxList[4] * width / 1000),
-        Math.round(bboxList[5] * height / 1000)
-    ];
-    const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;
-    throw new Error(msg);
-}
-function normalizeBboxInput(bbox) {
-    if (Array.isArray(bbox)) {
-        if (Array.isArray(bbox[0])) return bbox[0];
-    }
-    return bbox;
-}
-function adaptBbox(bbox, width, height, modelFamily) {
-    const normalizedBbox = normalizeBboxInput(bbox);
-    let result = [
-        0,
-        0,
-        0,
-        0
-    ];
-    result = 'doubao-vision' === modelFamily || 'doubao-seed' === modelFamily || isUITars(modelFamily) ? adaptDoubaoBbox(normalizedBbox, width, height) : 'gemini' === modelFamily ? adaptGeminiBbox(normalizedBbox, width, height) : 'qwen2.5-vl' === modelFamily ? adaptQwen2_5Bbox(normalizedBbox) : 'gpt-5' === modelFamily ? adaptGpt5Bbox(normalizedBbox) : normalized01000(normalizedBbox, width, height, modelFamily);
-    return result;
-}
-function normalized01000(bbox, width, height, modelFamily) {
-    if (void 0 !== modelFamily) {
-        if (!Array.isArray(bbox)) throw new Error(`Model family "${modelFamily}" returned a non-array bbox for the normalized [0, 1000] format: ${JSON.stringify(bbox)} (shotSize=${width}x${height}). Expected an array of 4 numbers.`);
-        for (const value of bbox)if ('number' != typeof value || !Number.isFinite(value) || value < 0 || value > 1000) throw new Error(`Model family "${modelFamily}" returned a bbox outside the expected [0, 1000] normalized range: ${JSON.stringify(bbox)} (shotSize=${width}x${height}). This usually means the model emitted pixel-style coordinates instead of normalized values. Consider switching MIDSCENE_MODEL_FAMILY to a more reliable visual grounding family (e.g. "qwen3-vl"), or revisit the bbox prompt for this model.`);
-    }
-    const normalizedBbox = bbox;
-    return [
-        Math.round(normalizedBbox[0] * width / 1000),
-        Math.round(normalizedBbox[1] * height / 1000),
-        Math.round(normalizedBbox[2] * width / 1000),
-        Math.round(normalizedBbox[3] * height / 1000)
-    ];
-}
-function adaptGeminiBbox(bbox, width, height) {
-    const left = Math.round(bbox[1] * width / 1000);
-    const top = Math.round(bbox[0] * height / 1000);
-    const right = Math.round(bbox[3] * width / 1000);
-    const bottom = Math.round(bbox[2] * height / 1000);
-    return [
-        left,
-        top,
-        right,
-        bottom
-    ];
-}
-function adaptBboxToRect(bbox, width, height, offsetX = 0, offsetY = 0, rightLimit = width, bottomLimit = height, modelFamily, scale = 1) {
-    debugInspectUtils('adaptBboxToRect', bbox, width, height, 'offset', offsetX, offsetY, 'limit', rightLimit, bottomLimit, 'modelFamily', modelFamily, 'scale', scale);
-    const [left, top, right, bottom] = adaptBbox(bbox, width, height, modelFamily);
-    const rectLeft = Math.max(0, left);
-    const rectTop = Math.max(0, top);
-    const boundedRight = Math.min(right, rightLimit);
-    const boundedBottom = Math.min(bottom, bottomLimit);
-    const rectWidth = boundedRight - rectLeft + 1;
-    const rectHeight = boundedBottom - rectTop + 1;
-    const finalLeft = 1 !== scale ? Math.round(rectLeft / scale) : rectLeft;
-    const finalTop = 1 !== scale ? Math.round(rectTop / scale) : rectTop;
-    const finalWidth = 1 !== scale ? Math.round(rectWidth / scale) : rectWidth;
-    const finalHeight = 1 !== scale ? Math.round(rectHeight / scale) : rectHeight;
-    const rect = {
-        left: finalLeft + offsetX,
-        top: finalTop + offsetY,
-        width: finalWidth,
-        height: finalHeight
-    };
-    debugInspectUtils('adaptBboxToRect, result=', rect);
-    return rect;
-}
-function mergeRects(rects) {
-    const minLeft = Math.min(...rects.map((r)=>r.left));
-    const minTop = Math.min(...rects.map((r)=>r.top));
-    const maxRight = Math.max(...rects.map((r)=>r.left + r.width));
-    const maxBottom = Math.max(...rects.map((r)=>r.top + r.height));
-    return {
-        left: minLeft,
-        top: minTop,
-        width: maxRight - minLeft,
-        height: maxBottom - minTop
-    };
-}
 function expandSearchArea(rect, screenSize) {
     const minArea = 160000;
     const expandSize = 100;
@@ -228,7 +56,7 @@ function buildYamlFlowFromPlans(plans, actionSpace) {
         }
         const flowKey = action.interfaceAlias || verb;
         const flowParam = action.paramSchema ? dumpActionParam(plan.param || {}, action.paramSchema) : {};
-        const shortcutField = 'Launch' === action.name || 'launch' === action.interfaceAlias ? 'uri' : 'Terminate' === action.name || 'terminate' === action.interfaceAlias ? 'uri' : 'RunAdbShell' === action.name || 'runAdbShell' === action.interfaceAlias ? 'command' : void 0;
+        const shortcutField = 'Launch' === action.name || 'launch' === action.interfaceAlias ? 'uri' : 'Terminate' === action.name || 'terminate' === action.interfaceAlias ? 'uri' : 'RunAdbShell' === action.name || 'runAdbShell' === action.interfaceAlias || 'RunHdcShell' === action.name || 'runHdcShell' === action.interfaceAlias ? 'command' : void 0;
         const shortcutKeys = shortcutField ? Object.keys(flowParam) : [];
         const canInlineShortcut = shortcutField && 1 === shortcutKeys.length && shortcutKeys[0] === shortcutField && 'string' == typeof flowParam[shortcutField];
         const flowItem = canInlineShortcut ? {
@@ -384,6 +212,6 @@ const getReadableTimeString = (format = 'YYYY-MM-DD HH:mm:ss', timestamp)=>{
     const timeString = format.replace('YYYY', String(year)).replace('MM', month).replace('DD', day).replace('HH', hours).replace('mm', minutes).replace('ss', seconds);
     return `${timeString} (${format})`;
 };
-export { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptGpt5Bbox, adaptQwen2_5Bbox, buildYamlFlowFromPlans, dumpActionParam, dumpMidsceneLocatorField, expandSearchArea, fillBboxParam, finalizeActionName, findAllMidsceneLocatorField, getMidsceneLocationSchema, getReadableTimeString, ifMidsceneLocatorField, markupImageForLLM, mergeRects, normalized01000, parseActionParam, pointToBbox };
+export { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, buildYamlFlowFromPlans, dumpActionParam, dumpMidsceneLocatorField, expandSearchArea, finalizeActionName, findAllMidsceneLocatorField, getMidsceneLocationSchema, getReadableTimeString, ifMidsceneLocatorField, markupImageForLLM, parseActionParam };
 //# sourceMappingURL=common.mjs.map

package/dist/es/common.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"common.mjs","sources":["../../src/common.ts"],"sourcesContent":["import type {\n BaseElement,\n DeviceAction,\n ElementTreeNode,\n MidsceneYamlFlowItem,\n PlanningAction,\n Rect,\n Size,\n} from '@/types';\nimport { assert, isPlainObject } from '@midscene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport { isUITars } from '@/ai-model/auto-glm/util';\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\ntype AdaptBboxInput = number[] \| string[] \| string \| (number[] \| string[])[];\n\n/*\n Convert a point coordinate [0, 1000] to a small bbox [0, 1000]\n * Creates a small bbox around the center point in the same coordinate space\n \n @param x - X coordinate in [0, 1000] range\n * @param y - Y coordinate in [0, 1000] range\n * @param bboxSize - Size of the bbox to create (default: 20)\n * @returns [x1, y1, x2, y2] bbox in [0, 1000] coordinate space\n /\nexport function pointToBbox(\n x: number,\n y: number,\n bboxSize = defaultBboxSize,\n): [number, number, number, number] {\n const halfSize = bboxSize / 2;\n const x1 = Math.max(x - halfSize, 0);\n const y1 = Math.max(y - halfSize, 0);\n const x2 = Math.min(x + halfSize, 1000);\n const y2 = Math.min(y + halfSize, 1000);\n\n return [x1, y1, x2, y2];\n}\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n locate: PlanningLocateParam,\n width: number,\n height: number,\n modelFamily: TModelFamily \| undefined,\n) {\n // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n if ((locate as any).bbox_2d && !locate?.bbox) {\n locate.bbox = (locate as any).bbox_2d;\n // biome-ignore lint/performance/noDelete: <explanation>\n delete (locate as any).bbox_2d;\n }\n\n if (locate?.bbox) {\n locate.bbox = adaptBbox(locate.bbox, width, height, modelFamily);\n }\n\n return locate;\n}\n\nexport function adaptQwen2_5Bbox(\n bbox: number[],\n): [number, number, number, number] {\n if (bbox.length < 2) {\n const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n throw new Error(msg);\n }\n\n const result: [number, number, number, number] = [\n Math.round(bbox[0]),\n Math.round(bbox[1]),\n typeof bbox[2] === 'number'\n ? Math.round(bbox[2])\n : Math.round(bbox[0] + defaultBboxSize),\n typeof bbox[3] === 'number'\n ? Math.round(bbox[3])\n : Math.round(bbox[1] + defaultBboxSize),\n ];\n return result;\n}\n\nexport function adaptGpt5Bbox(\n bbox: number[] \| string[] \| string,\n): [number, number, number, number] {\n if (\n !Array.isArray(bbox) \|\|\n bbox.length !== 4 \|\|\n !bbox.every((value) => typeof value === 'number' && Number.isFinite(value))\n ) {\n const msg = `invalid bbox data for gpt-5 mode: ${JSON.stringify(bbox)} `;\n throw new Error(msg);\n }\n\n const numericBbox = bbox as number[];\n return [numericBbox[0], numericBbox[1], numericBbox[2], numericBbox[3]];\n}\n\nexport function adaptDoubaoBbox(\n bbox: string[] \| number[] \| string,\n width: number,\n height: number,\n): [number, number, number, number] {\n assert(\n width > 0 && height > 0,\n 'width and height must be greater than 0 in doubao mode',\n );\n\n if (typeof bbox === 'string') {\n assert(\n /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n `invalid bbox data string for doubao-vision mode: ${bbox}`,\n );\n const splitted = bbox.split(' ');\n if (splitted.length === 4) {\n return [\n Math.round((Number(splitted[0]) width) / 1000),\n Math.round((Number(splitted[1]) * height) / 1000),\n Math.round((Number(splitted[2]) * width) / 1000),\n Math.round((Number(splitted[3]) * height) / 1000),\n ];\n }\n throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n }\n\n let bboxList: number[] = [];\n if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n bbox.forEach((item) => {\n if (typeof item === 'string' && item.includes(',')) {\n const [x, y] = item.split(',');\n bboxList.push(Number(x.trim()), Number(y.trim()));\n } else if (typeof item === 'string' && item.includes(' ')) {\n const [x, y] = item.split(' ');\n bboxList.push(Number(x.trim()), Number(y.trim()));\n } else {\n bboxList.push(Number(item));\n }\n });\n } else {\n bboxList = bbox as any;\n }\n\n if (bboxList.length === 4 \|\| bboxList.length === 5) {\n return [\n Math.round((bboxList[0] * width) / 1000),\n Math.round((bboxList[1] * height) / 1000),\n Math.round((bboxList[2] * width) / 1000),\n Math.round((bboxList[3] * height) / 1000),\n ];\n }\n\n // treat the bbox as a center point\n if (\n bboxList.length === 6 \|\|\n bboxList.length === 2 \|\|\n bboxList.length === 3 \|\|\n bboxList.length === 7\n ) {\n return [\n Math.max(\n 0,\n Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n ),\n Math.max(\n 0,\n Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n ),\n Math.min(\n width,\n Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n ),\n Math.min(\n height,\n Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n ),\n ];\n }\n\n if (bbox.length === 8) {\n return [\n Math.round((bboxList[0] * width) / 1000),\n Math.round((bboxList[1] * height) / 1000),\n Math.round((bboxList[4] * width) / 1000),\n Math.round((bboxList[5] * height) / 1000),\n ];\n }\n\n const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n throw new Error(msg);\n}\n\nfunction normalizeBboxInput(\n bbox: AdaptBboxInput,\n): number[] \| string[] \| string {\n if (Array.isArray(bbox)) {\n if (Array.isArray(bbox[0])) {\n return bbox[0] as number[] \| string[];\n }\n return bbox as number[] \| string[];\n }\n return bbox as string;\n}\n\nexport function adaptBbox(\n bbox: AdaptBboxInput,\n width: number,\n height: number,\n modelFamily: TModelFamily \| undefined,\n): [number, number, number, number] {\n const normalizedBbox = normalizeBboxInput(bbox);\n\n let result: [number, number, number, number] = [0, 0, 0, 0];\n if (\n modelFamily === 'doubao-vision' \|\|\n modelFamily === 'doubao-seed' \|\|\n isUITars(modelFamily)\n ) {\n result = adaptDoubaoBbox(normalizedBbox, width, height);\n } else if (modelFamily === 'gemini') {\n result = adaptGeminiBbox(normalizedBbox as number[], width, height);\n } else if (modelFamily === 'qwen2.5-vl') {\n result = adaptQwen2_5Bbox(normalizedBbox as number[]);\n } else if (modelFamily === 'gpt-5') {\n result = adaptGpt5Bbox(normalizedBbox);\n } else {\n // Default: normalized 0-1000 coordinate system\n // Includes: qwen3-vl, qwen3, qwen3.5, qwen3.6, glm-v, auto-glm, auto-glm-multilingual,\n // and future models.\n result = normalized01000(normalizedBbox, width, height, modelFamily);\n }\n\n return result;\n}\n\n// x1, y1, x2, y2 -> 0-1000\nexport function normalized01000(\n bbox: number[] \| string[] \| string,\n width: number,\n height: number,\n modelFamily?: TModelFamily \| undefined,\n): [number, number, number, number] {\n if (modelFamily !== undefined) {\n if (!Array.isArray(bbox)) {\n throw new Error(\n `Model family \"${modelFamily}\" returned a non-array bbox for the normalized [0, 1000] format: ${JSON.stringify(bbox)} (shotSize=${width}x${height}). Expected an array of 4 numbers.`,\n );\n }\n\n for (const value of bbox) {\n if (\n typeof value !== 'number' \|\|\n !Number.isFinite(value) \|\|\n value < 0 \|\|\n value > 1000\n ) {\n throw new Error(\n `Model family \"${modelFamily}\" returned a bbox outside the expected [0, 1000] normalized range: ${JSON.stringify(bbox)} (shotSize=${width}x${height}). This usually means the model emitted pixel-style coordinates instead of normalized values. Consider switching MIDSCENE_MODEL_FAMILY to a more reliable visual grounding family (e.g. \"qwen3-vl\"), or revisit the bbox prompt for this model.`,\n );\n }\n }\n }\n\n const normalizedBbox = bbox as number[];\n\n return [\n Math.round((normalizedBbox[0] * width) / 1000),\n Math.round((normalizedBbox[1] * height) / 1000),\n Math.round((normalizedBbox[2] * width) / 1000),\n Math.round((normalizedBbox[3] * height) / 1000),\n ];\n}\n\n// y1, x1, y2, x2 -> 0-1000\nexport function adaptGeminiBbox(\n bbox: number[],\n width: number,\n height: number,\n): [number, number, number, number] {\n const left = Math.round((bbox[1] * width) / 1000);\n const top = Math.round((bbox[0] * height) / 1000);\n const right = Math.round((bbox[3] * width) / 1000);\n const bottom = Math.round((bbox[2] * height) / 1000);\n return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n bbox: number[],\n width: number,\n height: number,\n offsetX = 0,\n offsetY = 0,\n rightLimit = width,\n bottomLimit = height,\n modelFamily?: TModelFamily \| undefined,\n scale = 1,\n): Rect {\n debugInspectUtils(\n 'adaptBboxToRect',\n bbox,\n width,\n height,\n 'offset',\n offsetX,\n offsetY,\n 'limit',\n rightLimit,\n bottomLimit,\n 'modelFamily',\n modelFamily,\n 'scale',\n scale,\n );\n\n const [left, top, right, bottom] = adaptBbox(\n bbox,\n width,\n height,\n modelFamily,\n );\n\n // Calculate initial rect dimensions and apply boundary constraints\n // For left and top: take max with 0 to ensure they're not negative\n const rectLeft = Math.max(0, left);\n const rectTop = Math.max(0, top);\n\n // For width and height: calculate from bounded coordinates and constrain to limits\n const boundedRight = Math.min(right, rightLimit);\n const boundedBottom = Math.min(bottom, bottomLimit);\n\n const rectWidth = boundedRight - rectLeft + 1;\n const rectHeight = boundedBottom - rectTop + 1;\n\n // Apply scale after calculating dimensions - scale back to original size\n const finalLeft = scale !== 1 ? Math.round(rectLeft / scale) : rectLeft;\n const finalTop = scale !== 1 ? Math.round(rectTop / scale) : rectTop;\n const finalWidth = scale !== 1 ? Math.round(rectWidth / scale) : rectWidth;\n const finalHeight = scale !== 1 ? Math.round(rectHeight / scale) : rectHeight;\n\n const rect = {\n left: finalLeft + offsetX,\n top: finalTop + offsetY,\n width: finalWidth,\n height: finalHeight,\n };\n debugInspectUtils('adaptBboxToRect, result=', rect);\n\n return rect;\n}\n\nexport function mergeRects(rects: Rect[]) {\n const minLeft = Math.min(...rects.map((r) => r.left));\n const minTop = Math.min(...rects.map((r) => r.top));\n const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n return {\n left: minLeft,\n top: minTop,\n width: maxRight - minLeft,\n height: maxBottom - minTop,\n };\n}\n\n/*\n Expand the search area to at least 400 x 400 pixels\n \n Step 1: Extend 100px on each side (top, right, bottom, left)\n * - If the element is near a boundary, expansion on that side will be limited\n * - No compensation is made for boundary limitations (this is intentional)\n \n Step 2: Ensure the area is at least 400x400 pixels\n * - Scale up proportionally from the center if needed\n * - Final result is clamped to screen boundaries\n /\nexport function expandSearchArea(rect: Rect, screenSize: Size): Rect {\n const minArea = 400 400;\n const expandSize = 100;\n\n // Step 1: Extend each side by expandSize (100px), clamped to screen boundaries\n // Note: If element is near boundary, actual expansion may be less than 100px on that side\n const expandedLeft = Math.max(rect.left - expandSize, 0);\n const expandedTop = Math.max(rect.top - expandSize, 0);\n\n const expandRect = {\n left: expandedLeft,\n top: expandedTop,\n width: Math.min(\n rect.left - expandedLeft + rect.width + expandSize,\n screenSize.width - expandedLeft,\n ),\n height: Math.min(\n rect.top - expandedTop + rect.height + expandSize,\n screenSize.height - expandedTop,\n ),\n };\n\n // Step 2: Check if area is already >= 400x400\n const currentArea = expandRect.width * expandRect.height;\n\n if (currentArea >= minArea) {\n return expandRect;\n }\n\n // Step 2: Scale up from center to reach minimum 400x400 area\n const centerX = expandRect.left + expandRect.width / 2;\n const centerY = expandRect.top + expandRect.height / 2;\n\n // Calculate scale factor needed to reach minimum area\n const scaleFactor = Math.sqrt(minArea / currentArea);\n const newWidth = Math.round(expandRect.width * scaleFactor);\n const newHeight = Math.round(expandRect.height * scaleFactor);\n\n // Calculate new position based on center point\n const newLeft = Math.round(centerX - newWidth / 2);\n const newTop = Math.round(centerY - newHeight / 2);\n\n // Clamp to screen boundaries\n const left = Math.max(newLeft, 0);\n const top = Math.max(newTop, 0);\n\n return {\n left,\n top,\n width: Math.min(newWidth, screenSize.width - left),\n height: Math.min(newHeight, screenSize.height - top),\n };\n}\n\nexport async function markupImageForLLM(\n screenshotBase64: string,\n tree: ElementTreeNode<BaseElement>,\n size: Size,\n) {\n const elementsInfo = treeToList(tree);\n const elementsPositionInfoWithoutText = elementsInfo!.filter(\n (elementInfo) => {\n if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n return false;\n }\n return true;\n },\n );\n\n const imagePayload = await compositeElementInfoImg({\n inputImgBase64: screenshotBase64,\n elementsPositionInfo: elementsPositionInfoWithoutText,\n size,\n });\n return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n plans: PlanningAction[],\n actionSpace: DeviceAction<any>[],\n): MidsceneYamlFlowItem[] {\n const flow: MidsceneYamlFlowItem[] = [];\n\n for (const plan of plans) {\n const verb = plan.type;\n\n const action = actionSpace.find((action) => action.name === verb);\n if (!action) {\n console.warn(\n `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n );\n continue;\n }\n\n const flowKey = action.interfaceAlias \|\| verb;\n const flowParam = action.paramSchema\n ? dumpActionParam(plan.param \|\| {}, action.paramSchema)\n : {};\n\n // For actions whose param is a single string field (e.g. Launch/Terminate's\n // `uri`, RunAdbShell's `command`), inline the value on the flowKey. Writing\n // `{ terminate: '', uri: '...' }` makes the YAML player treat the empty\n // string as the param and drop the sibling `uri`, so cache replay would\n // call the action with an empty argument.\n const shortcutField =\n action.name === 'Launch' \|\| action.interfaceAlias === 'launch'\n ? 'uri'\n : action.name === 'Terminate' \|\| action.interfaceAlias === 'terminate'\n ? 'uri'\n : action.name === 'RunAdbShell' \|\|\n action.interfaceAlias === 'runAdbShell'\n ? 'command'\n : undefined;\n const shortcutKeys = shortcutField ? Object.keys(flowParam) : [];\n const canInlineShortcut =\n shortcutField &&\n shortcutKeys.length === 1 &&\n shortcutKeys[0] === shortcutField &&\n typeof flowParam[shortcutField] === 'string';\n\n const flowItem: MidsceneYamlFlowItem = canInlineShortcut\n ? { [flowKey]: flowParam[shortcutField as string] }\n : { [flowKey]: '', ...flowParam };\n\n flow.push(flowItem);\n }\n\n return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n left: z.number(),\n top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n width: z.number(),\n height: z.number(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n z.object({\n zoom: z.number().optional(),\n }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n images: z\n .array(\n z.object({\n name: z.string(),\n url: z.string(),\n }),\n )\n .optional(),\n convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n z.string(),\n z\n .object({\n prompt: z.string(),\n })\n .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n .object({\n prompt: TUserPromptSchema,\n deepLocate: z.boolean().optional(),\n deepThink: z\n .boolean()\n .optional()\n .describe('@deprecated Use `deepLocate` instead.'),\n cacheable: z.boolean().optional(),\n xpath: z.union([z.string(), z.boolean()]).optional(),\n })\n .passthrough();\n\n/*\n Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n /\nexport const getMidsceneLocationSchema = () => {\n return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n // Handle optional fields by getting the inner type\n let actualField = field;\n if (actualField._def?.typeName === 'ZodOptional') {\n actualField = actualField._def.innerType;\n }\n\n // Check if this is a ZodObject\n if (actualField._def?.typeName === 'ZodObject') {\n const shape = actualField._def.shape();\n\n // Method 1: Check for the location field flag (for result schema)\n if (locateFieldFlagName in shape) {\n return true;\n }\n\n // Method 2: Check if it's the input schema by checking for 'prompt' field\n // Input schema has 'prompt' as a required field\n if ('prompt' in shape && shape.prompt) {\n return true;\n }\n }\n\n return false;\n};\n\nconst formatPromptWithImages = (\n promptObj: Exclude<TUserPrompt, string>,\n): string => {\n let promptString = promptObj.prompt;\n if (Array.isArray(promptObj.images) && promptObj.images.length > 0) {\n const imageCount = promptObj.images.length;\n promptString += ` (with ${imageCount} image${imageCount > 1 ? 's' : ''})`;\n }\n return promptString;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n assert(\n ifMidsceneLocatorField(field),\n 'field is not a midscene locator field',\n );\n\n // If field is a string, return it directly\n if (typeof field === 'string') {\n return field;\n }\n\n // If field is an object with prompt property\n if (field && typeof field === 'object' && field.prompt) {\n // If prompt is a string, return it directly\n if (typeof field.prompt === 'string') {\n return field.prompt;\n }\n // If prompt is a TUserPrompt object, extract the prompt string\n if (typeof field.prompt === 'object' && field.prompt.prompt) {\n return formatPromptWithImages(field.prompt);\n }\n }\n\n // Fallback: try to convert to string\n return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n zodType?: z.ZodType<any>,\n requiredOnly?: boolean,\n): string[] => {\n if (!zodType) {\n return [];\n }\n\n // Check if this is a ZodObject by checking if it has a shape property\n const zodObject = zodType as any;\n if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n const keys = Object.keys(zodObject.shape);\n return keys.filter((key) => {\n const field = zodObject.shape[key];\n if (!ifMidsceneLocatorField(field)) {\n return false;\n }\n\n // If requiredOnly is true, filter out optional fields\n if (requiredOnly) {\n return field._def?.typeName !== 'ZodOptional';\n }\n\n return true;\n });\n }\n\n // For other ZodType instances, we can't extract field names\n return [];\n};\n\nexport const dumpActionParam = (\n jsonObject: Record<string, any>,\n zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n // Prevent spreading strings into {0: 'c', 1: 'o', ...}\n if (!isPlainObject(jsonObject)) {\n return {};\n }\n\n const locatorFields = findAllMidsceneLocatorField(zodSchema);\n const result = { ...jsonObject };\n\n for (const fieldName of locatorFields) {\n const fieldValue = result[fieldName];\n if (fieldValue) {\n // If it's already a string, keep it as is\n if (typeof fieldValue === 'string') {\n result[fieldName] = fieldValue;\n } else if (typeof fieldValue === 'object') {\n // Check if this field is actually a MidsceneLocationType object\n if (fieldValue.prompt) {\n // If prompt is a string, use it directly\n if (typeof fieldValue.prompt === 'string') {\n result[fieldName] = fieldValue.prompt;\n } else if (\n typeof fieldValue.prompt === 'object' &&\n fieldValue.prompt.prompt\n ) {\n // If prompt is a TUserPrompt object, extract the prompt string\n result[fieldName] = formatPromptWithImages(fieldValue.prompt);\n }\n }\n }\n }\n }\n\n return result;\n};\n\n/\n Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n \n Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n \n When shrunkShotToLogicalRatio is provided and !== 1, coordinates in locate fields\n * are transformed from screenshot space to logical space.\n /\nexport const parseActionParam = (\n rawParam: Record<string, any> \| undefined,\n zodSchema?: z.ZodType<any>,\n options?: { shrunkShotToLogicalRatio?: number },\n): Record<string, any> \| undefined => {\n // If no schema is provided, return undefined (action takes no parameters)\n if (!zodSchema) {\n return undefined;\n }\n\n // Handle undefined or null rawParam by providing an empty object\n const param = rawParam ?? {};\n\n // Find all locate fields in the schema\n const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n // If there are no locate fields, just do normal validation\n if (locateFields.length === 0) {\n return zodSchema.parse(param);\n }\n\n // Extract locate field values to restore later\n const locateFieldValues: Record<string, any> = {};\n for (const fieldName of locateFields) {\n if (fieldName in param) {\n locateFieldValues[fieldName] = param[fieldName];\n }\n }\n\n // Build params for validation - skip locate fields and use dummy values\n const paramsForValidation: Record<string, any> = {};\n for (const key in param) {\n if (locateFields.includes(key)) {\n // Use dummy value to satisfy schema validation\n paramsForValidation[key] = { prompt: '_dummy_' };\n } else {\n paramsForValidation[key] = param[key];\n }\n }\n\n // Validate with dummy locate values\n const validated = zodSchema.parse(paramsForValidation);\n\n // Restore the actual locate field values (unvalidated, as per business requirement),\n // and transform coordinates from screenshot space to logical space if needed\n const ratio = options?.shrunkShotToLogicalRatio;\n for (const fieldName in locateFieldValues) {\n let value = locateFieldValues[fieldName];\n if (\n ratio !== undefined &&\n ratio !== 1 &&\n value &&\n typeof value === 'object' &&\n value.center &&\n value.rect\n ) {\n value = {\n ...value,\n center: [\n Math.round(value.center[0] / ratio),\n Math.round(value.center[1] / ratio),\n ],\n rect: {\n ...value.rect,\n left: Math.round(value.rect.left / ratio),\n top: Math.round(value.rect.top / ratio),\n width: Math.round(value.rect.width / ratio),\n height: Math.round(value.rect.height / ratio),\n },\n };\n }\n validated[fieldName] = value;\n }\n\n return validated;\n};\n\nexport const finalizeActionName = 'Finalize';\n\n/\n Get a readable time string for a given timestamp or the current time\n * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'\n * @param timestamp - Optional timestamp in milliseconds. If not provided, uses current system time.\n * @returns A formatted time string with format label\n */\nexport const getReadableTimeString = (\n format = 'YYYY-MM-DD HH:mm:ss',\n timestamp?: number,\n): string => {\n const now = timestamp !== undefined ? new Date(timestamp) : new Date();\n const year = now.getFullYear();\n const month = String(now.getMonth() + 1).padStart(2, '0');\n const day = String(now.getDate()).padStart(2, '0');\n const hours = String(now.getHours()).padStart(2, '0');\n const minutes = String(now.getMinutes()).padStart(2, '0');\n const seconds = String(now.getSeconds()).padStart(2, '0');\n\n const timeString = format\n .replace('YYYY', String(year))\n .replace('MM', month)\n .replace('DD', day)\n .replace('HH', hours)\n .replace('mm', minutes)\n .replace('ss', seconds);\n\n return `${timeString} (${format})`;\n};\n"],"names":["defaultBboxSize","debugInspectUtils","getDebug","pointToBbox","x","y","bboxSize","halfSize","x1","Math","y1","x2","y2","fillBboxParam","locate","width","height","modelFamily","adaptBbox","adaptQwen2_5Bbox","bbox","msg","JSON","Error","result","adaptGpt5Bbox","Array","value","Number","numericBbox","adaptDoubaoBbox","assert","splitted","bboxList","item","normalizeBboxInput","normalizedBbox","isUITars","adaptGeminiBbox","normalized01000","undefined","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rightLimit","bottomLimit","scale","rectLeft","rectTop","boundedRight","boundedBottom","rectWidth","rectHeight","finalLeft","finalTop","finalWidth","finalHeight","rect","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minArea","expandSize","expandedLeft","expandedTop","expandRect","currentArea","centerX","centerY","scaleFactor","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","size","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","flow","plan","verb","action","console","flowKey","flowParam","dumpActionParam","shortcutField","shortcutKeys","Object","canInlineShortcut","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","formatPromptWithImages","promptObj","promptString","imageCount","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","key","jsonObject","zodSchema","isPlainObject","locatorFields","fieldName","fieldValue","parseActionParam","rawParam","options","param","locateFields","locateFieldValues","paramsForValidation","validated","ratio","finalizeActionName","getReadableTimeString","format","timestamp","now","Date","year","month","day","hours","minutes","seconds","timeString"],"mappings":";;;;;;;AAwBA,MAAMA,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAY5B,SAASC,YACdC,CAAS,EACTC,CAAS,EACTC,WAAWN,eAAe;IAE1B,MAAMO,WAAWD,WAAW;IAC5B,MAAME,KAAKC,KAAK,GAAG,CAACL,IAAIG,UAAU;IAClC,MAAMG,KAAKD,KAAK,GAAG,CAACJ,IAAIE,UAAU;IAClC,MAAMI,KAAKF,KAAK,GAAG,CAACL,IAAIG,UAAU;IAClC,MAAMK,KAAKH,KAAK,GAAG,CAACJ,IAAIE,UAAU;IAElC,OAAO;QAACC;QAAIE;QAAIC;QAAIC;KAAG;AACzB;AAGO,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,WAAqC;IAGrC,IAAKH,OAAe,OAAO,IAAI,CAACA,QAAQ,MAAM;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAQ,MACVA,OAAO,IAAI,GAAGI,UAAUJ,OAAO,IAAI,EAAEC,OAAOC,QAAQC;IAGtD,OAAOH;AACT;AAEO,SAASK,iBACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/Cf,KAAK,KAAK,CAACW,IAAI,CAAC,EAAE;QAClBX,KAAK,KAAK,CAACW,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVX,KAAK,KAAK,CAACW,IAAI,CAAC,EAAE,IAClBX,KAAK,KAAK,CAACW,IAAI,CAAC,EAAE,GAAGpB;QACN,YAAnB,OAAOoB,IAAI,CAAC,EAAE,GACVX,KAAK,KAAK,CAACW,IAAI,CAAC,EAAE,IAClBX,KAAK,KAAK,CAACW,IAAI,CAAC,EAAE,GAAGpB;KAC1B;IACD,OAAOwB;AACT;AAEO,SAASC,cACdL,IAAkC;IAElC,IACE,CAACM,MAAM,OAAO,CAACN,SACfA,AAAgB,MAAhBA,KAAK,MAAM,IACX,CAACA,KAAK,KAAK,CAAC,CAACO,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD,SACpE;QACA,MAAMN,MAAM,CAAC,kCAAkC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QACxE,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMQ,cAAcT;IACpB,OAAO;QAACS,WAAW,CAAC,EAAE;QAAEA,WAAW,CAAC,EAAE;QAAEA,WAAW,CAAC,EAAE;QAAEA,WAAW,CAAC,EAAE;KAAC;AACzE;AAEO,SAASC,gBACdV,IAAkC,EAClCL,KAAa,EACbC,MAAc;IAEde,OACEhB,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOI,MAAmB;QAC5BW,OACE,+BAA+B,IAAI,CAACX,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMY,WAAWZ,KAAK,KAAK,CAAC;QAC5B,IAAIY,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLvB,KAAK,KAAK,CAAEmB,OAAOI,QAAQ,CAAC,EAAE,IAAIjB,QAAS;YAC3CN,KAAK,KAAK,CAAEmB,OAAOI,QAAQ,CAAC,EAAE,IAAIhB,SAAU;YAC5CP,KAAK,KAAK,CAAEmB,OAAOI,QAAQ,CAAC,EAAE,IAAIjB,QAAS;YAC3CN,KAAK,KAAK,CAAEmB,OAAOI,QAAQ,CAAC,EAAE,IAAIhB,SAAU;SAC7C;QAEH,MAAM,IAAIO,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIa,WAAqB,EAAE;IAC3B,IAAIP,MAAM,OAAO,CAACN,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACc;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAAC9B,GAAGC,EAAE,GAAG6B,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACL,OAAOxB,EAAE,IAAI,KAAKwB,OAAOvB,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAO6B,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAAC9B,GAAGC,EAAE,GAAG6B,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACL,OAAOxB,EAAE,IAAI,KAAKwB,OAAOvB,EAAE,IAAI;QAC/C,OACE4B,SAAS,IAAI,CAACL,OAAOM;IAEzB;SAEAD,WAAWb;IAGb,IAAIa,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLxB,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCN,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGjB,SAAU;QACpCP,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCN,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGjB,SAAU;KACrC;IAIH,IACEiB,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLxB,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGlB,QAAS,QAAQf,kBAAkB;QAE/DS,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGjB,SAAU,QAAQhB,kBAAkB;QAEhES,KAAK,GAAG,CACNM,OACAN,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGlB,QAAS,QAAQf,kBAAkB;QAE/DS,KAAK,GAAG,CACNO,QACAP,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGjB,SAAU,QAAQhB,kBAAkB;KAEjE;IAGH,IAAIoB,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLX,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCN,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGjB,SAAU;QACpCP,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCN,KAAK,KAAK,CAAEwB,QAAQ,CAAC,EAAE,GAAGjB,SAAU;KACrC;IAGH,MAAMK,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEA,SAASc,mBACPf,IAAoB;IAEpB,IAAIM,MAAM,OAAO,CAACN,OAChB;QAAA,IAAIM,MAAM,OAAO,CAACN,IAAI,CAAC,EAAE,GACvB,OAAOA,IAAI,CAAC,EAAE;IAChB;IAGF,OAAOA;AACT;AAEO,SAASF,UACdE,IAAoB,EACpBL,KAAa,EACbC,MAAc,EACdC,WAAqC;IAErC,MAAMmB,iBAAiBD,mBAAmBf;IAE1C,IAAII,SAA2C;QAAC;QAAG;QAAG;QAAG;KAAE;IAMzDA,SAJAP,AAAgB,oBAAhBA,eACAA,AAAgB,kBAAhBA,eACAoB,SAASpB,eAEAa,gBAAgBM,gBAAgBrB,OAAOC,UACvCC,AAAgB,aAAhBA,cACAqB,gBAAgBF,gBAA4BrB,OAAOC,UACnDC,AAAgB,iBAAhBA,cACAE,iBAAiBiB,kBACjBnB,AAAgB,YAAhBA,cACAQ,cAAcW,kBAKdG,gBAAgBH,gBAAgBrB,OAAOC,QAAQC;IAG1D,OAAOO;AACT;AAGO,SAASe,gBACdnB,IAAkC,EAClCL,KAAa,EACbC,MAAc,EACdC,WAAsC;IAEtC,IAAIA,AAAgBuB,WAAhBvB,aAA2B;QAC7B,IAAI,CAACS,MAAM,OAAO,CAACN,OACjB,MAAM,IAAIG,MACR,CAAC,cAAc,EAAEN,YAAY,iEAAiE,EAAEK,KAAK,SAAS,CAACF,MAAM,WAAW,EAAEL,MAAM,CAAC,EAAEC,OAAO,kCAAkC,CAAC;QAIzL,KAAK,MAAMW,SAASP,KAClB,IACE,AAAiB,YAAjB,OAAOO,SACP,CAACC,OAAO,QAAQ,CAACD,UACjBA,QAAQ,KACRA,QAAQ,MAER,MAAM,IAAIJ,MACR,CAAC,cAAc,EAAEN,YAAY,mEAAmE,EAAEK,KAAK,SAAS,CAACF,MAAM,WAAW,EAAEL,MAAM,CAAC,EAAEC,OAAO,+OAA+O,CAAC;IAI5Y;IAEA,MAAMoB,iBAAiBhB;IAEvB,OAAO;QACLX,KAAK,KAAK,CAAE2B,cAAc,CAAC,EAAE,GAAGrB,QAAS;QACzCN,KAAK,KAAK,CAAE2B,cAAc,CAAC,EAAE,GAAGpB,SAAU;QAC1CP,KAAK,KAAK,CAAE2B,cAAc,CAAC,EAAE,GAAGrB,QAAS;QACzCN,KAAK,KAAK,CAAE2B,cAAc,CAAC,EAAE,GAAGpB,SAAU;KAC3C;AACH;AAGO,SAASsB,gBACdlB,IAAc,EACdL,KAAa,EACbC,MAAc;IAEd,MAAMyB,OAAOhC,KAAK,KAAK,CAAEW,IAAI,CAAC,EAAE,GAAGL,QAAS;IAC5C,MAAM2B,MAAMjC,KAAK,KAAK,CAAEW,IAAI,CAAC,EAAE,GAAGJ,SAAU;IAC5C,MAAM2B,QAAQlC,KAAK,KAAK,CAAEW,IAAI,CAAC,EAAE,GAAGL,QAAS;IAC7C,MAAM6B,SAASnC,KAAK,KAAK,CAAEW,IAAI,CAAC,EAAE,GAAGJ,SAAU;IAC/C,OAAO;QAACyB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdzB,IAAc,EACdL,KAAa,EACbC,MAAc,EACd8B,UAAU,CAAC,EACXC,UAAU,CAAC,EACXC,aAAajC,KAAK,EAClBkC,cAAcjC,MAAM,EACpBC,WAAsC,EACtCiC,QAAQ,CAAC;IAETjD,kBACE,mBACAmB,MACAL,OACAC,QACA,UACA8B,SACAC,SACA,SACAC,YACAC,aACA,eACAhC,aACA,SACAiC;IAGF,MAAM,CAACT,MAAMC,KAAKC,OAAOC,OAAO,GAAG1B,UACjCE,MACAL,OACAC,QACAC;IAKF,MAAMkC,WAAW1C,KAAK,GAAG,CAAC,GAAGgC;IAC7B,MAAMW,UAAU3C,KAAK,GAAG,CAAC,GAAGiC;IAG5B,MAAMW,eAAe5C,KAAK,GAAG,CAACkC,OAAOK;IACrC,MAAMM,gBAAgB7C,KAAK,GAAG,CAACmC,QAAQK;IAEvC,MAAMM,YAAYF,eAAeF,WAAW;IAC5C,MAAMK,aAAaF,gBAAgBF,UAAU;IAG7C,MAAMK,YAAYP,AAAU,MAAVA,QAAczC,KAAK,KAAK,CAAC0C,WAAWD,SAASC;IAC/D,MAAMO,WAAWR,AAAU,MAAVA,QAAczC,KAAK,KAAK,CAAC2C,UAAUF,SAASE;IAC7D,MAAMO,aAAaT,AAAU,MAAVA,QAAczC,KAAK,KAAK,CAAC8C,YAAYL,SAASK;IACjE,MAAMK,cAAcV,AAAU,MAAVA,QAAczC,KAAK,KAAK,CAAC+C,aAAaN,SAASM;IAEnE,MAAMK,OAAO;QACX,MAAMJ,YAAYX;QAClB,KAAKY,WAAWX;QAChB,OAAOY;QACP,QAAQC;IACV;IACA3D,kBAAkB,4BAA4B4D;IAE9C,OAAOA;AACT;AAEO,SAASC,WAAWC,KAAa;IACtC,MAAMC,UAAUvD,KAAK,GAAG,IAAIsD,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAASzD,KAAK,GAAG,IAAIsD,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW1D,KAAK,GAAG,IAAIsD,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAY3D,KAAK,GAAG,IAAIsD,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAaO,SAASG,iBAAiBR,IAAU,EAAES,UAAgB;IAC3D,MAAMC,UAAU;IAChB,MAAMC,aAAa;IAInB,MAAMC,eAAehE,KAAK,GAAG,CAACoD,KAAK,IAAI,GAAGW,YAAY;IACtD,MAAME,cAAcjE,KAAK,GAAG,CAACoD,KAAK,GAAG,GAAGW,YAAY;IAEpD,MAAMG,aAAa;QACjB,MAAMF;QACN,KAAKC;QACL,OAAOjE,KAAK,GAAG,CACboD,KAAK,IAAI,GAAGY,eAAeZ,KAAK,KAAK,GAAGW,YACxCF,WAAW,KAAK,GAAGG;QAErB,QAAQhE,KAAK,GAAG,CACdoD,KAAK,GAAG,GAAGa,cAAcb,KAAK,MAAM,GAAGW,YACvCF,WAAW,MAAM,GAAGI;IAExB;IAGA,MAAME,cAAcD,WAAW,KAAK,GAAGA,WAAW,MAAM;IAExD,IAAIC,eAAeL,SACjB,OAAOI;IAIT,MAAME,UAAUF,WAAW,IAAI,GAAGA,WAAW,KAAK,GAAG;IACrD,MAAMG,UAAUH,WAAW,GAAG,GAAGA,WAAW,MAAM,GAAG;IAGrD,MAAMI,cAActE,KAAK,IAAI,CAAC8D,UAAUK;IACxC,MAAMI,WAAWvE,KAAK,KAAK,CAACkE,WAAW,KAAK,GAAGI;IAC/C,MAAME,YAAYxE,KAAK,KAAK,CAACkE,WAAW,MAAM,GAAGI;IAGjD,MAAMG,UAAUzE,KAAK,KAAK,CAACoE,UAAUG,WAAW;IAChD,MAAMG,SAAS1E,KAAK,KAAK,CAACqE,UAAUG,YAAY;IAGhD,MAAMxC,OAAOhC,KAAK,GAAG,CAACyE,SAAS;IAC/B,MAAMxC,MAAMjC,KAAK,GAAG,CAAC0E,QAAQ;IAE7B,OAAO;QACL1C;QACAC;QACA,OAAOjC,KAAK,GAAG,CAACuE,UAAUV,WAAW,KAAK,GAAG7B;QAC7C,QAAQhC,KAAK,GAAG,CAACwE,WAAWX,WAAW,MAAM,GAAG5B;IAClD;AACF;AAEO,eAAe0C,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCC,IAAU;IAEV,MAAMC,eAAeC,WAAWH;IAChC,MAAMI,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBT;QAChB,sBAAsBK;QACtBH;IACF;IACA,OAAOM;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC;IAEhC,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQH,MAAO;QACxB,MAAMI,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASJ,YAAY,IAAI,CAAC,CAACI,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEF,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMG,UAAUF,OAAO,cAAc,IAAID;QACzC,MAAMI,YAAYH,OAAO,WAAW,GAChCI,gBAAgBN,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAOL,MAAMK,gBACJL,AAAgB,aAAhBA,OAAO,IAAI,IAAiBA,AAA0B,aAA1BA,OAAO,cAAc,GAC7C,QACAA,AAAgB,gBAAhBA,OAAO,IAAI,IAAoBA,AAA0B,gBAA1BA,OAAO,cAAc,GAClD,QACAA,AAAgB,kBAAhBA,OAAO,IAAI,IACTA,AAA0B,kBAA1BA,OAAO,cAAc,GACrB,YACA7D;QACV,MAAMmE,eAAeD,gBAAgBE,OAAO,IAAI,CAACJ,aAAa,EAAE;QAChE,MAAMK,oBACJH,iBACAC,AAAwB,MAAxBA,aAAa,MAAM,IACnBA,YAAY,CAAC,EAAE,KAAKD,iBACpB,AAAoC,YAApC,OAAOF,SAAS,CAACE,cAAc;QAEjC,MAAMI,WAAiCD,oBACnC;YAAE,CAACN,QAAQ,EAAEC,SAAS,CAACE,cAAwB;QAAC,IAChD;YAAE,CAACH,QAAQ,EAAE;YAAI,GAAGC,SAAS;QAAC;QAElCN,KAAK,IAAI,CAACY;IACZ;IAEA,OAAOZ;AACT;AAGO,MAAMa,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;AAClB;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBN,EAAAA,MACrB,CAAC;IACN,QAAQI;IACR,YAAYJ,EAAE,OAAO,GAAG,QAAQ;IAChC,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,OAAO;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAMP,MAAMO,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIL,uBAAuBM,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEA,MAAMC,yBAAyB,CAC7BC;IAEA,IAAIC,eAAeD,UAAU,MAAM;IACnC,IAAInG,MAAM,OAAO,CAACmG,UAAU,MAAM,KAAKA,UAAU,MAAM,CAAC,MAAM,GAAG,GAAG;QAClE,MAAME,aAAaF,UAAU,MAAM,CAAC,MAAM;QAC1CC,gBAAgB,CAAC,OAAO,EAAEC,WAAW,MAAM,EAAEA,aAAa,IAAI,MAAM,GAAG,CAAC,CAAC;IAC3E;IACA,OAAOD;AACT;AAEO,MAAME,2BAA2B,CAACP;IACvC1F,OACEyF,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOG,uBAAuBH,MAAM,MAAM;IAE9C;IAGA,OAAOQ,OAAOR;AAChB;AAEO,MAAMS,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAO1B,OAAO,IAAI,CAACyB,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACC;YAClB,MAAMd,QAAQY,UAAU,KAAK,CAACE,IAAI;YAClC,IAAI,CAACf,uBAAuBC,QAC1B,OAAO;YAIT,IAAIW,cACF,OAAOX,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMhB,kBAAkB,CAC7B+B,YACAC;IAGA,IAAI,CAACC,cAAcF,aACjB,OAAO,CAAC;IAGV,MAAMG,gBAAgBT,4BAA4BO;IAClD,MAAMjH,SAAS;QAAE,GAAGgH,UAAU;IAAC;IAE/B,KAAK,MAAMI,aAAaD,cAAe;QACrC,MAAME,aAAarH,MAAM,CAACoH,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTrH,MAAM,CAACoH,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BrH,MAAM,CAACoH,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBrH,MAAM,CAACoH,UAAU,GAAGhB,uBAAuBiB,WAAW,MAAM;gBAC9D;YACF;QACF;IAEJ;IAEA,OAAOrH;AACT;AAaO,MAAMsH,mBAAmB,CAC9BC,UACAN,WACAO;IAGA,IAAI,CAACP,WACH;IAIF,MAAMQ,QAAQF,YAAY,CAAC;IAG3B,MAAMG,eAAehB,4BAA4BO;IAGjD,IAAIS,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOT,UAAU,KAAK,CAACQ;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMb,OAAOU,MAChB,IAAIC,aAAa,QAAQ,CAACX,MAExBa,mBAAmB,CAACb,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/Ca,mBAAmB,CAACb,IAAI,GAAGU,KAAK,CAACV,IAAI;IAKzC,MAAMc,YAAYZ,UAAU,KAAK,CAACW;IAIlC,MAAME,QAAQN,SAAS;IACvB,IAAK,MAAMJ,aAAaO,kBAAmB;QACzC,IAAIxH,QAAQwH,iBAAiB,CAACP,UAAU;QACxC,IACEU,AAAU9G,WAAV8G,SACAA,AAAU,MAAVA,SACA3H,SACA,AAAiB,YAAjB,OAAOA,SACPA,MAAM,MAAM,IACZA,MAAM,IAAI,EAEVA,QAAQ;YACN,GAAGA,KAAK;YACR,QAAQ;gBACNlB,KAAK,KAAK,CAACkB,MAAM,MAAM,CAAC,EAAE,GAAG2H;gBAC7B7I,KAAK,KAAK,CAACkB,MAAM,MAAM,CAAC,EAAE,GAAG2H;aAC9B;YACD,MAAM;gBACJ,GAAG3H,MAAM,IAAI;gBACb,MAAMlB,KAAK,KAAK,CAACkB,MAAM,IAAI,CAAC,IAAI,GAAG2H;gBACnC,KAAK7I,KAAK,KAAK,CAACkB,MAAM,IAAI,CAAC,GAAG,GAAG2H;gBACjC,OAAO7I,KAAK,KAAK,CAACkB,MAAM,IAAI,CAAC,KAAK,GAAG2H;gBACrC,QAAQ7I,KAAK,KAAK,CAACkB,MAAM,IAAI,CAAC,MAAM,GAAG2H;YACzC;QACF;QAEFD,SAAS,CAACT,UAAU,GAAGjH;IACzB;IAEA,OAAO0H;AACT;AAEO,MAAME,qBAAqB;AAQ3B,MAAMC,wBAAwB,CACnCC,SAAS,qBAAqB,EAC9BC;IAEA,MAAMC,MAAMD,AAAclH,WAAdkH,YAA0B,IAAIE,KAAKF,aAAa,IAAIE;IAChE,MAAMC,OAAOF,IAAI,WAAW;IAC5B,MAAMG,QAAQ7B,OAAO0B,IAAI,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACrD,MAAMI,MAAM9B,OAAO0B,IAAI,OAAO,IAAI,QAAQ,CAAC,GAAG;IAC9C,MAAMK,QAAQ/B,OAAO0B,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMM,UAAUhC,OAAO0B,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IACrD,MAAMO,UAAUjC,OAAO0B,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IAErD,MAAMQ,aAAaV,OAChB,OAAO,CAAC,QAAQxB,OAAO4B,OACvB,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,KACd,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,SACd,OAAO,CAAC,MAAMC;IAEjB,OAAO,GAAGC,WAAW,EAAE,EAAEV,OAAO,CAAC,CAAC;AACpC"}
1	+ {"version":3,"file":"common.mjs","sources":["../../src/common.ts"],"sourcesContent":["import type {\n BaseElement,\n DeviceAction,\n ElementTreeNode,\n MidsceneYamlFlowItem,\n PlanningAction,\n Rect,\n Size,\n} from '@/types';\nimport { assert, isPlainObject } from '@midscene/shared/utils';\n\nimport { NodeType } from '@midscene/shared/constants';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { z } from 'zod';\n\n/*\n Expand the search area to at least 400 x 400 pixels\n \n Step 1: Extend 100px on each side (top, right, bottom, left)\n * - If the element is near a boundary, expansion on that side will be limited\n * - No compensation is made for boundary limitations (this is intentional)\n \n Step 2: Ensure the area is at least 400x400 pixels\n * - Scale up proportionally from the center if needed\n * - Final result is clamped to screen boundaries\n /\nexport function expandSearchArea(rect: Rect, screenSize: Size): Rect {\n const minArea = 400 400;\n const expandSize = 100;\n\n // Step 1: Extend each side by expandSize (100px), clamped to screen boundaries\n // Note: If element is near boundary, actual expansion may be less than 100px on that side\n const expandedLeft = Math.max(rect.left - expandSize, 0);\n const expandedTop = Math.max(rect.top - expandSize, 0);\n\n const expandRect = {\n left: expandedLeft,\n top: expandedTop,\n width: Math.min(\n rect.left - expandedLeft + rect.width + expandSize,\n screenSize.width - expandedLeft,\n ),\n height: Math.min(\n rect.top - expandedTop + rect.height + expandSize,\n screenSize.height - expandedTop,\n ),\n };\n\n // Step 2: Check if area is already >= 400x400\n const currentArea = expandRect.width * expandRect.height;\n\n if (currentArea >= minArea) {\n return expandRect;\n }\n\n // Step 2: Scale up from center to reach minimum 400x400 area\n const centerX = expandRect.left + expandRect.width / 2;\n const centerY = expandRect.top + expandRect.height / 2;\n\n // Calculate scale factor needed to reach minimum area\n const scaleFactor = Math.sqrt(minArea / currentArea);\n const newWidth = Math.round(expandRect.width * scaleFactor);\n const newHeight = Math.round(expandRect.height * scaleFactor);\n\n // Calculate new position based on center point\n const newLeft = Math.round(centerX - newWidth / 2);\n const newTop = Math.round(centerY - newHeight / 2);\n\n // Clamp to screen boundaries\n const left = Math.max(newLeft, 0);\n const top = Math.max(newTop, 0);\n\n return {\n left,\n top,\n width: Math.min(newWidth, screenSize.width - left),\n height: Math.min(newHeight, screenSize.height - top),\n };\n}\n\nexport async function markupImageForLLM(\n screenshotBase64: string,\n tree: ElementTreeNode<BaseElement>,\n size: Size,\n) {\n const elementsInfo = treeToList(tree);\n const elementsPositionInfoWithoutText = elementsInfo!.filter(\n (elementInfo) => {\n if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n return false;\n }\n return true;\n },\n );\n\n const imagePayload = await compositeElementInfoImg({\n inputImgBase64: screenshotBase64,\n elementsPositionInfo: elementsPositionInfoWithoutText,\n size,\n });\n return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n plans: PlanningAction[],\n actionSpace: DeviceAction<any>[],\n): MidsceneYamlFlowItem[] {\n const flow: MidsceneYamlFlowItem[] = [];\n\n for (const plan of plans) {\n const verb = plan.type;\n\n const action = actionSpace.find((action) => action.name === verb);\n if (!action) {\n console.warn(\n `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n );\n continue;\n }\n\n const flowKey = action.interfaceAlias \|\| verb;\n const flowParam = action.paramSchema\n ? dumpActionParam(plan.param \|\| {}, action.paramSchema)\n : {};\n\n // For actions whose param is a single string field (e.g. Launch/Terminate's\n // `uri`, RunAdbShell's `command`), inline the value on the flowKey. Writing\n // `{ terminate: '', uri: '...' }` makes the YAML player treat the empty\n // string as the param and drop the sibling `uri`, so cache replay would\n // call the action with an empty argument.\n const shortcutField =\n action.name === 'Launch' \|\| action.interfaceAlias === 'launch'\n ? 'uri'\n : action.name === 'Terminate' \|\| action.interfaceAlias === 'terminate'\n ? 'uri'\n : action.name === 'RunAdbShell' \|\|\n action.interfaceAlias === 'runAdbShell' \|\|\n action.name === 'RunHdcShell' \|\|\n action.interfaceAlias === 'runHdcShell'\n ? 'command'\n : undefined;\n const shortcutKeys = shortcutField ? Object.keys(flowParam) : [];\n const canInlineShortcut =\n shortcutField &&\n shortcutKeys.length === 1 &&\n shortcutKeys[0] === shortcutField &&\n typeof flowParam[shortcutField] === 'string';\n\n const flowItem: MidsceneYamlFlowItem = canInlineShortcut\n ? { [flowKey]: flowParam[shortcutField as string] }\n : { [flowKey]: '', ...flowParam };\n\n flow.push(flowItem);\n }\n\n return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n left: z.number(),\n top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n width: z.number(),\n height: z.number(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n z.object({\n zoom: z.number().optional(),\n }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n images: z\n .array(\n z.object({\n name: z.string(),\n url: z.string(),\n }),\n )\n .optional(),\n convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n z.string(),\n z\n .object({\n prompt: z.string(),\n })\n .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n .object({\n prompt: TUserPromptSchema,\n deepLocate: z.boolean().optional(),\n deepThink: z\n .boolean()\n .optional()\n .describe('@deprecated Use `deepLocate` instead.'),\n cacheable: z.boolean().optional(),\n xpath: z.union([z.string(), z.boolean()]).optional(),\n })\n .passthrough();\n\n/*\n Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n /\nexport const getMidsceneLocationSchema = () => {\n return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n // Handle optional fields by getting the inner type\n let actualField = field;\n if (actualField._def?.typeName === 'ZodOptional') {\n actualField = actualField._def.innerType;\n }\n\n // Check if this is a ZodObject\n if (actualField._def?.typeName === 'ZodObject') {\n const shape = actualField._def.shape();\n\n // Method 1: Check for the location field flag (for result schema)\n if (locateFieldFlagName in shape) {\n return true;\n }\n\n // Method 2: Check if it's the input schema by checking for 'prompt' field\n // Input schema has 'prompt' as a required field\n if ('prompt' in shape && shape.prompt) {\n return true;\n }\n }\n\n return false;\n};\n\nconst formatPromptWithImages = (\n promptObj: Exclude<TUserPrompt, string>,\n): string => {\n let promptString = promptObj.prompt;\n if (Array.isArray(promptObj.images) && promptObj.images.length > 0) {\n const imageCount = promptObj.images.length;\n promptString += ` (with ${imageCount} image${imageCount > 1 ? 's' : ''})`;\n }\n return promptString;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n assert(\n ifMidsceneLocatorField(field),\n 'field is not a midscene locator field',\n );\n\n // If field is a string, return it directly\n if (typeof field === 'string') {\n return field;\n }\n\n // If field is an object with prompt property\n if (field && typeof field === 'object' && field.prompt) {\n // If prompt is a string, return it directly\n if (typeof field.prompt === 'string') {\n return field.prompt;\n }\n // If prompt is a TUserPrompt object, extract the prompt string\n if (typeof field.prompt === 'object' && field.prompt.prompt) {\n return formatPromptWithImages(field.prompt);\n }\n }\n\n // Fallback: try to convert to string\n return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n zodType?: z.ZodType<any>,\n requiredOnly?: boolean,\n): string[] => {\n if (!zodType) {\n return [];\n }\n\n // Check if this is a ZodObject by checking if it has a shape property\n const zodObject = zodType as any;\n if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n const keys = Object.keys(zodObject.shape);\n return keys.filter((key) => {\n const field = zodObject.shape[key];\n if (!ifMidsceneLocatorField(field)) {\n return false;\n }\n\n // If requiredOnly is true, filter out optional fields\n if (requiredOnly) {\n return field._def?.typeName !== 'ZodOptional';\n }\n\n return true;\n });\n }\n\n // For other ZodType instances, we can't extract field names\n return [];\n};\n\nexport const dumpActionParam = (\n jsonObject: Record<string, any>,\n zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n // Prevent spreading strings into {0: 'c', 1: 'o', ...}\n if (!isPlainObject(jsonObject)) {\n return {};\n }\n\n const locatorFields = findAllMidsceneLocatorField(zodSchema);\n const result = { ...jsonObject };\n\n for (const fieldName of locatorFields) {\n const fieldValue = result[fieldName];\n if (fieldValue) {\n // If it's already a string, keep it as is\n if (typeof fieldValue === 'string') {\n result[fieldName] = fieldValue;\n } else if (typeof fieldValue === 'object') {\n // Check if this field is actually a MidsceneLocationType object\n if (fieldValue.prompt) {\n // If prompt is a string, use it directly\n if (typeof fieldValue.prompt === 'string') {\n result[fieldName] = fieldValue.prompt;\n } else if (\n typeof fieldValue.prompt === 'object' &&\n fieldValue.prompt.prompt\n ) {\n // If prompt is a TUserPrompt object, extract the prompt string\n result[fieldName] = formatPromptWithImages(fieldValue.prompt);\n }\n }\n }\n }\n }\n\n return result;\n};\n\n/\n Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n \n Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n \n When shrunkShotToLogicalRatio is provided and !== 1, coordinates in locate fields\n * are transformed from screenshot space to logical space.\n /\nexport const parseActionParam = (\n rawParam: Record<string, any> \| undefined,\n zodSchema?: z.ZodType<any>,\n options?: { shrunkShotToLogicalRatio?: number },\n): Record<string, any> \| undefined => {\n // If no schema is provided, return undefined (action takes no parameters)\n if (!zodSchema) {\n return undefined;\n }\n\n // Handle undefined or null rawParam by providing an empty object\n const param = rawParam ?? {};\n\n // Find all locate fields in the schema\n const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n // If there are no locate fields, just do normal validation\n if (locateFields.length === 0) {\n return zodSchema.parse(param);\n }\n\n // Extract locate field values to restore later\n const locateFieldValues: Record<string, any> = {};\n for (const fieldName of locateFields) {\n if (fieldName in param) {\n locateFieldValues[fieldName] = param[fieldName];\n }\n }\n\n // Build params for validation - skip locate fields and use dummy values\n const paramsForValidation: Record<string, any> = {};\n for (const key in param) {\n if (locateFields.includes(key)) {\n // Use dummy value to satisfy schema validation\n paramsForValidation[key] = { prompt: '_dummy_' };\n } else {\n paramsForValidation[key] = param[key];\n }\n }\n\n // Validate with dummy locate values\n const validated = zodSchema.parse(paramsForValidation);\n\n // Restore the actual locate field values (unvalidated, as per business requirement),\n // and transform coordinates from screenshot space to logical space if needed\n const ratio = options?.shrunkShotToLogicalRatio;\n for (const fieldName in locateFieldValues) {\n let value = locateFieldValues[fieldName];\n if (\n ratio !== undefined &&\n ratio !== 1 &&\n value &&\n typeof value === 'object' &&\n value.center &&\n value.rect\n ) {\n value = {\n ...value,\n center: [\n Math.round(value.center[0] / ratio),\n Math.round(value.center[1] / ratio),\n ],\n rect: {\n ...value.rect,\n left: Math.round(value.rect.left / ratio),\n top: Math.round(value.rect.top / ratio),\n width: Math.round(value.rect.width / ratio),\n height: Math.round(value.rect.height / ratio),\n },\n };\n }\n validated[fieldName] = value;\n }\n\n return validated;\n};\n\nexport const finalizeActionName = 'Finalize';\n\n/\n Get a readable time string for a given timestamp or the current time\n * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'\n * @param timestamp - Optional timestamp in milliseconds. If not provided, uses current system time.\n * @returns A formatted time string with format label\n */\nexport const getReadableTimeString = (\n format = 'YYYY-MM-DD HH:mm:ss',\n timestamp?: number,\n): string => {\n const now = timestamp !== undefined ? new Date(timestamp) : new Date();\n const year = now.getFullYear();\n const month = String(now.getMonth() + 1).padStart(2, '0');\n const day = String(now.getDate()).padStart(2, '0');\n const hours = String(now.getHours()).padStart(2, '0');\n const minutes = String(now.getMinutes()).padStart(2, '0');\n const seconds = String(now.getSeconds()).padStart(2, '0');\n\n const timeString = format\n .replace('YYYY', String(year))\n .replace('MM', month)\n .replace('DD', day)\n .replace('HH', hours)\n .replace('mm', minutes)\n .replace('ss', seconds);\n\n return `${timeString} (${format})`;\n};\n"],"names":["expandSearchArea","rect","screenSize","minArea","expandSize","expandedLeft","Math","expandedTop","expandRect","currentArea","centerX","centerY","scaleFactor","newWidth","newHeight","newLeft","newTop","left","top","markupImageForLLM","screenshotBase64","tree","size","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","flow","plan","verb","action","console","flowKey","flowParam","dumpActionParam","shortcutField","undefined","shortcutKeys","Object","canInlineShortcut","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","formatPromptWithImages","promptObj","promptString","Array","imageCount","dumpMidsceneLocatorField","assert","String","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","key","jsonObject","zodSchema","isPlainObject","locatorFields","result","fieldName","fieldValue","parseActionParam","rawParam","options","param","locateFields","locateFieldValues","paramsForValidation","validated","ratio","value","finalizeActionName","getReadableTimeString","format","timestamp","now","Date","year","month","day","hours","minutes","seconds","timeString"],"mappings":";;;;;AA2BO,SAASA,iBAAiBC,IAAU,EAAEC,UAAgB;IAC3D,MAAMC,UAAU;IAChB,MAAMC,aAAa;IAInB,MAAMC,eAAeC,KAAK,GAAG,CAACL,KAAK,IAAI,GAAGG,YAAY;IACtD,MAAMG,cAAcD,KAAK,GAAG,CAACL,KAAK,GAAG,GAAGG,YAAY;IAEpD,MAAMI,aAAa;QACjB,MAAMH;QACN,KAAKE;QACL,OAAOD,KAAK,GAAG,CACbL,KAAK,IAAI,GAAGI,eAAeJ,KAAK,KAAK,GAAGG,YACxCF,WAAW,KAAK,GAAGG;QAErB,QAAQC,KAAK,GAAG,CACdL,KAAK,GAAG,GAAGM,cAAcN,KAAK,MAAM,GAAGG,YACvCF,WAAW,MAAM,GAAGK;IAExB;IAGA,MAAME,cAAcD,WAAW,KAAK,GAAGA,WAAW,MAAM;IAExD,IAAIC,eAAeN,SACjB,OAAOK;IAIT,MAAME,UAAUF,WAAW,IAAI,GAAGA,WAAW,KAAK,GAAG;IACrD,MAAMG,UAAUH,WAAW,GAAG,GAAGA,WAAW,MAAM,GAAG;IAGrD,MAAMI,cAAcN,KAAK,IAAI,CAACH,UAAUM;IACxC,MAAMI,WAAWP,KAAK,KAAK,CAACE,WAAW,KAAK,GAAGI;IAC/C,MAAME,YAAYR,KAAK,KAAK,CAACE,WAAW,MAAM,GAAGI;IAGjD,MAAMG,UAAUT,KAAK,KAAK,CAACI,UAAUG,WAAW;IAChD,MAAMG,SAASV,KAAK,KAAK,CAACK,UAAUG,YAAY;IAGhD,MAAMG,OAAOX,KAAK,GAAG,CAACS,SAAS;IAC/B,MAAMG,MAAMZ,KAAK,GAAG,CAACU,QAAQ;IAE7B,OAAO;QACLC;QACAC;QACA,OAAOZ,KAAK,GAAG,CAACO,UAAUX,WAAW,KAAK,GAAGe;QAC7C,QAAQX,KAAK,GAAG,CAACQ,WAAWZ,WAAW,MAAM,GAAGgB;IAClD;AACF;AAEO,eAAeC,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCC,IAAU;IAEV,MAAMC,eAAeC,WAAWH;IAChC,MAAMI,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBT;QAChB,sBAAsBK;QACtBH;IACF;IACA,OAAOM;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC;IAEhC,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQH,MAAO;QACxB,MAAMI,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASJ,YAAY,IAAI,CAAC,CAACI,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEF,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMG,UAAUF,OAAO,cAAc,IAAID;QACzC,MAAMI,YAAYH,OAAO,WAAW,GAChCI,gBAAgBN,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAOL,MAAMK,gBACJL,AAAgB,aAAhBA,OAAO,IAAI,IAAiBA,AAA0B,aAA1BA,OAAO,cAAc,GAC7C,QACAA,AAAgB,gBAAhBA,OAAO,IAAI,IAAoBA,AAA0B,gBAA1BA,OAAO,cAAc,GAClD,QACAA,AAAgB,kBAAhBA,OAAO,IAAI,IACTA,AAA0B,kBAA1BA,OAAO,cAAc,IACrBA,AAAgB,kBAAhBA,OAAO,IAAI,IACXA,AAA0B,kBAA1BA,OAAO,cAAc,GACrB,YACAM;QACV,MAAMC,eAAeF,gBAAgBG,OAAO,IAAI,CAACL,aAAa,EAAE;QAChE,MAAMM,oBACJJ,iBACAE,AAAwB,MAAxBA,aAAa,MAAM,IACnBA,YAAY,CAAC,EAAE,KAAKF,iBACpB,AAAoC,YAApC,OAAOF,SAAS,CAACE,cAAc;QAEjC,MAAMK,WAAiCD,oBACnC;YAAE,CAACP,QAAQ,EAAEC,SAAS,CAACE,cAAwB;QAAC,IAChD;YAAE,CAACH,QAAQ,EAAE;YAAI,GAAGC,SAAS;QAAC;QAElCN,KAAK,IAAI,CAACa;IACZ;IAEA,OAAOb;AACT;AAGO,MAAMc,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;AAClB;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBN,EAAAA,MACrB,CAAC;IACN,QAAQI;IACR,YAAYJ,EAAE,OAAO,GAAG,QAAQ;IAChC,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,OAAO;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAMP,MAAMO,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIL,uBAAuBM,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEA,MAAMC,yBAAyB,CAC7BC;IAEA,IAAIC,eAAeD,UAAU,MAAM;IACnC,IAAIE,MAAM,OAAO,CAACF,UAAU,MAAM,KAAKA,UAAU,MAAM,CAAC,MAAM,GAAG,GAAG;QAClE,MAAMG,aAAaH,UAAU,MAAM,CAAC,MAAM;QAC1CC,gBAAgB,CAAC,OAAO,EAAEE,WAAW,MAAM,EAAEA,aAAa,IAAI,MAAM,GAAG,CAAC,CAAC;IAC3E;IACA,OAAOF;AACT;AAEO,MAAMG,2BAA2B,CAACR;IACvCS,OACEV,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOG,uBAAuBH,MAAM,MAAM;IAE9C;IAGA,OAAOU,OAAOV;AAChB;AAEO,MAAMW,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAO5B,OAAO,IAAI,CAAC2B,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACC;YAClB,MAAMhB,QAAQc,UAAU,KAAK,CAACE,IAAI;YAClC,IAAI,CAACjB,uBAAuBC,QAC1B,OAAO;YAIT,IAAIa,cACF,OAAOb,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMjB,kBAAkB,CAC7BkC,YACAC;IAGA,IAAI,CAACC,cAAcF,aACjB,OAAO,CAAC;IAGV,MAAMG,gBAAgBT,4BAA4BO;IAClD,MAAMG,SAAS;QAAE,GAAGJ,UAAU;IAAC;IAE/B,KAAK,MAAMK,aAAaF,cAAe;QACrC,MAAMG,aAAaF,MAAM,CAACC,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTF,MAAM,CAACC,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BF,MAAM,CAACC,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBF,MAAM,CAACC,UAAU,GAAGnB,uBAAuBoB,WAAW,MAAM;gBAC9D;YACF;QACF;IAEJ;IAEA,OAAOF;AACT;AAaO,MAAMG,mBAAmB,CAC9BC,UACAP,WACAQ;IAGA,IAAI,CAACR,WACH;IAIF,MAAMS,QAAQF,YAAY,CAAC;IAG3B,MAAMG,eAAejB,4BAA4BO;IAGjD,IAAIU,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOV,UAAU,KAAK,CAACS;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMd,OAAOW,MAChB,IAAIC,aAAa,QAAQ,CAACZ,MAExBc,mBAAmB,CAACd,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/Cc,mBAAmB,CAACd,IAAI,GAAGW,KAAK,CAACX,IAAI;IAKzC,MAAMe,YAAYb,UAAU,KAAK,CAACY;IAIlC,MAAME,QAAQN,SAAS;IACvB,IAAK,MAAMJ,aAAaO,kBAAmB;QACzC,IAAII,QAAQJ,iBAAiB,CAACP,UAAU;QACxC,IACEU,AAAU/C,WAAV+C,SACAA,AAAU,MAAVA,SACAC,SACA,AAAiB,YAAjB,OAAOA,SACPA,MAAM,MAAM,IACZA,MAAM,IAAI,EAEVA,QAAQ;YACN,GAAGA,KAAK;YACR,QAAQ;gBACNpF,KAAK,KAAK,CAACoF,MAAM,MAAM,CAAC,EAAE,GAAGD;gBAC7BnF,KAAK,KAAK,CAACoF,MAAM,MAAM,CAAC,EAAE,GAAGD;aAC9B;YACD,MAAM;gBACJ,GAAGC,MAAM,IAAI;gBACb,MAAMpF,KAAK,KAAK,CAACoF,MAAM,IAAI,CAAC,IAAI,GAAGD;gBACnC,KAAKnF,KAAK,KAAK,CAACoF,MAAM,IAAI,CAAC,GAAG,GAAGD;gBACjC,OAAOnF,KAAK,KAAK,CAACoF,MAAM,IAAI,CAAC,KAAK,GAAGD;gBACrC,QAAQnF,KAAK,KAAK,CAACoF,MAAM,IAAI,CAAC,MAAM,GAAGD;YACzC;QACF;QAEFD,SAAS,CAACT,UAAU,GAAGW;IACzB;IAEA,OAAOF;AACT;AAEO,MAAMG,qBAAqB;AAQ3B,MAAMC,wBAAwB,CACnCC,SAAS,qBAAqB,EAC9BC;IAEA,MAAMC,MAAMD,AAAcpD,WAAdoD,YAA0B,IAAIE,KAAKF,aAAa,IAAIE;IAChE,MAAMC,OAAOF,IAAI,WAAW;IAC5B,MAAMG,QAAQ/B,OAAO4B,IAAI,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACrD,MAAMI,MAAMhC,OAAO4B,IAAI,OAAO,IAAI,QAAQ,CAAC,GAAG;IAC9C,MAAMK,QAAQjC,OAAO4B,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMM,UAAUlC,OAAO4B,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IACrD,MAAMO,UAAUnC,OAAO4B,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IAErD,MAAMQ,aAAaV,OAChB,OAAO,CAAC,QAAQ1B,OAAO8B,OACvB,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,KACd,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,SACd,OAAO,CAAC,MAAMC;IAEjB,OAAO,GAAGC,WAAW,EAAE,EAAEV,OAAO,CAAC,CAAC;AACpC"}

package/dist/es/device/index.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"device/index.mjs","sources":["../../../src/device/index.ts"],"sourcesContent":["import { getMidsceneLocationSchema } from '@/common';\nimport type {\n ActionScrollParam,\n DeviceAction,\n LocateResultElement,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport interface FileChooserHandler {\n accept(files: string[]): Promise<void>;\n}\n\nexport interface MjpegStreamFrame {\n /** Raw base64-encoded image bytes WITHOUT a `data:image/...;base64,` prefix. /\n data: string;\n contentType?: string;\n}\n\nexport interface MjpegStreamHandle {\n stop(): void \| Promise<void>;\n}\n\nexport interface MjpegStreamOptions {\n signal?: AbortSignal;\n onFrame(frame: MjpegStreamFrame): void;\n onError?(error: unknown): void;\n}\n\n/* A point in device-pixel coordinates on the screen. /\nexport interface PointerPoint {\n x: number;\n y: number;\n}\n\nexport interface PointerInputPrimitives {\n tap(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n doubleClick?(p: PointerPoint): Promise<void>;\n rightClick?(p: PointerPoint): Promise<void>;\n hover?(p: PointerPoint): Promise<void>;\n longPress?(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n dragAndDrop?(from: PointerPoint, to: PointerPoint): Promise<void>;\n}\n\nexport interface TouchInputPrimitives {\n swipe(\n start: PointerPoint,\n end: PointerPoint,\n opts?: { duration?: number; repeat?: number },\n ): Promise<void>;\n pinch?(\n center: PointerPoint,\n opts: { startDistance: number; endDistance: number; duration: number },\n ): Promise<void>;\n}\n\nexport interface KeyboardInputPrimitives {\n keyboardPress(keyName: string, opts?: { target?: unknown }): Promise<void>;\n cursorMove?(direction: 'left' \| 'right', times?: number): Promise<void>;\n typeText(\n value: string,\n opts?: {\n autoDismissKeyboard?: boolean;\n keyboardDismissStrategy?: 'esc-first' \| 'back-first';\n target?: unknown;\n replace?: boolean;\n focusOnly?: boolean;\n },\n ): Promise<void>;\n clearInput(target?: unknown): Promise<void>;\n}\n\nexport interface ScrollInputPrimitives {\n scroll(param: ActionScrollParam): Promise<void>;\n}\n\nexport interface SystemInputPrimitives {\n backButton?(): Promise<void>;\n homeButton?(): Promise<void>;\n recentAppsButton?(): Promise<void>;\n}\n\nexport interface InputPrimitives {\n pointer?: PointerInputPrimitives;\n keyboard?: KeyboardInputPrimitives;\n touch?: TouchInputPrimitives;\n scroll?: ScrollInputPrimitives;\n system?: SystemInputPrimitives;\n}\n\nexport interface MobileInputPrimitives extends InputPrimitives {\n pointer: PointerInputPrimitives & {\n doubleClick(p: PointerPoint): Promise<void>;\n longPress(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n };\n keyboard: KeyboardInputPrimitives;\n touch: TouchInputPrimitives;\n}\n\nexport interface BrowserInputPrimitives extends InputPrimitives {\n pointer: PointerInputPrimitives & {\n doubleClick(p: PointerPoint): Promise<void>;\n rightClick(p: PointerPoint): Promise<void>;\n hover(p: PointerPoint): Promise<void>;\n dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n longPress(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n };\n keyboard: KeyboardInputPrimitives;\n scroll: ScrollInputPrimitives;\n touch: TouchInputPrimitives;\n}\n\nexport interface ComputerInputPrimitives extends InputPrimitives {\n pointer: PointerInputPrimitives & {\n doubleClick(p: PointerPoint): Promise<void>;\n rightClick(p: PointerPoint): Promise<void>;\n hover(p: PointerPoint): Promise<void>;\n dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n };\n keyboard: KeyboardInputPrimitives;\n scroll: ScrollInputPrimitives;\n}\n\nexport abstract class AbstractInterface {\n abstract interfaceType: string;\n\n abstract screenshotBase64(): Promise<string>;\n abstract size(): Promise<Size>;\n abstract actionSpace(): DeviceAction[];\n\n abstract cacheFeatureForPoint?(\n center: [number, number],\n options?: {\n targetDescription?: string;\n modelConfig?: IModelConfig;\n },\n ): Promise<ElementCacheFeature>;\n abstract rectMatchesCacheFeature?(\n feature: ElementCacheFeature,\n ): Promise<Rect>;\n\n abstract destroy?(): Promise<void>;\n\n abstract describe?(): string;\n abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n // for web only\n registerFileChooserListener?(\n handler: (chooser: FileChooserHandler) => Promise<void>,\n ): Promise<{ dispose: () => void; getError: () => Error \| undefined }>;\n\n // @deprecated do NOT extend this method\n abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n // @deprecated do NOT extend this method\n abstract url?: () => string \| Promise<string>;\n\n // @deprecated do NOT extend this method\n abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n /\n Get the current device-local time as a formatted string.\n * Prefer this for user-visible time because timestamps alone do not preserve\n * the target device's timezone when formatted on the host machine.\n /\n getDeviceLocalTimeString?(format?: string): Promise<string>;\n\n /* URL of native MJPEG stream for real-time screen preview (e.g. WDA MJPEG server) /\n mjpegStreamUrl?: string;\n\n /\n Optional in-process MJPEG frame producer. Implementations can push raw\n * base64 frames here when there is no standalone native MJPEG URL, e.g.\n * Chromium CDP Page.startScreencast for web previews.\n /\n startMjpegStream?(\n options: MjpegStreamOptions,\n ): MjpegStreamHandle \| undefined \| Promise<MjpegStreamHandle \| undefined>;\n\n /\n Optional hook used after keyboard-only actions to force a fresh frame on\n * the active MJPEG stream. Implementations should be a no-op when no stream\n * is active.\n /\n flushPendingVisualUpdate?(): Promise<void>;\n\n /\n Optional navigation state probe for browser-like interfaces, used to drive\n * loading indicators in playground UIs. Returning `undefined` means the\n * interface does not expose this concept.\n /\n navigationState?(): Promise<{ isLoading: boolean }>;\n\n /\n Low-level device input surface. Platform implementations expose transport\n * primitives here; higher-level AI actions and manual pointer dispatch should\n * adapt to this instead of duplicating platform gesture logic.\n */\n inputPrimitives?: InputPrimitives;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\n// TReturn allows specifying the return type of the action\nexport const defineAction = <\n TSchema extends z.ZodType \| undefined = undefined,\n TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined,\n TReturn = any,\n>(\n config: {\n name: string;\n description: string;\n interfaceAlias?: string;\n paramSchema?: TSchema;\n call: (param: TRuntime) => Promise<TReturn> \| TReturn;\n } & Partial<\n Omit<\n DeviceAction<TRuntime, TReturn>,\n 'name' \| 'description' \| 'interfaceAlias' \| 'paramSchema' \| 'call'\n >\n >,\n): DeviceAction<TRuntime, TReturn> => {\n return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\nfunction pointFromLocate(\n locate: LocateResultElement \| undefined,\n missingMessage: string,\n): PointerPoint {\n if (!locate) {\n throw new Error(missingMessage);\n }\n return { x: locate.center[0], y: locate.center[1] };\n}\n\nfunction defineLocatedPointAction<\n TSchema extends z.ZodType,\n TParam extends { locate: LocateResultElement },\n>(config: {\n name: string;\n description: string;\n interfaceAlias?: string;\n paramSchema: TSchema;\n sample: DeviceAction<TParam>['sample'];\n missingLocateMessage: string;\n call: (point: PointerPoint, param: TParam) => Promise<void>;\n}): DeviceAction<TParam> {\n return defineAction<TSchema, TParam>({\n name: config.name,\n description: config.description,\n interfaceAlias: config.interfaceAlias,\n paramSchema: config.paramSchema,\n sample: config.sample,\n call: async (param) => {\n await config.call(\n pointFromLocate(param.locate, config.missingLocateMessage),\n param,\n );\n },\n });\n}\n\n// Tap\nexport const actionTapParamSchema = z.object({\n locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\nexport type ActionTapParam = {\n locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n tap: PointerInputPrimitives['tap'],\n): DeviceAction<ActionTapParam> => {\n return defineLocatedPointAction<typeof actionTapParamSchema, ActionTapParam>({\n name: 'Tap',\n description: 'Tap the element',\n interfaceAlias: 'aiTap',\n paramSchema: actionTapParamSchema,\n sample: {\n locate: { prompt: 'the \"Submit\" button' },\n },\n missingLocateMessage: 'Element not found, cannot tap',\n call: async (point) => {\n await tap(point);\n },\n });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n locate: getMidsceneLocationSchema().describe(\n 'The element to be right clicked',\n ),\n});\nexport type ActionRightClickParam = {\n locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n rightClick: NonNullable<PointerInputPrimitives['rightClick']>,\n): DeviceAction<ActionRightClickParam> => {\n return defineLocatedPointAction<\n typeof actionRightClickParamSchema,\n ActionRightClickParam\n >({\n name: 'RightClick',\n description: 'Right click the element',\n interfaceAlias: 'aiRightClick',\n paramSchema: actionRightClickParamSchema,\n sample: {\n locate: { prompt: 'the file icon on the desktop' },\n },\n missingLocateMessage: 'Element not found, cannot right click',\n call: async (point) => {\n await rightClick(point);\n },\n });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n locate: getMidsceneLocationSchema().describe(\n 'The element to be double clicked',\n ),\n});\nexport type ActionDoubleClickParam = {\n locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n doubleClick: NonNullable<PointerInputPrimitives['doubleClick']>,\n): DeviceAction<ActionDoubleClickParam> => {\n return defineLocatedPointAction<\n typeof actionDoubleClickParamSchema,\n ActionDoubleClickParam\n >({\n name: 'DoubleClick',\n description: 'Double click the element',\n interfaceAlias: 'aiDoubleClick',\n paramSchema: actionDoubleClickParamSchema,\n sample: {\n locate: { prompt: 'the folder icon' },\n },\n missingLocateMessage: 'Element not found, cannot double click',\n call: async (point) => {\n await doubleClick(point);\n },\n });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n hover: NonNullable<PointerInputPrimitives['hover']>,\n): DeviceAction<ActionHoverParam> => {\n return defineLocatedPointAction<\n typeof actionHoverParamSchema,\n ActionHoverParam\n >({\n name: 'Hover',\n description: 'Move the mouse to the element',\n interfaceAlias: 'aiHover',\n paramSchema: actionHoverParamSchema,\n sample: {\n locate: { prompt: 'the navigation menu item \"Products\"' },\n },\n missingLocateMessage: 'Element not found, cannot hover',\n call: async (point) => {\n await hover(point);\n },\n });\n};\n\n// Input\nconst inputLocateDescription =\n 'the position of the placeholder or text content in the target input field. If there is no content, locate the center of the input field.';\nexport const actionInputParamSchema = z.object({\n value: z\n .union([z.string(), z.number()])\n .transform((val) => String(val))\n .describe(\n 'The text to input. Provide the final content for replace mode, only the inserted characters for typeOnly mode, or an empty string when using clear mode to remove existing text.',\n ),\n locate: getMidsceneLocationSchema()\n .describe(inputLocateDescription)\n .optional(),\n mode: z\n .enum(['replace', 'clear', 'typeOnly'])\n .default('replace')\n .describe(\n 'Input mode: \"replace\" (default) - clear the field and input the value; \"typeOnly\" - type the value directly without clearing the field first, and should be set explicitly for incremental edits after moving the cursor; \"clear\" - clear the field without inputting new text.',\n ),\n autoDismissKeyboard: z\n .boolean()\n .optional()\n .describe(\n 'If true, the keyboard will be dismissed after the input is completed. Do not set it unless the user asks you to do so.',\n ),\n});\nexport type ActionInputParam = {\n value: string;\n locate?: LocateResultElement;\n mode?: 'replace' \| 'clear' \| 'typeOnly' \| 'append';\n autoDismissKeyboard?: boolean;\n};\n\nexport const defineActionInput = (\n keyboard: KeyboardInputPrimitives,\n): DeviceAction<ActionInputParam> => {\n return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n name: 'Input',\n description: 'Input the value into the element',\n interfaceAlias: 'aiInput',\n paramSchema: actionInputParamSchema,\n sample: {\n value: 'test@example.com',\n locate: { prompt: 'the email input field' },\n },\n call: async (param) => {\n // backward compat: convert deprecated 'append' to 'typeOnly'\n if ((param.mode as string) === 'append') {\n param.mode = 'typeOnly';\n }\n\n if (param.mode === 'clear') {\n await keyboard.clearInput(param.locate);\n return;\n }\n\n if (!param \|\| !param.value) {\n return;\n }\n\n await keyboard.typeText(param.value, {\n target: param.locate,\n replace: param.mode !== 'typeOnly',\n autoDismissKeyboard: param.autoDismissKeyboard,\n });\n },\n });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n locate: getMidsceneLocationSchema()\n .describe('The element to be clicked before pressing the key')\n .optional(),\n keyName: z\n .string()\n .describe(\n \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n ),\n});\nexport type ActionKeyboardPressParam = {\n locate?: LocateResultElement;\n keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n keyboardPress: KeyboardInputPrimitives['keyboardPress'],\n): DeviceAction<ActionKeyboardPressParam> => {\n return defineAction<\n typeof actionKeyboardPressParamSchema,\n ActionKeyboardPressParam\n >({\n name: 'KeyboardPress',\n description:\n 'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n interfaceAlias: 'aiKeyboardPress',\n paramSchema: actionKeyboardPressParamSchema,\n sample: {\n keyName: 'Enter',\n },\n call: async (param) => {\n await keyboardPress(param.keyName, {\n target: param.locate,\n });\n },\n });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n scrollType: z\n .enum([\n 'singleAction',\n 'scrollToBottom',\n 'scrollToTop',\n 'scrollToRight',\n 'scrollToLeft',\n ])\n .default('singleAction')\n .describe(\n 'The scroll behavior: \"singleAction\" for a single scroll action, \"scrollToBottom\" for scrolling all the way to the bottom by rapidly scrolling 5-10 times (skipping intermediate content until reaching the bottom), \"scrollToTop\" for scrolling all the way to the top by rapidly scrolling 5-10 times (skipping intermediate content until reaching the top), \"scrollToRight\" for scrolling all the way to the right by rapidly scrolling multiple times, \"scrollToLeft\" for scrolling all the way to the left by rapidly scrolling multiple times',\n ),\n direction: z\n .enum(['down', 'up', 'right', 'left'])\n .default('down')\n .describe(\n 'The direction to scroll. Only effective when scrollType is \"singleAction\".',\n ),\n distance: z\n .number()\n .nullable()\n .optional()\n .describe('The distance in pixels to scroll'),\n locate: getMidsceneLocationSchema()\n .optional()\n .describe(\n 'Describe the target element to be scrolled on, like \"the table\" or \"the list\" or \"the content area\" or \"the scrollable area\". Do NOT provide a general intent like \"scroll to find some element\"',\n ),\n});\n\nexport const defineActionScroll = (\n scroll: ScrollInputPrimitives['scroll'],\n): DeviceAction<ActionScrollParam> => {\n return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n name: 'Scroll',\n description:\n 'Scroll the page or a scrollable element to browse content. This is the preferred way to scroll on all platforms, including mobile. Supports scrollToBottom/scrollToTop for boundary navigation. Default: direction `down`, scrollType `singleAction`, distance `null`.',\n interfaceAlias: 'aiScroll',\n paramSchema: actionScrollParamSchema,\n sample: {\n direction: 'down',\n scrollType: 'singleAction',\n locate: { prompt: 'the center of the product list area' },\n },\n call: async (param) => {\n await scroll(param);\n },\n });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n from: getMidsceneLocationSchema().describe('The position to be dragged'),\n to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n from: LocateResultElement;\n to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n dragAndDrop: NonNullable<PointerInputPrimitives['dragAndDrop']>,\n): DeviceAction<ActionDragAndDropParam> => {\n return defineAction<\n typeof actionDragAndDropParamSchema,\n ActionDragAndDropParam\n >({\n name: 'DragAndDrop',\n description:\n 'Pick up a specific UI element and move it to a new position (e.g., reorder a card, move a file into a folder, sort list items). The element itself moves with your finger/mouse.',\n interfaceAlias: 'aiDragAndDrop',\n paramSchema: actionDragAndDropParamSchema,\n sample: {\n from: { prompt: 'the \"report.pdf\" file icon' },\n to: { prompt: 'the upload drop zone' },\n },\n call: async (param) => {\n const from = param.from;\n const to = param.to;\n if (!from) {\n throw new Error('missing \"from\" param for drag and drop');\n }\n if (!to) {\n throw new Error('missing \"to\" param for drag and drop');\n }\n await dragAndDrop(\n { x: from.center[0], y: from.center[1] },\n { x: to.center[0], y: to.center[1] },\n );\n },\n });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n locate: getMidsceneLocationSchema().describe(\n 'The element to be long pressed',\n ),\n duration: z\n .number()\n .optional()\n .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n locate: LocateResultElement;\n duration?: number;\n};\nexport const defineActionLongPress = (\n longPress: NonNullable<PointerInputPrimitives['longPress']>,\n): DeviceAction<ActionLongPressParam> => {\n return defineLocatedPointAction<\n typeof ActionLongPressParamSchema,\n ActionLongPressParam\n >({\n name: 'LongPress',\n description: 'Long press the element',\n interfaceAlias: 'aiLongPress',\n paramSchema: ActionLongPressParamSchema,\n sample: {\n locate: { prompt: 'the message bubble' },\n },\n missingLocateMessage: 'LongPress requires an element to be located',\n call: async (point, param) => {\n await longPress(point, { duration: param.duration });\n },\n });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n start: getMidsceneLocationSchema()\n .optional()\n .describe(\n 'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n ),\n direction: z\n .enum(['up', 'down', 'left', 'right'])\n .optional()\n .describe(\n 'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n ),\n distance: z\n .number()\n .optional()\n .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n end: getMidsceneLocationSchema()\n .optional()\n .describe(\n 'Ending point of the swipe gesture (mutually exclusive with distance)',\n ),\n duration: z\n .number()\n .default(300)\n .describe('Duration of the swipe gesture in milliseconds'),\n repeat: z\n .number()\n .optional()\n .describe(\n 'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n ),\n});\n\nexport type ActionSwipeParam = {\n start?: LocateResultElement;\n direction?: 'up' \| 'down' \| 'left' \| 'right';\n distance?: number;\n end?: LocateResultElement;\n duration?: number;\n repeat?: number;\n};\n\nexport function normalizeMobileSwipeParam(\n param: ActionSwipeParam,\n screenSize: { width: number; height: number },\n): {\n startPoint: { x: number; y: number };\n endPoint: { x: number; y: number };\n duration: number;\n repeatCount: number;\n} {\n const { width, height } = screenSize;\n const { start, end } = param;\n\n const startPoint = start\n ? { x: start.center[0], y: start.center[1] }\n : { x: width / 2, y: height / 2 };\n\n let endPoint: { x: number; y: number };\n\n if (end) {\n endPoint = { x: end.center[0], y: end.center[1] };\n } else if (param.distance) {\n const direction = param.direction;\n if (!direction) {\n throw new Error('direction is required for swipe gesture');\n }\n endPoint = {\n x:\n startPoint.x +\n (direction === 'right'\n ? param.distance\n : direction === 'left'\n ? -param.distance\n : 0),\n y:\n startPoint.y +\n (direction === 'down'\n ? param.distance\n : direction === 'up'\n ? -param.distance\n : 0),\n };\n } else {\n throw new Error(\n 'Either end or distance must be specified for swipe gesture',\n );\n }\n\n endPoint.x = Math.max(0, Math.min(endPoint.x, width));\n endPoint.y = Math.max(0, Math.min(endPoint.y, height));\n\n const duration = param.duration ?? 300;\n\n let repeatCount = typeof param.repeat === 'number' ? param.repeat : 1;\n if (repeatCount === 0) {\n repeatCount = 10;\n }\n\n return { startPoint, endPoint, duration, repeatCount };\n}\n\nexport const defineActionSwipe = (config: {\n swipe: TouchInputPrimitives['swipe'];\n size(): Promise<Size>;\n}): DeviceAction<ActionSwipeParam> => {\n return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n name: 'Swipe',\n description:\n 'Perform a touch gesture for interactions beyond regular scrolling (e.g., flip pages in a carousel, dismiss a notification, swipe-to-delete a list item). For regular content scrolling, use Scroll instead. Use \"distance\" + \"direction\" for relative movement, or \"end\" for precise endpoint.',\n paramSchema: ActionSwipeParamSchema,\n sample: {\n start: { prompt: 'center of the notification' },\n end: { prompt: 'upper edge of the screen' },\n },\n call: async (param) => {\n const { startPoint, endPoint, duration, repeatCount } =\n normalizeMobileSwipeParam(param, await config.size());\n for (let i = 0; i < repeatCount; i++) {\n await config.swipe(startPoint, endPoint, { duration });\n }\n },\n });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n locate: getMidsceneLocationSchema()\n .describe('The input field to be cleared')\n .optional(),\n});\nexport type ActionClearInputParam = {\n locate?: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n clearInput: KeyboardInputPrimitives['clearInput'],\n): DeviceAction<ActionClearInputParam> => {\n return defineAction<\n typeof actionClearInputParamSchema,\n ActionClearInputParam\n >({\n name: 'ClearInput',\n description: inputLocateDescription,\n interfaceAlias: 'aiClearInput',\n paramSchema: actionClearInputParamSchema,\n sample: {\n locate: { prompt: 'the search input field' },\n },\n call: async (param) => {\n await clearInput(param.locate);\n },\n });\n};\n\n// CursorMove\nexport const actionCursorMoveParamSchema = z.object({\n direction: z\n .enum(['left', 'right'])\n .describe('The direction to move the cursor'),\n times: z\n .number()\n .int()\n .min(1)\n .default(1)\n .describe(\n 'The number of times to move the cursor in the specified direction',\n ),\n});\nexport type ActionCursorMoveParam = {\n direction: 'left' \| 'right';\n times?: number;\n};\n\nexport const defineActionCursorMove = (config: {\n keyboard: Pick<KeyboardInputPrimitives, 'keyboardPress' \| 'cursorMove'>;\n sleep?(timeMs: number): Promise<void>;\n}): DeviceAction<ActionCursorMoveParam> => {\n return defineAction<\n typeof actionCursorMoveParamSchema,\n ActionCursorMoveParam\n >({\n name: 'CursorMove',\n description:\n 'Move the text cursor (caret) left or right within an input field or text area. Use this to reposition the cursor without selecting text.',\n paramSchema: actionCursorMoveParamSchema,\n sample: {\n direction: 'left',\n times: 3,\n },\n call: async (param) => {\n const times = param.times ?? 1;\n if (config.keyboard.cursorMove) {\n await config.keyboard.cursorMove(param.direction, times);\n return;\n }\n\n const wait =\n config.sleep ??\n ((timeMs: number) =>\n new Promise<void>((resolve) => setTimeout(resolve, timeMs)));\n const arrowKey = param.direction === 'left' ? 'ArrowLeft' : 'ArrowRight';\n for (let i = 0; i < times; i++) {\n await config.keyboard.keyboardPress(arrowKey);\n await wait(100);\n }\n },\n });\n};\n\n// Pinch\nexport const ActionPinchParamSchema = z.object({\n locate: getMidsceneLocationSchema()\n .optional()\n .describe(\n 'The element to pinch on. If not specified, the center of the screen will be used',\n ),\n direction: z\n .enum(['in', 'out'])\n .describe(\n 'Pinch direction. \"in\" = pinch fingers together (zoom out / shrink), \"out\" = spread fingers apart (zoom in / enlarge).',\n ),\n distance: z\n .number()\n .positive()\n .optional()\n .describe(\n 'How far each finger moves in pixels. Defaults to a quarter of the shorter screen dimension.',\n ),\n duration: z\n .number()\n .default(500)\n .optional()\n .describe('Duration of the pinch gesture in milliseconds'),\n});\n\nexport type ActionPinchParam = {\n locate?: LocateResultElement;\n direction: 'in' \| 'out';\n distance?: number;\n duration?: number;\n};\n\nexport const defineActionPinch = (config: {\n pinch: TouchInputPrimitives['pinch'];\n size(): Promise<Size>;\n}): DeviceAction<ActionPinchParam> \| undefined => {\n if (!config.pinch) {\n return undefined;\n }\n\n return defineAction<typeof ActionPinchParamSchema, ActionPinchParam>({\n name: 'Pinch',\n description:\n 'Perform a two-finger pinch gesture. Use direction \"in\" to pinch fingers together (zoom out), or \"out\" to spread fingers apart (zoom in). Optionally specify distance for how far each finger moves.',\n interfaceAlias: 'aiPinch',\n paramSchema: ActionPinchParamSchema,\n sample: {\n locate: { prompt: 'the map area' },\n direction: 'out',\n distance: 200,\n },\n call: async (param) => {\n const { centerX, centerY, startDistance, endDistance, duration } =\n normalizePinchParam(param, await config.size());\n await config.pinch?.(\n { x: centerX, y: centerY },\n { startDistance, endDistance, duration },\n );\n },\n });\n};\n\nexport function normalizePinchParam(\n param: ActionPinchParam,\n screenSize: { width: number; height: number },\n): {\n centerX: number;\n centerY: number;\n startDistance: number;\n endDistance: number;\n duration: number;\n} {\n const { width, height } = screenSize;\n const element = param.locate;\n const centerX = element\n ? Math.round(element.center[0])\n : Math.round(width / 2);\n const centerY = element\n ? Math.round(element.center[1])\n : Math.round(height / 2);\n const duration = param.duration ?? 500;\n\n const baseDistance = Math.round(Math.min(width, height) / 4);\n const fingerDistance = param.distance ?? baseDistance;\n\n const startDistance = baseDistance;\n const endDistance =\n param.direction === 'out'\n ? baseDistance + fingerDistance\n : Math.max(10, baseDistance - fingerDistance);\n\n return { centerX, centerY, startDistance, endDistance, duration };\n}\n\nexport interface MobileInputActionContext {\n input: MobileInputPrimitives;\n size(): Promise<Size>;\n sleep?(timeMs: number): Promise<void>;\n getDefaultAutoDismissKeyboard?(): boolean \| undefined;\n systemActions?: SystemInputActionOptions;\n}\n\nexport interface SystemInputActionConfig {\n name: string;\n description: string;\n interfaceAlias?: string;\n delayBeforeRunner?: number;\n delayAfterRunner?: number;\n}\n\nexport interface SystemInputActionOptions {\n backButton?: SystemInputActionConfig;\n homeButton?: SystemInputActionConfig;\n recentAppsButton?: SystemInputActionConfig;\n}\n\nexport interface InputPrimitiveActionOptions {\n size?: () => Promise<Size>;\n sleep?: (timeMs: number) => Promise<void>;\n includeSwipe?: boolean;\n includePinch?: boolean;\n systemActions?: SystemInputActionOptions;\n}\n\nfunction defineSystemInputAction(\n config: SystemInputActionConfig,\n call: () => Promise<void>,\n): DeviceAction<undefined, void> {\n return defineAction<undefined, undefined, void>({\n name: config.name,\n description: config.description,\n interfaceAlias: config.interfaceAlias,\n delayBeforeRunner: config.delayBeforeRunner,\n delayAfterRunner: config.delayAfterRunner,\n call,\n });\n}\n\nexport function defineActionsFromInputPrimitives(\n input: InputPrimitives,\n options: InputPrimitiveActionOptions = {},\n): DeviceAction<any>[] {\n const actions: Array<DeviceAction<any> \| undefined> = [];\n const { pointer, keyboard, scroll, touch, system } = input;\n\n if (pointer) {\n actions.push(defineActionTap(pointer.tap));\n if (pointer.doubleClick) {\n actions.push(defineActionDoubleClick(pointer.doubleClick));\n }\n if (pointer.rightClick) {\n actions.push(defineActionRightClick(pointer.rightClick));\n }\n if (pointer.hover) {\n actions.push(defineActionHover(pointer.hover));\n }\n if (pointer.dragAndDrop) {\n actions.push(defineActionDragAndDrop(pointer.dragAndDrop));\n }\n if (pointer.longPress) {\n actions.push(defineActionLongPress(pointer.longPress));\n }\n }\n\n if (keyboard) {\n actions.push(\n defineActionInput(keyboard),\n defineActionClearInput(keyboard.clearInput),\n defineActionKeyboardPress(keyboard.keyboardPress),\n defineActionCursorMove({ keyboard, sleep: options.sleep }),\n );\n }\n\n if (scroll) {\n actions.push(defineActionScroll(scroll.scroll));\n }\n\n if (touch?.swipe && options.size && options.includeSwipe !== false) {\n actions.push(defineActionSwipe({ swipe: touch.swipe, size: options.size }));\n }\n\n if (touch?.pinch && options.size && options.includePinch !== false) {\n actions.push(defineActionPinch({ pinch: touch.pinch, size: options.size }));\n }\n\n if (system && options.systemActions) {\n const { systemActions } = options;\n if (system.backButton && systemActions.backButton) {\n actions.push(\n defineSystemInputAction(systemActions.backButton, system.backButton),\n );\n }\n if (system.homeButton && systemActions.homeButton) {\n actions.push(\n defineSystemInputAction(systemActions.homeButton, system.homeButton),\n );\n }\n if (system.recentAppsButton && systemActions.recentAppsButton) {\n actions.push(\n defineSystemInputAction(\n systemActions.recentAppsButton,\n system.recentAppsButton,\n ),\n );\n }\n }\n\n return actions.filter((action): action is DeviceAction<any> =>\n Boolean(action),\n );\n}\n\nexport function createDefaultMobileActions(\n context: MobileInputActionContext,\n): DeviceAction<any>[] {\n return defineActionsFromInputPrimitives(context.input, {\n size: context.size,\n sleep: context.sleep,\n systemActions: context.systemActions,\n });\n}\n\n// Sleep\nexport const ActionSleepParamSchema = z.object({\n timeMs: z\n .number()\n .default(1000)\n .optional()\n .describe('Sleep duration in milliseconds, defaults to 1000ms (1 second)'),\n});\n\nexport type ActionSleepParam = {\n timeMs?: number;\n};\n\nexport const defineActionSleep = (): DeviceAction<ActionSleepParam> => {\n return defineAction<typeof ActionSleepParamSchema, ActionSleepParam>({\n name: 'Sleep',\n description:\n 'Wait for a specified duration before continuing. Defaults to 1 second (1000ms) if not specified.',\n paramSchema: ActionSleepParamSchema,\n sample: {\n timeMs: 2000,\n },\n call: async (param) => {\n const duration = param?.timeMs ?? 1000;\n getDebug('device:common-action')(`Sleeping for ${duration}ms`);\n await new Promise((resolve) => setTimeout(resolve, duration));\n },\n });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n AndroidDeviceOpt,\n AndroidDeviceInputOpt,\n IOSDeviceOpt,\n IOSDeviceInputOpt,\n HarmonyDeviceOpt,\n HarmonyDeviceInputOpt,\n} from './device-options';\n"],"names":["AbstractInterface","defineAction","config","pointFromLocate","locate","missingMessage","Error","defineLocatedPointAction","param","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","tap","point","actionRightClickParamSchema","defineActionRightClick","rightClick","actionDoubleClickParamSchema","defineActionDoubleClick","doubleClick","actionHoverParamSchema","defineActionHover","hover","inputLocateDescription","actionInputParamSchema","val","String","defineActionInput","keyboard","actionKeyboardPressParamSchema","defineActionKeyboardPress","keyboardPress","actionScrollParamSchema","defineActionScroll","scroll","actionDragAndDropParamSchema","defineActionDragAndDrop","dragAndDrop","from","to","ActionLongPressParamSchema","defineActionLongPress","longPress","ActionSwipeParamSchema","normalizeMobileSwipeParam","screenSize","width","height","start","end","startPoint","endPoint","direction","Math","duration","repeatCount","defineActionSwipe","i","actionClearInputParamSchema","defineActionClearInput","clearInput","actionCursorMoveParamSchema","defineActionCursorMove","times","wait","timeMs","Promise","resolve","setTimeout","arrowKey","ActionPinchParamSchema","defineActionPinch","centerX","centerY","startDistance","endDistance","normalizePinchParam","element","baseDistance","fingerDistance","defineSystemInputAction","call","defineActionsFromInputPrimitives","input","options","actions","pointer","touch","system","systemActions","action","Boolean","createDefaultMobileActions","context","ActionSleepParamSchema","defineActionSleep","getDebug"],"mappings":";;;;;;;;;;;;;AAgIO,MAAeA;;QA8CpB;QA8BA;;AACF;AAKO,MAAMC,eAAe,CAK1BC,SAaOA;AAGT,SAASC,gBACPC,MAAuC,EACvCC,cAAsB;IAEtB,IAAI,CAACD,QACH,MAAM,IAAIE,MAAMD;IAElB,OAAO;QAAE,GAAGD,OAAO,MAAM,CAAC,EAAE;QAAE,GAAGA,OAAO,MAAM,CAAC,EAAE;IAAC;AACpD;AAEA,SAASG,yBAGPL,MAQD;IACC,OAAOD,aAA8B;QACnC,MAAMC,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW;QAC/B,gBAAgBA,OAAO,cAAc;QACrC,aAAaA,OAAO,WAAW;QAC/B,QAAQA,OAAO,MAAM;QACrB,MAAM,OAAOM;YACX,MAAMN,OAAO,IAAI,CACfC,gBAAgBK,MAAM,MAAM,EAAEN,OAAO,oBAAoB,GACzDM;QAEJ;IACF;AACF;AAGO,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMC,kBAAkB,CAC7BC,MAEON,yBAAsE;QAC3E,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsB;QAC1C;QACA,sBAAsB;QACtB,MAAM,OAAOK;YACX,MAAMD,IAAIC;QACZ;IACF;AAIK,MAAMC,8BAA8BL,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMK,yBAAyB,CACpCC,aAEOV,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaQ;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAA+B;QACnD;QACA,sBAAsB;QACtB,MAAM,OAAOD;YACX,MAAMG,WAAWH;QACnB;IACF;AAIK,MAAMI,+BAA+BR,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMQ,0BAA0B,CACrCC,cAEOb,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAkB;QACtC;QACA,sBAAsB;QACtB,MAAM,OAAOJ;YACX,MAAMM,YAAYN;QACpB;IACF;AAIK,MAAMO,yBAAyBX,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMW,oBAAoB,CAC/BC,QAEOhB,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAac;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACA,sBAAsB;QACtB,MAAM,OAAOP;YACX,MAAMS,MAAMT;QACd;IACF;AAIF,MAAMU,yBACJ;AACK,MAAMC,yBAAyBf,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACgB,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQf,4BACL,QAAQ,CAACa,wBACT,QAAQ;IACX,MAAMd,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAW,EACrC,OAAO,CAAC,WACR,QAAQ,CACP;IAEJ,qBAAqBA,EAAAA,OACX,GACP,QAAQ,GACR,QAAQ,CACP;AAEN;AAQO,MAAMkB,oBAAoB,CAC/BC,WAEO5B,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAawB;QACb,QAAQ;YACN,OAAO;YACP,QAAQ;gBAAE,QAAQ;YAAwB;QAC5C;QACA,MAAM,OAAOjB;YAEX,IAAKA,AAA0B,aAA1BA,MAAM,IAAI,EACbA,MAAM,IAAI,GAAG;YAGf,IAAIA,AAAe,YAAfA,MAAM,IAAI,EAAc,YAC1B,MAAMqB,SAAS,UAAU,CAACrB,MAAM,MAAM;YAIxC,IAAI,CAACA,SAAS,CAACA,MAAM,KAAK,EACxB;YAGF,MAAMqB,SAAS,QAAQ,CAACrB,MAAM,KAAK,EAAE;gBACnC,QAAQA,MAAM,MAAM;gBACpB,SAASA,AAAe,eAAfA,MAAM,IAAI;gBACnB,qBAAqBA,MAAM,mBAAmB;YAChD;QACF;IACF;AAIK,MAAMsB,iCAAiCpB,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMqB,4BAA4B,CACvCC,gBAEO/B,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAa6B;QACb,QAAQ;YACN,SAAS;QACX;QACA,MAAM,OAAOtB;YACX,MAAMwB,cAAcxB,MAAM,OAAO,EAAE;gBACjC,QAAQA,MAAM,MAAM;YACtB;QACF;IACF;AAIK,MAAMyB,0BAA0BvB,EAAE,MAAM,CAAC;IAC9C,YAAYA,CAAC,CAADA,OACL,CAAC;QACJ;QACA;QACA;QACA;QACA;KACD,EACA,OAAO,CAAC,gBACR,QAAQ,CACP;IAEJ,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMuB,qBAAqB,CAChCC,SAEOlC,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAagC;QACb,QAAQ;YACN,WAAW;YACX,YAAY;YACZ,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACA,MAAM,OAAOzB;YACX,MAAM2B,OAAO3B;QACf;IACF;AAIK,MAAM4B,+BAA+B1B,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAM0B,0BAA0B,CACrCC,cAEOrC,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamC;QACb,QAAQ;YACN,MAAM;gBAAE,QAAQ;YAA6B;YAC7C,IAAI;gBAAE,QAAQ;YAAuB;QACvC;QACA,MAAM,OAAO5B;YACX,MAAM+B,OAAO/B,MAAM,IAAI;YACvB,MAAMgC,KAAKhC,MAAM,EAAE;YACnB,IAAI,CAAC+B,MACH,MAAM,IAAIjC,MAAM;YAElB,IAAI,CAACkC,IACH,MAAM,IAAIlC,MAAM;YAElB,MAAMgC,YACJ;gBAAE,GAAGC,KAAK,MAAM,CAAC,EAAE;gBAAE,GAAGA,KAAK,MAAM,CAAC,EAAE;YAAC,GACvC;gBAAE,GAAGC,GAAG,MAAM,CAAC,EAAE;gBAAE,GAAGA,GAAG,MAAM,CAAC,EAAE;YAAC;QAEvC;IACF;AAGK,MAAMC,6BAA6B/B,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMgC,wBAAwB,CACnCC,YAEOpC,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAakC;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAqB;QACzC;QACA,sBAAsB;QACtB,MAAM,OAAO3B,OAAON;YAClB,MAAMmC,UAAU7B,OAAO;gBAAE,UAAUN,MAAM,QAAQ;YAAC;QACpD;IACF;AAGK,MAAMoC,yBAAyBlC,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,SAASmC,0BACdrC,KAAuB,EACvBsC,UAA6C;IAO7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAM,EAAEG,KAAK,EAAEC,GAAG,EAAE,GAAG1C;IAEvB,MAAM2C,aAAaF,QACf;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;IAAC,IACzC;QAAE,GAAGF,QAAQ;QAAG,GAAGC,SAAS;IAAE;IAElC,IAAII;IAEJ,IAAIF,KACFE,WAAW;QAAE,GAAGF,IAAI,MAAM,CAAC,EAAE;QAAE,GAAGA,IAAI,MAAM,CAAC,EAAE;IAAC;SAC3C,IAAI1C,MAAM,QAAQ,EAAE;QACzB,MAAM6C,YAAY7C,MAAM,SAAS;QACjC,IAAI,CAAC6C,WACH,MAAM,IAAI/C,MAAM;QAElB8C,WAAW;YACT,GACED,WAAW,CAAC,GACXE,CAAAA,AAAc,YAAdA,YACG7C,MAAM,QAAQ,GACd6C,AAAc,WAAdA,YACE,CAAC7C,MAAM,QAAQ,GACf;YACR,GACE2C,WAAW,CAAC,GACXE,CAAAA,AAAc,WAAdA,YACG7C,MAAM,QAAQ,GACd6C,AAAc,SAAdA,YACE,CAAC7C,MAAM,QAAQ,GACf;QACV;IACF,OACE,MAAM,IAAIF,MACR;IAIJ8C,SAAS,CAAC,GAAGE,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACF,SAAS,CAAC,EAAEL;IAC9CK,SAAS,CAAC,GAAGE,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACF,SAAS,CAAC,EAAEJ;IAE9C,MAAMO,WAAW/C,MAAM,QAAQ,IAAI;IAEnC,IAAIgD,cAAc,AAAwB,YAAxB,OAAOhD,MAAM,MAAM,GAAgBA,MAAM,MAAM,GAAG;IACpE,IAAIgD,AAAgB,MAAhBA,aACFA,cAAc;IAGhB,OAAO;QAAEL;QAAYC;QAAUG;QAAUC;IAAY;AACvD;AAEO,MAAMC,oBAAoB,CAACvD,SAIzBD,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa2C;QACb,QAAQ;YACN,OAAO;gBAAE,QAAQ;YAA6B;YAC9C,KAAK;gBAAE,QAAQ;YAA2B;QAC5C;QACA,MAAM,OAAOpC;YACX,MAAM,EAAE2C,UAAU,EAAEC,QAAQ,EAAEG,QAAQ,EAAEC,WAAW,EAAE,GACnDX,0BAA0BrC,OAAO,MAAMN,OAAO,IAAI;YACpD,IAAK,IAAIwD,IAAI,GAAGA,IAAIF,aAAaE,IAC/B,MAAMxD,OAAO,KAAK,CAACiD,YAAYC,UAAU;gBAAEG;YAAS;QAExD;IACF;AAIK,MAAMI,8BAA8BjD,EAAE,MAAM,CAAC;IAClD,QAAQC,4BACL,QAAQ,CAAC,iCACT,QAAQ;AACb;AAKO,MAAMiD,yBAAyB,CACpCC,aAEO5D,aAGL;QACA,MAAM;QACN,aAAauB;QACb,gBAAgB;QAChB,aAAamC;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAyB;QAC7C;QACA,MAAM,OAAOnD;YACX,MAAMqD,WAAWrD,MAAM,MAAM;QAC/B;IACF;AAIK,MAAMsD,8BAA8BpD,EAAE,MAAM,CAAC;IAClD,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;KAAQ,EACtB,QAAQ,CAAC;IACZ,OAAOA,EAAAA,MACE,GACN,GAAG,GACH,GAAG,CAAC,GACJ,OAAO,CAAC,GACR,QAAQ,CACP;AAEN;AAMO,MAAMqD,yBAAyB,CAAC7D,SAI9BD,aAGL;QACA,MAAM;QACN,aACE;QACF,aAAa6D;QACb,QAAQ;YACN,WAAW;YACX,OAAO;QACT;QACA,MAAM,OAAOtD;YACX,MAAMwD,QAAQxD,MAAM,KAAK,IAAI;YAC7B,IAAIN,OAAO,QAAQ,CAAC,UAAU,EAAE,YAC9B,MAAMA,OAAO,QAAQ,CAAC,UAAU,CAACM,MAAM,SAAS,EAAEwD;YAIpD,MAAMC,OACJ/D,OAAO,KAAK,IACV,EAAAgE,SACA,IAAIC,QAAc,CAACC,UAAYC,WAAWD,SAASF,QAAO;YAC9D,MAAMI,WAAW9D,AAAoB,WAApBA,MAAM,SAAS,GAAc,cAAc;YAC5D,IAAK,IAAIkD,IAAI,GAAGA,IAAIM,OAAON,IAAK;gBAC9B,MAAMxD,OAAO,QAAQ,CAAC,aAAa,CAACoE;gBACpC,MAAML,KAAK;YACb;QACF;IACF;AAIK,MAAMM,yBAAyB7D,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BACL,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;KAAM,EAClB,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AASO,MAAM8D,oBAAoB,CAACtE;IAIhC,IAAI,CAACA,OAAO,KAAK,EACf;IAGF,OAAOD,aAA8D;QACnE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAasE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAe;YACjC,WAAW;YACX,UAAU;QACZ;QACA,MAAM,OAAO/D;YACX,MAAM,EAAEiE,OAAO,EAAEC,OAAO,EAAEC,aAAa,EAAEC,WAAW,EAAErB,QAAQ,EAAE,GAC9DsB,oBAAoBrE,OAAO,MAAMN,OAAO,IAAI;YAC9C,MAAMA,OAAO,KAAK,GAChB;gBAAE,GAAGuE;gBAAS,GAAGC;YAAQ,GACzB;gBAAEC;gBAAeC;gBAAarB;YAAS;QAE3C;IACF;AACF;AAEO,SAASsB,oBACdrE,KAAuB,EACvBsC,UAA6C;IAQ7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAMgC,UAAUtE,MAAM,MAAM;IAC5B,MAAMiE,UAAUK,UACZxB,KAAK,KAAK,CAACwB,QAAQ,MAAM,CAAC,EAAE,IAC5BxB,KAAK,KAAK,CAACP,QAAQ;IACvB,MAAM2B,UAAUI,UACZxB,KAAK,KAAK,CAACwB,QAAQ,MAAM,CAAC,EAAE,IAC5BxB,KAAK,KAAK,CAACN,SAAS;IACxB,MAAMO,WAAW/C,MAAM,QAAQ,IAAI;IAEnC,MAAMuE,eAAezB,KAAK,KAAK,CAACA,KAAK,GAAG,CAACP,OAAOC,UAAU;IAC1D,MAAMgC,iBAAiBxE,MAAM,QAAQ,IAAIuE;IAEzC,MAAMJ,gBAAgBI;IACtB,MAAMH,cACJpE,AAAoB,UAApBA,MAAM,SAAS,GACXuE,eAAeC,iBACf1B,KAAK,GAAG,CAAC,IAAIyB,eAAeC;IAElC,OAAO;QAAEP;QAASC;QAASC;QAAeC;QAAarB;IAAS;AAClE;AAgCA,SAAS0B,wBACP/E,MAA+B,EAC/BgF,IAAyB;IAEzB,OAAOjF,aAAyC;QAC9C,MAAMC,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW;QAC/B,gBAAgBA,OAAO,cAAc;QACrC,mBAAmBA,OAAO,iBAAiB;QAC3C,kBAAkBA,OAAO,gBAAgB;QACzCgF;IACF;AACF;AAEO,SAASC,iCACdC,KAAsB,EACtBC,UAAuC,CAAC,CAAC;IAEzC,MAAMC,UAAgD,EAAE;IACxD,MAAM,EAAEC,OAAO,EAAE1D,QAAQ,EAAEM,MAAM,EAAEqD,KAAK,EAAEC,MAAM,EAAE,GAAGL;IAErD,IAAIG,SAAS;QACXD,QAAQ,IAAI,CAAC1E,gBAAgB2E,QAAQ,GAAG;QACxC,IAAIA,QAAQ,WAAW,EACrBD,QAAQ,IAAI,CAACnE,wBAAwBoE,QAAQ,WAAW;QAE1D,IAAIA,QAAQ,UAAU,EACpBD,QAAQ,IAAI,CAACtE,uBAAuBuE,QAAQ,UAAU;QAExD,IAAIA,QAAQ,KAAK,EACfD,QAAQ,IAAI,CAAChE,kBAAkBiE,QAAQ,KAAK;QAE9C,IAAIA,QAAQ,WAAW,EACrBD,QAAQ,IAAI,CAACjD,wBAAwBkD,QAAQ,WAAW;QAE1D,IAAIA,QAAQ,SAAS,EACnBD,QAAQ,IAAI,CAAC5C,sBAAsB6C,QAAQ,SAAS;IAExD;IAEA,IAAI1D,UACFyD,QAAQ,IAAI,CACV1D,kBAAkBC,WAClB+B,uBAAuB/B,SAAS,UAAU,GAC1CE,0BAA0BF,SAAS,aAAa,GAChDkC,uBAAuB;QAAElC;QAAU,OAAOwD,QAAQ,KAAK;IAAC;IAI5D,IAAIlD,QACFmD,QAAQ,IAAI,CAACpD,mBAAmBC,OAAO,MAAM;IAG/C,IAAIqD,OAAO,SAASH,QAAQ,IAAI,IAAIA,AAAyB,UAAzBA,QAAQ,YAAY,EACtDC,QAAQ,IAAI,CAAC7B,kBAAkB;QAAE,OAAO+B,MAAM,KAAK;QAAE,MAAMH,QAAQ,IAAI;IAAC;IAG1E,IAAIG,OAAO,SAASH,QAAQ,IAAI,IAAIA,AAAyB,UAAzBA,QAAQ,YAAY,EACtDC,QAAQ,IAAI,CAACd,kBAAkB;QAAE,OAAOgB,MAAM,KAAK;QAAE,MAAMH,QAAQ,IAAI;IAAC;IAG1E,IAAII,UAAUJ,QAAQ,aAAa,EAAE;QACnC,MAAM,EAAEK,aAAa,EAAE,GAAGL;QAC1B,IAAII,OAAO,UAAU,IAAIC,cAAc,UAAU,EAC/CJ,QAAQ,IAAI,CACVL,wBAAwBS,cAAc,UAAU,EAAED,OAAO,UAAU;QAGvE,IAAIA,OAAO,UAAU,IAAIC,cAAc,UAAU,EAC/CJ,QAAQ,IAAI,CACVL,wBAAwBS,cAAc,UAAU,EAAED,OAAO,UAAU;QAGvE,IAAIA,OAAO,gBAAgB,IAAIC,cAAc,gBAAgB,EAC3DJ,QAAQ,IAAI,CACVL,wBACES,cAAc,gBAAgB,EAC9BD,OAAO,gBAAgB;IAI/B;IAEA,OAAOH,QAAQ,MAAM,CAAC,CAACK,SACrBC,QAAQD;AAEZ;AAEO,SAASE,2BACdC,OAAiC;IAEjC,OAAOX,iCAAiCW,QAAQ,KAAK,EAAE;QACrD,MAAMA,QAAQ,IAAI;QAClB,OAAOA,QAAQ,KAAK;QACpB,eAAeA,QAAQ,aAAa;IACtC;AACF;AAGO,MAAMC,yBAAyBrF,EAAE,MAAM,CAAC;IAC7C,QAAQA,EAAAA,MACC,GACN,OAAO,CAAC,MACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMsF,oBAAoB,IACxB/F,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa8F;QACb,QAAQ;YACN,QAAQ;QACV;QACA,MAAM,OAAOvF;YACX,MAAM+C,WAAW/C,OAAO,UAAU;YAClCyF,SAAS,wBAAwB,CAAC,aAAa,EAAE1C,SAAS,EAAE,CAAC;YAC7D,MAAM,IAAIY,QAAQ,CAACC,UAAYC,WAAWD,SAASb;QACrD;IACF"}
1	+ {"version":3,"file":"device/index.mjs","sources":["../../../src/device/index.ts"],"sourcesContent":["import type { ModelRuntime } from '@/ai-model/models';\nimport { getMidsceneLocationSchema } from '@/common';\nimport type {\n ActionScrollParam,\n DeviceAction,\n LocateResultElement,\n} from '@/types';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport interface FileChooserHandler {\n accept(files: string[]): Promise<void>;\n}\n\nexport interface MjpegStreamFrame {\n /** Raw base64-encoded image bytes WITHOUT a `data:image/...;base64,` prefix. /\n data: string;\n contentType?: string;\n}\n\nexport interface MjpegStreamHandle {\n stop(): void \| Promise<void>;\n}\n\nexport interface MjpegStreamOptions {\n signal?: AbortSignal;\n onFrame(frame: MjpegStreamFrame): void;\n onError?(error: unknown): void;\n}\n\n/* A point in device-pixel coordinates on the screen. /\nexport interface PointerPoint {\n x: number;\n y: number;\n}\n\nexport interface PointerInputPrimitives {\n tap(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n doubleClick?(p: PointerPoint): Promise<void>;\n rightClick?(p: PointerPoint): Promise<void>;\n hover?(p: PointerPoint): Promise<void>;\n longPress?(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n dragAndDrop?(from: PointerPoint, to: PointerPoint): Promise<void>;\n}\n\nexport interface TouchInputPrimitives {\n swipe(\n start: PointerPoint,\n end: PointerPoint,\n opts?: { duration?: number; repeat?: number },\n ): Promise<void>;\n pinch?(\n center: PointerPoint,\n opts: { startDistance: number; endDistance: number; duration: number },\n ): Promise<void>;\n}\n\nexport interface KeyboardInputPrimitives {\n keyboardPress(keyName: string, opts?: { target?: unknown }): Promise<void>;\n cursorMove?(direction: 'left' \| 'right', times?: number): Promise<void>;\n typeText(\n value: string,\n opts?: {\n autoDismissKeyboard?: boolean;\n keyboardDismissStrategy?: 'esc-first' \| 'back-first';\n target?: unknown;\n replace?: boolean;\n focusOnly?: boolean;\n },\n ): Promise<void>;\n clearInput(target?: unknown): Promise<void>;\n}\n\nexport interface ScrollInputPrimitives {\n scroll(param: ActionScrollParam): Promise<void>;\n}\n\nexport interface SystemInputPrimitives {\n backButton?(): Promise<void>;\n homeButton?(): Promise<void>;\n recentAppsButton?(): Promise<void>;\n}\n\nexport interface InputPrimitives {\n pointer?: PointerInputPrimitives;\n keyboard?: KeyboardInputPrimitives;\n touch?: TouchInputPrimitives;\n scroll?: ScrollInputPrimitives;\n system?: SystemInputPrimitives;\n}\n\nexport interface MobileInputPrimitives extends InputPrimitives {\n pointer: PointerInputPrimitives & {\n doubleClick(p: PointerPoint): Promise<void>;\n longPress(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n };\n keyboard: KeyboardInputPrimitives;\n touch: TouchInputPrimitives;\n}\n\nexport interface BrowserInputPrimitives extends InputPrimitives {\n pointer: PointerInputPrimitives & {\n doubleClick(p: PointerPoint): Promise<void>;\n rightClick(p: PointerPoint): Promise<void>;\n hover(p: PointerPoint): Promise<void>;\n dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n longPress(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n };\n keyboard: KeyboardInputPrimitives;\n scroll: ScrollInputPrimitives;\n touch: TouchInputPrimitives;\n}\n\nexport interface ComputerInputPrimitives extends InputPrimitives {\n pointer: PointerInputPrimitives & {\n doubleClick(p: PointerPoint): Promise<void>;\n rightClick(p: PointerPoint): Promise<void>;\n hover(p: PointerPoint): Promise<void>;\n dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n };\n keyboard: KeyboardInputPrimitives;\n scroll: ScrollInputPrimitives;\n}\n\nexport abstract class AbstractInterface {\n abstract interfaceType: string;\n\n abstract screenshotBase64(): Promise<string>;\n abstract size(): Promise<Size>;\n abstract actionSpace(): DeviceAction[];\n\n abstract cacheFeatureForPoint?(\n center: [number, number],\n options?: {\n targetDescription?: string;\n modelRuntime?: ModelRuntime;\n },\n ): Promise<ElementCacheFeature>;\n abstract rectMatchesCacheFeature?(\n feature: ElementCacheFeature,\n ): Promise<Rect>;\n\n abstract destroy?(): Promise<void>;\n\n abstract describe?(): string;\n abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n // for web only\n registerFileChooserListener?(\n handler: (chooser: FileChooserHandler) => Promise<void>,\n ): Promise<{ dispose: () => void; getError: () => Error \| undefined }>;\n\n // @deprecated do NOT extend this method\n abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n // @deprecated do NOT extend this method\n abstract url?: () => string \| Promise<string>;\n\n // @deprecated do NOT extend this method\n abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n /\n Get the current device-local time as a formatted string.\n * Prefer this for user-visible time because timestamps alone do not preserve\n * the target device's timezone when formatted on the host machine.\n /\n getDeviceLocalTimeString?(format?: string): Promise<string>;\n\n /* URL of native MJPEG stream for real-time screen preview (e.g. WDA MJPEG server) /\n mjpegStreamUrl?: string;\n\n /\n Optional in-process MJPEG frame producer. Implementations can push raw\n * base64 frames here when there is no standalone native MJPEG URL, e.g.\n * Chromium CDP Page.startScreencast for web previews.\n /\n startMjpegStream?(\n options: MjpegStreamOptions,\n ): MjpegStreamHandle \| undefined \| Promise<MjpegStreamHandle \| undefined>;\n\n /\n Optional hook used after keyboard-only actions to force a fresh frame on\n * the active MJPEG stream. Implementations should be a no-op when no stream\n * is active.\n /\n flushPendingVisualUpdate?(): Promise<void>;\n\n /\n Optional navigation state probe for browser-like interfaces, used to drive\n * loading indicators in playground UIs. Returning `undefined` means the\n * interface does not expose this concept.\n /\n navigationState?(): Promise<{ isLoading: boolean }>;\n\n /\n Low-level device input surface. Platform implementations expose transport\n * primitives here; higher-level AI actions and manual pointer dispatch should\n * adapt to this instead of duplicating platform gesture logic.\n */\n inputPrimitives?: InputPrimitives;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\n// TReturn allows specifying the return type of the action\nexport const defineAction = <\n TSchema extends z.ZodType \| undefined = undefined,\n TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined,\n TReturn = any,\n>(\n config: {\n name: string;\n description: string;\n interfaceAlias?: string;\n paramSchema?: TSchema;\n call: (param: TRuntime) => Promise<TReturn> \| TReturn;\n } & Partial<\n Omit<\n DeviceAction<TRuntime, TReturn>,\n 'name' \| 'description' \| 'interfaceAlias' \| 'paramSchema' \| 'call'\n >\n >,\n): DeviceAction<TRuntime, TReturn> => {\n return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\nfunction pointFromLocate(\n locate: LocateResultElement \| undefined,\n missingMessage: string,\n): PointerPoint {\n if (!locate) {\n throw new Error(missingMessage);\n }\n return { x: locate.center[0], y: locate.center[1] };\n}\n\nfunction defineLocatedPointAction<\n TSchema extends z.ZodType,\n TParam extends { locate: LocateResultElement },\n>(config: {\n name: string;\n description: string;\n interfaceAlias?: string;\n paramSchema: TSchema;\n sample: DeviceAction<TParam>['sample'];\n missingLocateMessage: string;\n call: (point: PointerPoint, param: TParam) => Promise<void>;\n}): DeviceAction<TParam> {\n return defineAction<TSchema, TParam>({\n name: config.name,\n description: config.description,\n interfaceAlias: config.interfaceAlias,\n paramSchema: config.paramSchema,\n sample: config.sample,\n call: async (param) => {\n await config.call(\n pointFromLocate(param.locate, config.missingLocateMessage),\n param,\n );\n },\n });\n}\n\n// Tap\nexport const actionTapParamSchema = z.object({\n locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\nexport type ActionTapParam = {\n locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n tap: PointerInputPrimitives['tap'],\n): DeviceAction<ActionTapParam> => {\n return defineLocatedPointAction<typeof actionTapParamSchema, ActionTapParam>({\n name: 'Tap',\n description: 'Tap the element',\n interfaceAlias: 'aiTap',\n paramSchema: actionTapParamSchema,\n sample: {\n locate: { prompt: 'the \"Submit\" button' },\n },\n missingLocateMessage: 'Element not found, cannot tap',\n call: async (point) => {\n await tap(point);\n },\n });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n locate: getMidsceneLocationSchema().describe(\n 'The element to be right clicked',\n ),\n});\nexport type ActionRightClickParam = {\n locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n rightClick: NonNullable<PointerInputPrimitives['rightClick']>,\n): DeviceAction<ActionRightClickParam> => {\n return defineLocatedPointAction<\n typeof actionRightClickParamSchema,\n ActionRightClickParam\n >({\n name: 'RightClick',\n description: 'Right click the element',\n interfaceAlias: 'aiRightClick',\n paramSchema: actionRightClickParamSchema,\n sample: {\n locate: { prompt: 'the file icon on the desktop' },\n },\n missingLocateMessage: 'Element not found, cannot right click',\n call: async (point) => {\n await rightClick(point);\n },\n });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n locate: getMidsceneLocationSchema().describe(\n 'The element to be double clicked',\n ),\n});\nexport type ActionDoubleClickParam = {\n locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n doubleClick: NonNullable<PointerInputPrimitives['doubleClick']>,\n): DeviceAction<ActionDoubleClickParam> => {\n return defineLocatedPointAction<\n typeof actionDoubleClickParamSchema,\n ActionDoubleClickParam\n >({\n name: 'DoubleClick',\n description: 'Double click the element',\n interfaceAlias: 'aiDoubleClick',\n paramSchema: actionDoubleClickParamSchema,\n sample: {\n locate: { prompt: 'the folder icon' },\n },\n missingLocateMessage: 'Element not found, cannot double click',\n call: async (point) => {\n await doubleClick(point);\n },\n });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n hover: NonNullable<PointerInputPrimitives['hover']>,\n): DeviceAction<ActionHoverParam> => {\n return defineLocatedPointAction<\n typeof actionHoverParamSchema,\n ActionHoverParam\n >({\n name: 'Hover',\n description: 'Move the mouse to the element',\n interfaceAlias: 'aiHover',\n paramSchema: actionHoverParamSchema,\n sample: {\n locate: { prompt: 'the navigation menu item \"Products\"' },\n },\n missingLocateMessage: 'Element not found, cannot hover',\n call: async (point) => {\n await hover(point);\n },\n });\n};\n\n// Input\nconst inputLocateDescription =\n 'the position of the placeholder or text content in the target input field. If there is no content, locate the center of the input field.';\nexport const actionInputParamSchema = z.object({\n value: z\n .union([z.string(), z.number()])\n .transform((val) => String(val))\n .describe(\n 'The text to input. Provide the final content for replace mode, only the inserted characters for typeOnly mode, or an empty string when using clear mode to remove existing text.',\n ),\n locate: getMidsceneLocationSchema()\n .describe(inputLocateDescription)\n .optional(),\n mode: z\n .enum(['replace', 'clear', 'typeOnly'])\n .default('replace')\n .describe(\n 'Input mode: \"replace\" (default) - clear the field and input the value; \"typeOnly\" - type the value directly without clearing the field first, and should be set explicitly for incremental edits after moving the cursor; \"clear\" - clear the field without inputting new text.',\n ),\n autoDismissKeyboard: z\n .boolean()\n .optional()\n .describe(\n 'If true, the keyboard will be dismissed after the input is completed. Do not set it unless the user asks you to do so.',\n ),\n});\nexport type ActionInputParam = {\n value: string;\n locate?: LocateResultElement;\n mode?: 'replace' \| 'clear' \| 'typeOnly' \| 'append';\n autoDismissKeyboard?: boolean;\n};\n\nexport const defineActionInput = (\n keyboard: KeyboardInputPrimitives,\n): DeviceAction<ActionInputParam> => {\n return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n name: 'Input',\n description: 'Input the value into the element',\n interfaceAlias: 'aiInput',\n paramSchema: actionInputParamSchema,\n sample: {\n value: 'test@example.com',\n locate: { prompt: 'the email input field' },\n },\n call: async (param) => {\n // backward compat: convert deprecated 'append' to 'typeOnly'\n if ((param.mode as string) === 'append') {\n param.mode = 'typeOnly';\n }\n\n if (param.mode === 'clear') {\n await keyboard.clearInput(param.locate);\n return;\n }\n\n if (!param \|\| !param.value) {\n return;\n }\n\n await keyboard.typeText(param.value, {\n target: param.locate,\n replace: param.mode !== 'typeOnly',\n autoDismissKeyboard: param.autoDismissKeyboard,\n });\n },\n });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n locate: getMidsceneLocationSchema()\n .describe('The element to be clicked before pressing the key')\n .optional(),\n keyName: z\n .string()\n .describe(\n \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n ),\n});\nexport type ActionKeyboardPressParam = {\n locate?: LocateResultElement;\n keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n keyboardPress: KeyboardInputPrimitives['keyboardPress'],\n): DeviceAction<ActionKeyboardPressParam> => {\n return defineAction<\n typeof actionKeyboardPressParamSchema,\n ActionKeyboardPressParam\n >({\n name: 'KeyboardPress',\n description:\n 'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n interfaceAlias: 'aiKeyboardPress',\n paramSchema: actionKeyboardPressParamSchema,\n sample: {\n keyName: 'Enter',\n },\n call: async (param) => {\n await keyboardPress(param.keyName, {\n target: param.locate,\n });\n },\n });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n scrollType: z\n .enum([\n 'singleAction',\n 'scrollToBottom',\n 'scrollToTop',\n 'scrollToRight',\n 'scrollToLeft',\n ])\n .default('singleAction')\n .describe(\n 'The scroll behavior: \"singleAction\" for a single scroll action, \"scrollToBottom\" for scrolling all the way to the bottom by rapidly scrolling 5-10 times (skipping intermediate content until reaching the bottom), \"scrollToTop\" for scrolling all the way to the top by rapidly scrolling 5-10 times (skipping intermediate content until reaching the top), \"scrollToRight\" for scrolling all the way to the right by rapidly scrolling multiple times, \"scrollToLeft\" for scrolling all the way to the left by rapidly scrolling multiple times',\n ),\n direction: z\n .enum(['down', 'up', 'right', 'left'])\n .default('down')\n .describe(\n 'The direction to scroll. Only effective when scrollType is \"singleAction\".',\n ),\n distance: z\n .number()\n .nullable()\n .optional()\n .describe('The distance in pixels to scroll'),\n locate: getMidsceneLocationSchema()\n .optional()\n .describe(\n 'Describe the target element to be scrolled on, like \"the table\" or \"the list\" or \"the content area\" or \"the scrollable area\". Do NOT provide a general intent like \"scroll to find some element\"',\n ),\n});\n\nexport const defineActionScroll = (\n scroll: ScrollInputPrimitives['scroll'],\n): DeviceAction<ActionScrollParam> => {\n return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n name: 'Scroll',\n description:\n 'Scroll the page or a scrollable element to browse content. This is the preferred way to scroll on all platforms, including mobile. Supports scrollToBottom/scrollToTop for boundary navigation. Default: direction `down`, scrollType `singleAction`, distance `null`.',\n interfaceAlias: 'aiScroll',\n paramSchema: actionScrollParamSchema,\n sample: {\n direction: 'down',\n scrollType: 'singleAction',\n locate: { prompt: 'the center of the product list area' },\n },\n call: async (param) => {\n await scroll(param);\n },\n });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n from: getMidsceneLocationSchema().describe('The position to be dragged'),\n to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n from: LocateResultElement;\n to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n dragAndDrop: NonNullable<PointerInputPrimitives['dragAndDrop']>,\n): DeviceAction<ActionDragAndDropParam> => {\n return defineAction<\n typeof actionDragAndDropParamSchema,\n ActionDragAndDropParam\n >({\n name: 'DragAndDrop',\n description:\n 'Pick up a specific UI element and move it to a new position (e.g., reorder a card, move a file into a folder, sort list items). The element itself moves with your finger/mouse.',\n interfaceAlias: 'aiDragAndDrop',\n paramSchema: actionDragAndDropParamSchema,\n sample: {\n from: { prompt: 'the \"report.pdf\" file icon' },\n to: { prompt: 'the upload drop zone' },\n },\n call: async (param) => {\n const from = param.from;\n const to = param.to;\n if (!from) {\n throw new Error('missing \"from\" param for drag and drop');\n }\n if (!to) {\n throw new Error('missing \"to\" param for drag and drop');\n }\n await dragAndDrop(\n { x: from.center[0], y: from.center[1] },\n { x: to.center[0], y: to.center[1] },\n );\n },\n });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n locate: getMidsceneLocationSchema().describe(\n 'The element to be long pressed',\n ),\n duration: z\n .number()\n .optional()\n .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n locate: LocateResultElement;\n duration?: number;\n};\nexport const defineActionLongPress = (\n longPress: NonNullable<PointerInputPrimitives['longPress']>,\n): DeviceAction<ActionLongPressParam> => {\n return defineLocatedPointAction<\n typeof ActionLongPressParamSchema,\n ActionLongPressParam\n >({\n name: 'LongPress',\n description: 'Long press the element',\n interfaceAlias: 'aiLongPress',\n paramSchema: ActionLongPressParamSchema,\n sample: {\n locate: { prompt: 'the message bubble' },\n },\n missingLocateMessage: 'LongPress requires an element to be located',\n call: async (point, param) => {\n await longPress(point, { duration: param.duration });\n },\n });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n start: getMidsceneLocationSchema()\n .optional()\n .describe(\n 'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n ),\n direction: z\n .enum(['up', 'down', 'left', 'right'])\n .optional()\n .describe(\n 'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n ),\n distance: z\n .number()\n .optional()\n .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n end: getMidsceneLocationSchema()\n .optional()\n .describe(\n 'Ending point of the swipe gesture (mutually exclusive with distance)',\n ),\n duration: z\n .number()\n .default(300)\n .describe('Duration of the swipe gesture in milliseconds'),\n repeat: z\n .number()\n .optional()\n .describe(\n 'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n ),\n});\n\nexport type ActionSwipeParam = {\n start?: LocateResultElement;\n direction?: 'up' \| 'down' \| 'left' \| 'right';\n distance?: number;\n end?: LocateResultElement;\n duration?: number;\n repeat?: number;\n};\n\nexport function normalizeMobileSwipeParam(\n param: ActionSwipeParam,\n screenSize: { width: number; height: number },\n): {\n startPoint: { x: number; y: number };\n endPoint: { x: number; y: number };\n duration: number;\n repeatCount: number;\n} {\n const { width, height } = screenSize;\n const { start, end } = param;\n\n const startPoint = start\n ? { x: start.center[0], y: start.center[1] }\n : { x: width / 2, y: height / 2 };\n\n let endPoint: { x: number; y: number };\n\n if (end) {\n endPoint = { x: end.center[0], y: end.center[1] };\n } else if (param.distance) {\n const direction = param.direction;\n if (!direction) {\n throw new Error('direction is required for swipe gesture');\n }\n endPoint = {\n x:\n startPoint.x +\n (direction === 'right'\n ? param.distance\n : direction === 'left'\n ? -param.distance\n : 0),\n y:\n startPoint.y +\n (direction === 'down'\n ? param.distance\n : direction === 'up'\n ? -param.distance\n : 0),\n };\n } else {\n throw new Error(\n 'Either end or distance must be specified for swipe gesture',\n );\n }\n\n endPoint.x = Math.max(0, Math.min(endPoint.x, width));\n endPoint.y = Math.max(0, Math.min(endPoint.y, height));\n\n const duration = param.duration ?? 300;\n\n let repeatCount = typeof param.repeat === 'number' ? param.repeat : 1;\n if (repeatCount === 0) {\n repeatCount = 10;\n }\n\n return { startPoint, endPoint, duration, repeatCount };\n}\n\nexport const defineActionSwipe = (config: {\n swipe: TouchInputPrimitives['swipe'];\n size(): Promise<Size>;\n}): DeviceAction<ActionSwipeParam> => {\n return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n name: 'Swipe',\n description:\n 'Perform a touch gesture for interactions beyond regular scrolling (e.g., flip pages in a carousel, dismiss a notification, swipe-to-delete a list item). For regular content scrolling, use Scroll instead. Use \"distance\" + \"direction\" for relative movement, or \"end\" for precise endpoint.',\n paramSchema: ActionSwipeParamSchema,\n sample: {\n start: { prompt: 'center of the notification' },\n end: { prompt: 'upper edge of the screen' },\n },\n call: async (param) => {\n const { startPoint, endPoint, duration, repeatCount } =\n normalizeMobileSwipeParam(param, await config.size());\n for (let i = 0; i < repeatCount; i++) {\n await config.swipe(startPoint, endPoint, { duration });\n }\n },\n });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n locate: getMidsceneLocationSchema()\n .describe('The input field to be cleared')\n .optional(),\n});\nexport type ActionClearInputParam = {\n locate?: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n clearInput: KeyboardInputPrimitives['clearInput'],\n): DeviceAction<ActionClearInputParam> => {\n return defineAction<\n typeof actionClearInputParamSchema,\n ActionClearInputParam\n >({\n name: 'ClearInput',\n description: inputLocateDescription,\n interfaceAlias: 'aiClearInput',\n paramSchema: actionClearInputParamSchema,\n sample: {\n locate: { prompt: 'the search input field' },\n },\n call: async (param) => {\n await clearInput(param.locate);\n },\n });\n};\n\n// CursorMove\nexport const actionCursorMoveParamSchema = z.object({\n direction: z\n .enum(['left', 'right'])\n .describe('The direction to move the cursor'),\n times: z\n .number()\n .int()\n .min(1)\n .default(1)\n .describe(\n 'The number of times to move the cursor in the specified direction',\n ),\n});\nexport type ActionCursorMoveParam = {\n direction: 'left' \| 'right';\n times?: number;\n};\n\nexport const defineActionCursorMove = (config: {\n keyboard: Pick<KeyboardInputPrimitives, 'keyboardPress' \| 'cursorMove'>;\n sleep?(timeMs: number): Promise<void>;\n}): DeviceAction<ActionCursorMoveParam> => {\n return defineAction<\n typeof actionCursorMoveParamSchema,\n ActionCursorMoveParam\n >({\n name: 'CursorMove',\n description:\n 'Move the text cursor (caret) left or right within an input field or text area. Use this to reposition the cursor without selecting text.',\n paramSchema: actionCursorMoveParamSchema,\n sample: {\n direction: 'left',\n times: 3,\n },\n call: async (param) => {\n const times = param.times ?? 1;\n if (config.keyboard.cursorMove) {\n await config.keyboard.cursorMove(param.direction, times);\n return;\n }\n\n const wait =\n config.sleep ??\n ((timeMs: number) =>\n new Promise<void>((resolve) => setTimeout(resolve, timeMs)));\n const arrowKey = param.direction === 'left' ? 'ArrowLeft' : 'ArrowRight';\n for (let i = 0; i < times; i++) {\n await config.keyboard.keyboardPress(arrowKey);\n await wait(100);\n }\n },\n });\n};\n\n// Pinch\nexport const ActionPinchParamSchema = z.object({\n locate: getMidsceneLocationSchema()\n .optional()\n .describe(\n 'The element to pinch on. If not specified, the center of the screen will be used',\n ),\n direction: z\n .enum(['in', 'out'])\n .describe(\n 'Pinch direction. \"in\" = pinch fingers together (zoom out / shrink), \"out\" = spread fingers apart (zoom in / enlarge).',\n ),\n distance: z\n .number()\n .positive()\n .optional()\n .describe(\n 'How far each finger moves in pixels. Defaults to a quarter of the shorter screen dimension.',\n ),\n duration: z\n .number()\n .default(500)\n .optional()\n .describe('Duration of the pinch gesture in milliseconds'),\n});\n\nexport type ActionPinchParam = {\n locate?: LocateResultElement;\n direction: 'in' \| 'out';\n distance?: number;\n duration?: number;\n};\n\nexport const defineActionPinch = (config: {\n pinch: TouchInputPrimitives['pinch'];\n size(): Promise<Size>;\n}): DeviceAction<ActionPinchParam> \| undefined => {\n if (!config.pinch) {\n return undefined;\n }\n\n return defineAction<typeof ActionPinchParamSchema, ActionPinchParam>({\n name: 'Pinch',\n description:\n 'Perform a two-finger pinch gesture. Use direction \"in\" to pinch fingers together (zoom out), or \"out\" to spread fingers apart (zoom in). Optionally specify distance for how far each finger moves.',\n interfaceAlias: 'aiPinch',\n paramSchema: ActionPinchParamSchema,\n sample: {\n locate: { prompt: 'the map area' },\n direction: 'out',\n distance: 200,\n },\n call: async (param) => {\n const { centerX, centerY, startDistance, endDistance, duration } =\n normalizePinchParam(param, await config.size());\n await config.pinch?.(\n { x: centerX, y: centerY },\n { startDistance, endDistance, duration },\n );\n },\n });\n};\n\nexport function normalizePinchParam(\n param: ActionPinchParam,\n screenSize: { width: number; height: number },\n): {\n centerX: number;\n centerY: number;\n startDistance: number;\n endDistance: number;\n duration: number;\n} {\n const { width, height } = screenSize;\n const element = param.locate;\n const centerX = element\n ? Math.round(element.center[0])\n : Math.round(width / 2);\n const centerY = element\n ? Math.round(element.center[1])\n : Math.round(height / 2);\n const duration = param.duration ?? 500;\n\n const baseDistance = Math.round(Math.min(width, height) / 4);\n const fingerDistance = param.distance ?? baseDistance;\n\n const startDistance = baseDistance;\n const endDistance =\n param.direction === 'out'\n ? baseDistance + fingerDistance\n : Math.max(10, baseDistance - fingerDistance);\n\n return { centerX, centerY, startDistance, endDistance, duration };\n}\n\nexport interface MobileInputActionContext {\n input: MobileInputPrimitives;\n size(): Promise<Size>;\n sleep?(timeMs: number): Promise<void>;\n getDefaultAutoDismissKeyboard?(): boolean \| undefined;\n systemActions?: SystemInputActionOptions;\n}\n\nexport interface SystemInputActionConfig {\n name: string;\n description: string;\n interfaceAlias?: string;\n delayBeforeRunner?: number;\n delayAfterRunner?: number;\n}\n\nexport interface SystemInputActionOptions {\n backButton?: SystemInputActionConfig;\n homeButton?: SystemInputActionConfig;\n recentAppsButton?: SystemInputActionConfig;\n}\n\nexport interface InputPrimitiveActionOptions {\n size?: () => Promise<Size>;\n sleep?: (timeMs: number) => Promise<void>;\n includeSwipe?: boolean;\n includePinch?: boolean;\n systemActions?: SystemInputActionOptions;\n}\n\nfunction defineSystemInputAction(\n config: SystemInputActionConfig,\n call: () => Promise<void>,\n): DeviceAction<undefined, void> {\n return defineAction<undefined, undefined, void>({\n name: config.name,\n description: config.description,\n interfaceAlias: config.interfaceAlias,\n delayBeforeRunner: config.delayBeforeRunner,\n delayAfterRunner: config.delayAfterRunner,\n call,\n });\n}\n\nexport function defineActionsFromInputPrimitives(\n input: InputPrimitives,\n options: InputPrimitiveActionOptions = {},\n): DeviceAction<any>[] {\n const actions: Array<DeviceAction<any> \| undefined> = [];\n const { pointer, keyboard, scroll, touch, system } = input;\n\n if (pointer) {\n actions.push(defineActionTap(pointer.tap));\n if (pointer.doubleClick) {\n actions.push(defineActionDoubleClick(pointer.doubleClick));\n }\n if (pointer.rightClick) {\n actions.push(defineActionRightClick(pointer.rightClick));\n }\n if (pointer.hover) {\n actions.push(defineActionHover(pointer.hover));\n }\n if (pointer.dragAndDrop) {\n actions.push(defineActionDragAndDrop(pointer.dragAndDrop));\n }\n if (pointer.longPress) {\n actions.push(defineActionLongPress(pointer.longPress));\n }\n }\n\n if (keyboard) {\n actions.push(\n defineActionInput(keyboard),\n defineActionClearInput(keyboard.clearInput),\n defineActionKeyboardPress(keyboard.keyboardPress),\n defineActionCursorMove({ keyboard, sleep: options.sleep }),\n );\n }\n\n if (scroll) {\n actions.push(defineActionScroll(scroll.scroll));\n }\n\n if (touch?.swipe && options.size && options.includeSwipe !== false) {\n actions.push(defineActionSwipe({ swipe: touch.swipe, size: options.size }));\n }\n\n if (touch?.pinch && options.size && options.includePinch !== false) {\n actions.push(defineActionPinch({ pinch: touch.pinch, size: options.size }));\n }\n\n if (system && options.systemActions) {\n const { systemActions } = options;\n if (system.backButton && systemActions.backButton) {\n actions.push(\n defineSystemInputAction(systemActions.backButton, system.backButton),\n );\n }\n if (system.homeButton && systemActions.homeButton) {\n actions.push(\n defineSystemInputAction(systemActions.homeButton, system.homeButton),\n );\n }\n if (system.recentAppsButton && systemActions.recentAppsButton) {\n actions.push(\n defineSystemInputAction(\n systemActions.recentAppsButton,\n system.recentAppsButton,\n ),\n );\n }\n }\n\n return actions.filter((action): action is DeviceAction<any> =>\n Boolean(action),\n );\n}\n\nexport function createDefaultMobileActions(\n context: MobileInputActionContext,\n): DeviceAction<any>[] {\n return defineActionsFromInputPrimitives(context.input, {\n size: context.size,\n sleep: context.sleep,\n systemActions: context.systemActions,\n });\n}\n\n// Sleep\nexport const ActionSleepParamSchema = z.object({\n timeMs: z\n .number()\n .default(1000)\n .optional()\n .describe('Sleep duration in milliseconds, defaults to 1000ms (1 second)'),\n});\n\nexport type ActionSleepParam = {\n timeMs?: number;\n};\n\nexport const defineActionSleep = (): DeviceAction<ActionSleepParam> => {\n return defineAction<typeof ActionSleepParamSchema, ActionSleepParam>({\n name: 'Sleep',\n description:\n 'Wait for a specified duration before continuing. Defaults to 1 second (1000ms) if not specified.',\n paramSchema: ActionSleepParamSchema,\n sample: {\n timeMs: 2000,\n },\n call: async (param) => {\n const duration = param?.timeMs ?? 1000;\n getDebug('device:common-action')(`Sleeping for ${duration}ms`);\n await new Promise((resolve) => setTimeout(resolve, duration));\n },\n });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n AndroidDeviceOpt,\n AndroidDeviceInputOpt,\n IOSDeviceOpt,\n IOSDeviceInputOpt,\n HarmonyDeviceOpt,\n HarmonyDeviceInputOpt,\n} from './device-options';\n"],"names":["AbstractInterface","defineAction","config","pointFromLocate","locate","missingMessage","Error","defineLocatedPointAction","param","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","tap","point","actionRightClickParamSchema","defineActionRightClick","rightClick","actionDoubleClickParamSchema","defineActionDoubleClick","doubleClick","actionHoverParamSchema","defineActionHover","hover","inputLocateDescription","actionInputParamSchema","val","String","defineActionInput","keyboard","actionKeyboardPressParamSchema","defineActionKeyboardPress","keyboardPress","actionScrollParamSchema","defineActionScroll","scroll","actionDragAndDropParamSchema","defineActionDragAndDrop","dragAndDrop","from","to","ActionLongPressParamSchema","defineActionLongPress","longPress","ActionSwipeParamSchema","normalizeMobileSwipeParam","screenSize","width","height","start","end","startPoint","endPoint","direction","Math","duration","repeatCount","defineActionSwipe","i","actionClearInputParamSchema","defineActionClearInput","clearInput","actionCursorMoveParamSchema","defineActionCursorMove","times","wait","timeMs","Promise","resolve","setTimeout","arrowKey","ActionPinchParamSchema","defineActionPinch","centerX","centerY","startDistance","endDistance","normalizePinchParam","element","baseDistance","fingerDistance","defineSystemInputAction","call","defineActionsFromInputPrimitives","input","options","actions","pointer","touch","system","systemActions","action","Boolean","createDefaultMobileActions","context","ActionSleepParamSchema","defineActionSleep","getDebug"],"mappings":";;;;;;;;;;;;;AAgIO,MAAeA;;QA8CpB;QA8BA;;AACF;AAKO,MAAMC,eAAe,CAK1BC,SAaOA;AAGT,SAASC,gBACPC,MAAuC,EACvCC,cAAsB;IAEtB,IAAI,CAACD,QACH,MAAM,IAAIE,MAAMD;IAElB,OAAO;QAAE,GAAGD,OAAO,MAAM,CAAC,EAAE;QAAE,GAAGA,OAAO,MAAM,CAAC,EAAE;IAAC;AACpD;AAEA,SAASG,yBAGPL,MAQD;IACC,OAAOD,aAA8B;QACnC,MAAMC,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW;QAC/B,gBAAgBA,OAAO,cAAc;QACrC,aAAaA,OAAO,WAAW;QAC/B,QAAQA,OAAO,MAAM;QACrB,MAAM,OAAOM;YACX,MAAMN,OAAO,IAAI,CACfC,gBAAgBK,MAAM,MAAM,EAAEN,OAAO,oBAAoB,GACzDM;QAEJ;IACF;AACF;AAGO,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMC,kBAAkB,CAC7BC,MAEON,yBAAsE;QAC3E,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsB;QAC1C;QACA,sBAAsB;QACtB,MAAM,OAAOK;YACX,MAAMD,IAAIC;QACZ;IACF;AAIK,MAAMC,8BAA8BL,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMK,yBAAyB,CACpCC,aAEOV,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaQ;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAA+B;QACnD;QACA,sBAAsB;QACtB,MAAM,OAAOD;YACX,MAAMG,WAAWH;QACnB;IACF;AAIK,MAAMI,+BAA+BR,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMQ,0BAA0B,CACrCC,cAEOb,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAkB;QACtC;QACA,sBAAsB;QACtB,MAAM,OAAOJ;YACX,MAAMM,YAAYN;QACpB;IACF;AAIK,MAAMO,yBAAyBX,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMW,oBAAoB,CAC/BC,QAEOhB,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAac;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACA,sBAAsB;QACtB,MAAM,OAAOP;YACX,MAAMS,MAAMT;QACd;IACF;AAIF,MAAMU,yBACJ;AACK,MAAMC,yBAAyBf,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACgB,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQf,4BACL,QAAQ,CAACa,wBACT,QAAQ;IACX,MAAMd,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAW,EACrC,OAAO,CAAC,WACR,QAAQ,CACP;IAEJ,qBAAqBA,EAAAA,OACX,GACP,QAAQ,GACR,QAAQ,CACP;AAEN;AAQO,MAAMkB,oBAAoB,CAC/BC,WAEO5B,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAawB;QACb,QAAQ;YACN,OAAO;YACP,QAAQ;gBAAE,QAAQ;YAAwB;QAC5C;QACA,MAAM,OAAOjB;YAEX,IAAKA,AAA0B,aAA1BA,MAAM,IAAI,EACbA,MAAM,IAAI,GAAG;YAGf,IAAIA,AAAe,YAAfA,MAAM,IAAI,EAAc,YAC1B,MAAMqB,SAAS,UAAU,CAACrB,MAAM,MAAM;YAIxC,IAAI,CAACA,SAAS,CAACA,MAAM,KAAK,EACxB;YAGF,MAAMqB,SAAS,QAAQ,CAACrB,MAAM,KAAK,EAAE;gBACnC,QAAQA,MAAM,MAAM;gBACpB,SAASA,AAAe,eAAfA,MAAM,IAAI;gBACnB,qBAAqBA,MAAM,mBAAmB;YAChD;QACF;IACF;AAIK,MAAMsB,iCAAiCpB,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMqB,4BAA4B,CACvCC,gBAEO/B,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAa6B;QACb,QAAQ;YACN,SAAS;QACX;QACA,MAAM,OAAOtB;YACX,MAAMwB,cAAcxB,MAAM,OAAO,EAAE;gBACjC,QAAQA,MAAM,MAAM;YACtB;QACF;IACF;AAIK,MAAMyB,0BAA0BvB,EAAE,MAAM,CAAC;IAC9C,YAAYA,CAAC,CAADA,OACL,CAAC;QACJ;QACA;QACA;QACA;QACA;KACD,EACA,OAAO,CAAC,gBACR,QAAQ,CACP;IAEJ,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMuB,qBAAqB,CAChCC,SAEOlC,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAagC;QACb,QAAQ;YACN,WAAW;YACX,YAAY;YACZ,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACA,MAAM,OAAOzB;YACX,MAAM2B,OAAO3B;QACf;IACF;AAIK,MAAM4B,+BAA+B1B,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAM0B,0BAA0B,CACrCC,cAEOrC,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamC;QACb,QAAQ;YACN,MAAM;gBAAE,QAAQ;YAA6B;YAC7C,IAAI;gBAAE,QAAQ;YAAuB;QACvC;QACA,MAAM,OAAO5B;YACX,MAAM+B,OAAO/B,MAAM,IAAI;YACvB,MAAMgC,KAAKhC,MAAM,EAAE;YACnB,IAAI,CAAC+B,MACH,MAAM,IAAIjC,MAAM;YAElB,IAAI,CAACkC,IACH,MAAM,IAAIlC,MAAM;YAElB,MAAMgC,YACJ;gBAAE,GAAGC,KAAK,MAAM,CAAC,EAAE;gBAAE,GAAGA,KAAK,MAAM,CAAC,EAAE;YAAC,GACvC;gBAAE,GAAGC,GAAG,MAAM,CAAC,EAAE;gBAAE,GAAGA,GAAG,MAAM,CAAC,EAAE;YAAC;QAEvC;IACF;AAGK,MAAMC,6BAA6B/B,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMgC,wBAAwB,CACnCC,YAEOpC,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAakC;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAqB;QACzC;QACA,sBAAsB;QACtB,MAAM,OAAO3B,OAAON;YAClB,MAAMmC,UAAU7B,OAAO;gBAAE,UAAUN,MAAM,QAAQ;YAAC;QACpD;IACF;AAGK,MAAMoC,yBAAyBlC,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,SAASmC,0BACdrC,KAAuB,EACvBsC,UAA6C;IAO7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAM,EAAEG,KAAK,EAAEC,GAAG,EAAE,GAAG1C;IAEvB,MAAM2C,aAAaF,QACf;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;IAAC,IACzC;QAAE,GAAGF,QAAQ;QAAG,GAAGC,SAAS;IAAE;IAElC,IAAII;IAEJ,IAAIF,KACFE,WAAW;QAAE,GAAGF,IAAI,MAAM,CAAC,EAAE;QAAE,GAAGA,IAAI,MAAM,CAAC,EAAE;IAAC;SAC3C,IAAI1C,MAAM,QAAQ,EAAE;QACzB,MAAM6C,YAAY7C,MAAM,SAAS;QACjC,IAAI,CAAC6C,WACH,MAAM,IAAI/C,MAAM;QAElB8C,WAAW;YACT,GACED,WAAW,CAAC,GACXE,CAAAA,AAAc,YAAdA,YACG7C,MAAM,QAAQ,GACd6C,AAAc,WAAdA,YACE,CAAC7C,MAAM,QAAQ,GACf;YACR,GACE2C,WAAW,CAAC,GACXE,CAAAA,AAAc,WAAdA,YACG7C,MAAM,QAAQ,GACd6C,AAAc,SAAdA,YACE,CAAC7C,MAAM,QAAQ,GACf;QACV;IACF,OACE,MAAM,IAAIF,MACR;IAIJ8C,SAAS,CAAC,GAAGE,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACF,SAAS,CAAC,EAAEL;IAC9CK,SAAS,CAAC,GAAGE,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACF,SAAS,CAAC,EAAEJ;IAE9C,MAAMO,WAAW/C,MAAM,QAAQ,IAAI;IAEnC,IAAIgD,cAAc,AAAwB,YAAxB,OAAOhD,MAAM,MAAM,GAAgBA,MAAM,MAAM,GAAG;IACpE,IAAIgD,AAAgB,MAAhBA,aACFA,cAAc;IAGhB,OAAO;QAAEL;QAAYC;QAAUG;QAAUC;IAAY;AACvD;AAEO,MAAMC,oBAAoB,CAACvD,SAIzBD,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa2C;QACb,QAAQ;YACN,OAAO;gBAAE,QAAQ;YAA6B;YAC9C,KAAK;gBAAE,QAAQ;YAA2B;QAC5C;QACA,MAAM,OAAOpC;YACX,MAAM,EAAE2C,UAAU,EAAEC,QAAQ,EAAEG,QAAQ,EAAEC,WAAW,EAAE,GACnDX,0BAA0BrC,OAAO,MAAMN,OAAO,IAAI;YACpD,IAAK,IAAIwD,IAAI,GAAGA,IAAIF,aAAaE,IAC/B,MAAMxD,OAAO,KAAK,CAACiD,YAAYC,UAAU;gBAAEG;YAAS;QAExD;IACF;AAIK,MAAMI,8BAA8BjD,EAAE,MAAM,CAAC;IAClD,QAAQC,4BACL,QAAQ,CAAC,iCACT,QAAQ;AACb;AAKO,MAAMiD,yBAAyB,CACpCC,aAEO5D,aAGL;QACA,MAAM;QACN,aAAauB;QACb,gBAAgB;QAChB,aAAamC;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAyB;QAC7C;QACA,MAAM,OAAOnD;YACX,MAAMqD,WAAWrD,MAAM,MAAM;QAC/B;IACF;AAIK,MAAMsD,8BAA8BpD,EAAE,MAAM,CAAC;IAClD,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;KAAQ,EACtB,QAAQ,CAAC;IACZ,OAAOA,EAAAA,MACE,GACN,GAAG,GACH,GAAG,CAAC,GACJ,OAAO,CAAC,GACR,QAAQ,CACP;AAEN;AAMO,MAAMqD,yBAAyB,CAAC7D,SAI9BD,aAGL;QACA,MAAM;QACN,aACE;QACF,aAAa6D;QACb,QAAQ;YACN,WAAW;YACX,OAAO;QACT;QACA,MAAM,OAAOtD;YACX,MAAMwD,QAAQxD,MAAM,KAAK,IAAI;YAC7B,IAAIN,OAAO,QAAQ,CAAC,UAAU,EAAE,YAC9B,MAAMA,OAAO,QAAQ,CAAC,UAAU,CAACM,MAAM,SAAS,EAAEwD;YAIpD,MAAMC,OACJ/D,OAAO,KAAK,IACV,EAAAgE,SACA,IAAIC,QAAc,CAACC,UAAYC,WAAWD,SAASF,QAAO;YAC9D,MAAMI,WAAW9D,AAAoB,WAApBA,MAAM,SAAS,GAAc,cAAc;YAC5D,IAAK,IAAIkD,IAAI,GAAGA,IAAIM,OAAON,IAAK;gBAC9B,MAAMxD,OAAO,QAAQ,CAAC,aAAa,CAACoE;gBACpC,MAAML,KAAK;YACb;QACF;IACF;AAIK,MAAMM,yBAAyB7D,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BACL,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;KAAM,EAClB,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AASO,MAAM8D,oBAAoB,CAACtE;IAIhC,IAAI,CAACA,OAAO,KAAK,EACf;IAGF,OAAOD,aAA8D;QACnE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAasE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAe;YACjC,WAAW;YACX,UAAU;QACZ;QACA,MAAM,OAAO/D;YACX,MAAM,EAAEiE,OAAO,EAAEC,OAAO,EAAEC,aAAa,EAAEC,WAAW,EAAErB,QAAQ,EAAE,GAC9DsB,oBAAoBrE,OAAO,MAAMN,OAAO,IAAI;YAC9C,MAAMA,OAAO,KAAK,GAChB;gBAAE,GAAGuE;gBAAS,GAAGC;YAAQ,GACzB;gBAAEC;gBAAeC;gBAAarB;YAAS;QAE3C;IACF;AACF;AAEO,SAASsB,oBACdrE,KAAuB,EACvBsC,UAA6C;IAQ7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAMgC,UAAUtE,MAAM,MAAM;IAC5B,MAAMiE,UAAUK,UACZxB,KAAK,KAAK,CAACwB,QAAQ,MAAM,CAAC,EAAE,IAC5BxB,KAAK,KAAK,CAACP,QAAQ;IACvB,MAAM2B,UAAUI,UACZxB,KAAK,KAAK,CAACwB,QAAQ,MAAM,CAAC,EAAE,IAC5BxB,KAAK,KAAK,CAACN,SAAS;IACxB,MAAMO,WAAW/C,MAAM,QAAQ,IAAI;IAEnC,MAAMuE,eAAezB,KAAK,KAAK,CAACA,KAAK,GAAG,CAACP,OAAOC,UAAU;IAC1D,MAAMgC,iBAAiBxE,MAAM,QAAQ,IAAIuE;IAEzC,MAAMJ,gBAAgBI;IACtB,MAAMH,cACJpE,AAAoB,UAApBA,MAAM,SAAS,GACXuE,eAAeC,iBACf1B,KAAK,GAAG,CAAC,IAAIyB,eAAeC;IAElC,OAAO;QAAEP;QAASC;QAASC;QAAeC;QAAarB;IAAS;AAClE;AAgCA,SAAS0B,wBACP/E,MAA+B,EAC/BgF,IAAyB;IAEzB,OAAOjF,aAAyC;QAC9C,MAAMC,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW;QAC/B,gBAAgBA,OAAO,cAAc;QACrC,mBAAmBA,OAAO,iBAAiB;QAC3C,kBAAkBA,OAAO,gBAAgB;QACzCgF;IACF;AACF;AAEO,SAASC,iCACdC,KAAsB,EACtBC,UAAuC,CAAC,CAAC;IAEzC,MAAMC,UAAgD,EAAE;IACxD,MAAM,EAAEC,OAAO,EAAE1D,QAAQ,EAAEM,MAAM,EAAEqD,KAAK,EAAEC,MAAM,EAAE,GAAGL;IAErD,IAAIG,SAAS;QACXD,QAAQ,IAAI,CAAC1E,gBAAgB2E,QAAQ,GAAG;QACxC,IAAIA,QAAQ,WAAW,EACrBD,QAAQ,IAAI,CAACnE,wBAAwBoE,QAAQ,WAAW;QAE1D,IAAIA,QAAQ,UAAU,EACpBD,QAAQ,IAAI,CAACtE,uBAAuBuE,QAAQ,UAAU;QAExD,IAAIA,QAAQ,KAAK,EACfD,QAAQ,IAAI,CAAChE,kBAAkBiE,QAAQ,KAAK;QAE9C,IAAIA,QAAQ,WAAW,EACrBD,QAAQ,IAAI,CAACjD,wBAAwBkD,QAAQ,WAAW;QAE1D,IAAIA,QAAQ,SAAS,EACnBD,QAAQ,IAAI,CAAC5C,sBAAsB6C,QAAQ,SAAS;IAExD;IAEA,IAAI1D,UACFyD,QAAQ,IAAI,CACV1D,kBAAkBC,WAClB+B,uBAAuB/B,SAAS,UAAU,GAC1CE,0BAA0BF,SAAS,aAAa,GAChDkC,uBAAuB;QAAElC;QAAU,OAAOwD,QAAQ,KAAK;IAAC;IAI5D,IAAIlD,QACFmD,QAAQ,IAAI,CAACpD,mBAAmBC,OAAO,MAAM;IAG/C,IAAIqD,OAAO,SAASH,QAAQ,IAAI,IAAIA,AAAyB,UAAzBA,QAAQ,YAAY,EACtDC,QAAQ,IAAI,CAAC7B,kBAAkB;QAAE,OAAO+B,MAAM,KAAK;QAAE,MAAMH,QAAQ,IAAI;IAAC;IAG1E,IAAIG,OAAO,SAASH,QAAQ,IAAI,IAAIA,AAAyB,UAAzBA,QAAQ,YAAY,EACtDC,QAAQ,IAAI,CAACd,kBAAkB;QAAE,OAAOgB,MAAM,KAAK;QAAE,MAAMH,QAAQ,IAAI;IAAC;IAG1E,IAAII,UAAUJ,QAAQ,aAAa,EAAE;QACnC,MAAM,EAAEK,aAAa,EAAE,GAAGL;QAC1B,IAAII,OAAO,UAAU,IAAIC,cAAc,UAAU,EAC/CJ,QAAQ,IAAI,CACVL,wBAAwBS,cAAc,UAAU,EAAED,OAAO,UAAU;QAGvE,IAAIA,OAAO,UAAU,IAAIC,cAAc,UAAU,EAC/CJ,QAAQ,IAAI,CACVL,wBAAwBS,cAAc,UAAU,EAAED,OAAO,UAAU;QAGvE,IAAIA,OAAO,gBAAgB,IAAIC,cAAc,gBAAgB,EAC3DJ,QAAQ,IAAI,CACVL,wBACES,cAAc,gBAAgB,EAC9BD,OAAO,gBAAgB;IAI/B;IAEA,OAAOH,QAAQ,MAAM,CAAC,CAACK,SACrBC,QAAQD;AAEZ;AAEO,SAASE,2BACdC,OAAiC;IAEjC,OAAOX,iCAAiCW,QAAQ,KAAK,EAAE;QACrD,MAAMA,QAAQ,IAAI;QAClB,OAAOA,QAAQ,KAAK;QACpB,eAAeA,QAAQ,aAAa;IACtC;AACF;AAGO,MAAMC,yBAAyBrF,EAAE,MAAM,CAAC;IAC7C,QAAQA,EAAAA,MACC,GACN,OAAO,CAAC,MACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMsF,oBAAoB,IACxB/F,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa8F;QACb,QAAQ;YACN,QAAQ;QACV;QACA,MAAM,OAAOvF;YACX,MAAM+C,WAAW/C,OAAO,UAAU;YAClCyF,SAAS,wBAAwB,CAAC,aAAa,EAAE1C,SAAS,EAAE,CAAC;YAC7D,MAAM,IAAIY,QAAQ,CAACC,UAAYC,WAAWD,SAASb;QACrD;IACF"}