@donggui/core 1.5.4-donggui.3

package/dist/es/ai-model/ui-tars-planning.mjs

@@ -0,0 +1,249 @@
+import { getDebug } from "@midscene/shared/logger";
+import { transformHotkeyInput } from "@midscene/shared/us-keyboard-layout";
+import { assert } from "@midscene/shared/utils";
+import { actionParser } from "@ui-tars/action-parser";
+import { getSummary, getUiTarsPlanningPrompt } from "./prompt/ui-tars-planning.mjs";
+import { AIResponseParseError, callAIWithStringResponse } from "./service-caller/index.mjs";
+const debug = getDebug('ui-tars-planning');
+const warnLog = getDebug('ui-tars-planning', {
+    console: true
+});
+const bboxSize = 10;
+const pointToBbox = (point, width, height)=>[
+        Math.round(Math.max(point.x - bboxSize / 2, 0)),
+        Math.round(Math.max(point.y - bboxSize / 2, 0)),
+        Math.round(Math.min(point.x + bboxSize / 2, width)),
+        Math.round(Math.min(point.y + bboxSize / 2, height))
+    ];
+async function uiTarsPlanning(userInstruction, options) {
+    const { conversationHistory, context, modelConfig, actionContext } = options;
+    const { uiTarsModelVersion } = modelConfig;
+    let instruction = userInstruction;
+    if (actionContext) instruction = `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\n<user_instruction>${userInstruction}</user_instruction>`;
+    const systemPrompt = getUiTarsPlanningPrompt() + instruction;
+    const screenshotBase64 = context.screenshot.base64;
+    conversationHistory.append({
+        role: 'user',
+        content: [
+            {
+                type: 'image_url',
+                image_url: {
+                    url: screenshotBase64
+                }
+            }
+        ]
+    });
+    const res = await callAIWithStringResponse([
+        {
+            role: 'user',
+            content: systemPrompt
+        },
+        ...conversationHistory.snapshot()
+    ], modelConfig, {
+        abortSignal: options.abortSignal
+    });
+    let convertedText;
+    let parsed;
+    try {
+        convertedText = convertBboxToCoordinates(res.content);
+        const { shotSize } = context;
+        const parseResult = actionParser({
+            prediction: convertedText,
+            factor: [
+                1000,
+                1000
+            ],
+            screenContext: {
+                width: shotSize.width,
+                height: shotSize.height
+            },
+            modelVer: uiTarsModelVersion
+        });
+        parsed = parseResult.parsed;
+    } catch (parseError) {
+        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
+        throw new AIResponseParseError(`Parse error: ${errorMessage}`, JSON.stringify(res.content, void 0, 2), res.usage);
+    }
+    const { shotSize } = context;
+    debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parsed));
+    const transformActions = [];
+    const unhandledActions = [];
+    let shouldContinue = true;
+    parsed.forEach((action)=>{
+        const actionType = (action.action_type || '').toLowerCase();
+        if ('click' === actionType) {
+            assert(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box, shotSize);
+            const locate = {
+                prompt: action.thought || '',
+                bbox: pointToBbox({
+                    x: point[0],
+                    y: point[1]
+                }, shotSize.width, shotSize.height)
+            };
+            transformActions.push({
+                type: 'Tap',
+                param: {
+                    locate: locate
+                }
+            });
+        } else if ('left_double' === actionType) {
+            assert(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box, shotSize);
+            const locate = {
+                prompt: action.thought || '',
+                bbox: pointToBbox({
+                    x: point[0],
+                    y: point[1]
+                }, shotSize.width, shotSize.height)
+            };
+            transformActions.push({
+                type: 'DoubleClick',
+                param: {
+                    locate: locate
+                },
+                thought: action.thought || ''
+            });
+        } else if ('right_single' === actionType) {
+            assert(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box, shotSize);
+            const locate = {
+                prompt: action.thought || '',
+                bbox: pointToBbox({
+                    x: point[0],
+                    y: point[1]
+                }, shotSize.width, shotSize.height)
+            };
+            transformActions.push({
+                type: 'RightClick',
+                param: {
+                    locate: locate
+                },
+                thought: action.thought || ''
+            });
+        } else if ('drag' === actionType) {
+            assert(action.action_inputs.start_box, 'start_box is required');
+            assert(action.action_inputs.end_box, 'end_box is required');
+            const startPoint = getPoint(action.action_inputs.start_box, shotSize);
+            const endPoint = getPoint(action.action_inputs.end_box, shotSize);
+            transformActions.push({
+                type: 'DragAndDrop',
+                param: {
+                    from: {
+                        prompt: action.thought || '',
+                        bbox: pointToBbox({
+                            x: startPoint[0],
+                            y: startPoint[1]
+                        }, shotSize.width, shotSize.height)
+                    },
+                    to: {
+                        prompt: action.thought || '',
+                        bbox: pointToBbox({
+                            x: endPoint[0],
+                            y: endPoint[1]
+                        }, shotSize.width, shotSize.height)
+                    }
+                },
+                thought: action.thought || ''
+            });
+        } else if ('type' === actionType) transformActions.push({
+            type: 'Input',
+            param: {
+                value: action.action_inputs.content
+            },
+            thought: action.thought || ''
+        });
+        else if ('scroll' === actionType) transformActions.push({
+            type: 'Scroll',
+            param: {
+                direction: action.action_inputs.direction
+            },
+            thought: action.thought || ''
+        });
+        else if ('finished' === actionType) {
+            shouldContinue = false;
+            transformActions.push({
+                type: 'Finished',
+                param: {},
+                thought: action.thought || ''
+            });
+        } else if ('hotkey' === actionType) if (action.action_inputs.key) {
+            const keys = transformHotkeyInput(action.action_inputs.key);
+            transformActions.push({
+                type: 'KeyboardPress',
+                param: {
+                    keyName: keys.join('+')
+                },
+                thought: action.thought || ''
+            });
+        } else warnLog('No key found in action: hotkey. Will not perform action.');
+        else if ('wait' === actionType) transformActions.push({
+            type: 'Sleep',
+            param: {
+                timeMs: 1000
+            },
+            thought: action.thought || ''
+        });
+        else if (actionType) {
+            unhandledActions.push({
+                type: actionType,
+                thought: action.thought || ''
+            });
+            debug('Unhandled action type:', actionType, 'thought:', action.thought);
+        }
+    });
+    if (0 === transformActions.length) {
+        const errorDetails = [];
+        if (0 === parsed.length) {
+            errorDetails.push('Action parser returned no actions');
+            if (res.content.includes('Thought:') && !res.content.includes('Action:')) errorDetails.push('Response contains "Thought:" but missing "Action:" line');
+            else errorDetails.push('Response may be malformed or empty');
+        }
+        if (unhandledActions.length > 0) {
+            const types = unhandledActions.map((a)=>a.type).join(', ');
+            errorDetails.push(`Unhandled action types: ${types}`);
+        }
+        const errorMessage = [
+            'No actions found in UI-TARS response.',
+            ...errorDetails
+        ].join('\n');
+        throw new AIResponseParseError(errorMessage, JSON.stringify(res.content, void 0, 2), res.usage);
+    }
+    debug('transformActions', JSON.stringify(transformActions, null, 2));
+    const log = getSummary(res.content);
+    conversationHistory.append({
+        role: 'assistant',
+        content: log
+    });
+    return {
+        actions: transformActions,
+        log,
+        usage: res.usage,
+        rawResponse: JSON.stringify(res.content, void 0, 2),
+        shouldContinuePlanning: shouldContinue
+    };
+}
+function convertBboxToCoordinates(text) {
+    const pattern = /<bbox>(\d+)\s+(\d+)\s+(\d+)\s+(\d+)<\/bbox>/g;
+    function replaceMatch(match, x1, y1, x2, y2) {
+        const x1Num = Number.parseInt(x1, 10);
+        const y1Num = Number.parseInt(y1, 10);
+        const x2Num = Number.parseInt(x2, 10);
+        const y2Num = Number.parseInt(y2, 10);
+        const x = Math.floor((x1Num + x2Num) / 2);
+        const y = Math.floor((y1Num + y2Num) / 2);
+        return `(${x},${y})`;
+    }
+    const cleanedText = text.replace(/\[EOS\]/g, '');
+    return cleanedText.replace(pattern, replaceMatch).trim();
+}
+function getPoint(startBox, size) {
+    const [x, y] = JSON.parse(startBox);
+    return [
+        x * size.width,
+        y * size.height
+    ];
+}
+export { uiTarsPlanning };
+
+//# sourceMappingURL=ui-tars-planning.mjs.map

package/dist/es/ai-model/ui-tars-planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/ui-tars-planning.mjs","sources":["../../../src/ai-model/ui-tars-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  Size,\n  UIContext,\n} from '@/types';\nimport { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport { actionParser } from '@ui-tars/action-parser';\nimport type { ConversationHistory } from './conversation-history';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt/ui-tars-planning';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from './service-caller/index';\n\ntype ActionType =\n  | 'click'\n  | 'left_double'\n  | 'right_single'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\nconst warnLog = getDebug('ui-tars-planning', { console: true });\nconst bboxSize = 10;\nconst pointToBbox = (\n  point: { x: number; y: number },\n  width: number,\n  height: number,\n): [number, number, number, number] => {\n  return [\n    Math.round(Math.max(point.x - bboxSize / 2, 0)),\n    Math.round(Math.max(point.y - bboxSize / 2, 0)),\n    Math.round(Math.min(point.x + bboxSize / 2, width)),\n    Math.round(Math.min(point.y + bboxSize / 2, height)),\n  ];\n};\n\nexport async function uiTarsPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n    actionContext?: string;\n    abortSignal?: AbortSignal;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig, actionContext } = options;\n  const { uiTarsModelVersion } = modelConfig;\n\n  let instruction = userInstruction;\n  if (actionContext) {\n    instruction = `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\\n<user_instruction>${userInstruction}</user_instruction>`;\n  }\n\n  const systemPrompt = getUiTarsPlanningPrompt() + instruction;\n\n  const screenshotBase64 = context.screenshot.base64;\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n        },\n      },\n    ],\n  });\n\n  const res = await callAIWithStringResponse(\n    [\n      {\n        role: 'user',\n        content: systemPrompt,\n      },\n      ...conversationHistory.snapshot(),\n    ],\n    modelConfig,\n    { abortSignal: options.abortSignal },\n  );\n\n  let convertedText: string;\n  let parsed: ReturnType<typeof actionParser>['parsed'];\n\n  try {\n    convertedText = convertBboxToCoordinates(res.content);\n\n    const { shotSize } = context;\n    const parseResult = actionParser({\n      prediction: convertedText,\n      factor: [1000, 1000],\n      screenContext: {\n        width: shotSize.width,\n        height: shotSize.height,\n      },\n      modelVer: uiTarsModelVersion,\n    });\n    parsed = parseResult.parsed;\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `Parse error: ${errorMessage}`,\n      JSON.stringify(res.content, undefined, 2),\n      res.usage,\n    );\n  }\n\n  const { shotSize } = context;\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parsed),\n  );\n\n  const transformActions: PlanningAction[] = [];\n  const unhandledActions: Array<{ type: string; thought: string }> = [];\n  let shouldContinue = true;\n  parsed.forEach((action) => {\n    const actionType = (action.action_type || '').toLowerCase();\n    if (actionType === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, shotSize);\n\n      const locate = {\n        prompt: action.thought || '',\n        bbox: pointToBbox(\n          { x: point[0], y: point[1] },\n          shotSize.width,\n          shotSize.height,\n        ),\n      };\n\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate: locate,\n        },\n      });\n    } else if (actionType === 'left_double') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, shotSize);\n\n      const locate = {\n        prompt: action.thought || '',\n        bbox: pointToBbox(\n          { x: point[0], y: point[1] },\n          shotSize.width,\n          shotSize.height,\n        ),\n      };\n\n      transformActions.push({\n        type: 'DoubleClick',\n        param: {\n          locate: locate,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'right_single') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, shotSize);\n\n      const locate = {\n        prompt: action.thought || '',\n        bbox: pointToBbox(\n          { x: point[0], y: point[1] },\n          shotSize.width,\n          shotSize.height,\n        ),\n      };\n\n      transformActions.push({\n        type: 'RightClick',\n        param: {\n          locate: locate,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box, shotSize);\n      const endPoint = getPoint(action.action_inputs.end_box, shotSize);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: startPoint[0], y: startPoint[1] },\n              shotSize.width,\n              shotSize.height,\n            ),\n          },\n          to: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: endPoint[0], y: endPoint[1] },\n              shotSize.width,\n              shotSize.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'finished') {\n      shouldContinue = false;\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'hotkey') {\n      if (!action.action_inputs.key) {\n        warnLog('No key found in action: hotkey. Will not perform action.');\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys.join('+'),\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (actionType === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType) {\n      // Track unhandled action types\n      unhandledActions.push({\n        type: actionType,\n        thought: action.thought || '',\n      });\n      debug('Unhandled action type:', actionType, 'thought:', action.thought);\n    }\n  });\n\n  if (transformActions.length === 0) {\n    const errorDetails: string[] = [];\n\n    // Check if parsing failed\n    if (parsed.length === 0) {\n      errorDetails.push('Action parser returned no actions');\n\n      // Check if response has Thought but no Action\n      if (\n        res.content.includes('Thought:') &&\n        !res.content.includes('Action:')\n      ) {\n        errorDetails.push(\n          'Response contains \"Thought:\" but missing \"Action:\" line',\n        );\n      } else {\n        errorDetails.push('Response may be malformed or empty');\n      }\n    }\n\n    // Check if we have unhandled action types\n    if (unhandledActions.length > 0) {\n      const types = unhandledActions.map((a) => a.type).join(', ');\n      errorDetails.push(`Unhandled action types: ${types}`);\n    }\n\n    const errorMessage = [\n      'No actions found in UI-TARS response.',\n      ...errorDetails,\n    ].join('\\n');\n\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(res.content, undefined, 2),\n      res.usage,\n    );\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n  const log = getSummary(res.content);\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: log,\n  });\n\n  return {\n    actions: transformActions,\n    log,\n    usage: res.usage,\n    rawResponse: JSON.stringify(res.content, undefined, 2),\n    shouldContinuePlanning: shouldContinue,\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points\n * @param text - The text containing bbox tags to be converted\n * @returns The text with bbox tags replaced by coordinate points\n */\nfunction convertBboxToCoordinates(text: string): string {\n  // Match the four numbers after <bbox>\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    // Convert strings to numbers and calculate center point\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    // Use Math.floor to truncate and calculate center point\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    // Return formatted coordinate string\n    return `(${x},${y})`;\n  }\n\n  // Remove [EOS] and replace <bbox> coordinates\n  const cleanedText = text.replace(/\\[EOS\\]/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\nfunction getPoint(startBox: string, size: { width: number; height: number }) {\n  const [x, y] = JSON.parse(startBox);\n  return [x * size.width, y * size.height];\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n    end_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface LeftDoubleAction extends BaseAction {\n  action_type: 'left_double';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface RightSingleAction extends BaseAction {\n  action_type: 'right_single';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: Record<string, never>;\n}\n\nexport type Action =\n  | ClickAction\n  | LeftDoubleAction\n  | RightSingleAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n"],"names":["debug","getDebug","warnLog","bboxSize","pointToBbox","point","width","height","Math","uiTarsPlanning","userInstruction","options","conversationHistory","context","modelConfig","actionContext","uiTarsModelVersion","instruction","systemPrompt","getUiTarsPlanningPrompt","screenshotBase64","res","callAIWithStringResponse","convertedText","parsed","convertBboxToCoordinates","shotSize","parseResult","actionParser","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","transformActions","unhandledActions","shouldContinue","action","actionType","assert","getPoint","locate","startPoint","endPoint","keys","transformHotkeyInput","errorDetails","types","a","log","getSummary","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","y","cleanedText","startBox","size"],"mappings":";;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,oBAAoB;IAAE,SAAS;AAAK;AAC7D,MAAME,WAAW;AACjB,MAAMC,cAAc,CAClBC,OACAC,OACAC,SAEO;QACLC,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGG;QAC5CE,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGI;KAC7C;AAGI,eAAeE,eACpBC,eAAuB,EACvBC,OAMC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAEC,aAAa,EAAE,GAAGJ;IACrE,MAAM,EAAEK,kBAAkB,EAAE,GAAGF;IAE/B,IAAIG,cAAcP;IAClB,IAAIK,eACFE,cAAc,CAAC,yBAAyB,EAAEF,cAAc,8CAA8C,EAAEL,gBAAgB,mBAAmB,CAAC;IAG9I,MAAMQ,eAAeC,4BAA4BF;IAEjD,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElDD,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKQ;gBACP;YACF;SACD;IACH;IAEA,MAAMC,MAAM,MAAMC,yBAChB;QACE;YACE,MAAM;YACN,SAASJ;QACX;WACGN,oBAAoB,QAAQ;KAChC,EACDE,aACA;QAAE,aAAaH,QAAQ,WAAW;IAAC;IAGrC,IAAIY;IACJ,IAAIC;IAEJ,IAAI;QACFD,gBAAgBE,yBAAyBJ,IAAI,OAAO;QAEpD,MAAM,EAAEK,QAAQ,EAAE,GAAGb;QACrB,MAAMc,cAAcC,aAAa;YAC/B,YAAYL;YACZ,QAAQ;gBAAC;gBAAM;aAAK;YACpB,eAAe;gBACb,OAAOG,SAAS,KAAK;gBACrB,QAAQA,SAAS,MAAM;YACzB;YACA,UAAUV;QACZ;QACAQ,SAASG,YAAY,MAAM;IAC7B,EAAE,OAAOE,YAAY;QAEnB,MAAMC,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;QAC5D,MAAM,IAAII,qBACR,CAAC,aAAa,EAAEH,cAAc,EAC9BI,KAAK,SAAS,CAACb,IAAI,OAAO,EAAEc,QAAW,IACvCd,IAAI,KAAK;IAEb;IAEA,MAAM,EAAEK,QAAQ,EAAE,GAAGb;IAErBb,MACE,oBACAgB,oBACA,YACAkB,KAAK,SAAS,CAACV;IAGjB,MAAMY,mBAAqC,EAAE;IAC7C,MAAMC,mBAA6D,EAAE;IACrE,IAAIC,iBAAiB;IACrBd,OAAO,OAAO,CAAC,CAACe;QACd,MAAMC,aAAcD,AAAAA,CAAAA,OAAO,WAAW,IAAI,EAAC,EAAG,WAAW;QACzD,IAAIC,AAAe,YAAfA,YAAwB;YAC1BC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMlC,QAAQqC,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEb;YAEvD,MAAMiB,SAAS;gBACb,QAAQJ,OAAO,OAAO,IAAI;gBAC1B,MAAMnC,YACJ;oBAAE,GAAGC,KAAK,CAAC,EAAE;oBAAE,GAAGA,KAAK,CAAC,EAAE;gBAAC,GAC3BqB,SAAS,KAAK,EACdA,SAAS,MAAM;YAEnB;YAEAU,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQO;gBACV;YACF;QACF,OAAO,IAAIH,AAAe,kBAAfA,YAA8B;YACvCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMlC,QAAQqC,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEb;YAEvD,MAAMiB,SAAS;gBACb,QAAQJ,OAAO,OAAO,IAAI;gBAC1B,MAAMnC,YACJ;oBAAE,GAAGC,KAAK,CAAC,EAAE;oBAAE,GAAGA,KAAK,CAAC,EAAE;gBAAC,GAC3BqB,SAAS,KAAK,EACdA,SAAS,MAAM;YAEnB;YAEAU,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQO;gBACV;gBACA,SAASJ,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,mBAAfA,YAA+B;YACxCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMlC,QAAQqC,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEb;YAEvD,MAAMiB,SAAS;gBACb,QAAQJ,OAAO,OAAO,IAAI;gBAC1B,MAAMnC,YACJ;oBAAE,GAAGC,KAAK,CAAC,EAAE;oBAAE,GAAGA,KAAK,CAAC,EAAE;gBAAC,GAC3BqB,SAAS,KAAK,EACdA,SAAS,MAAM;YAEnB;YAEAU,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQO;gBACV;gBACA,SAASJ,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YAAuB;YAChCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCE,OAAOF,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMK,aAAaF,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEb;YAC5D,MAAMmB,WAAWH,SAASH,OAAO,aAAa,CAAC,OAAO,EAAEb;YACxDU,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,QAAQG,OAAO,OAAO,IAAI;wBAC1B,MAAMnC,YACJ;4BAAE,GAAGwC,UAAU,CAAC,EAAE;4BAAE,GAAGA,UAAU,CAAC,EAAE;wBAAC,GACrClB,SAAS,KAAK,EACdA,SAAS,MAAM;oBAEnB;oBACA,IAAI;wBACF,QAAQa,OAAO,OAAO,IAAI;wBAC1B,MAAMnC,YACJ;4BAAE,GAAGyC,QAAQ,CAAC,EAAE;4BAAE,GAAGA,QAAQ,CAAC,EAAE;wBAAC,GACjCnB,SAAS,KAAK,EACdA,SAAS,MAAM;oBAEnB;gBACF;gBACA,SAASa,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YACTJ,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOG,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,aAAfA,YACTJ,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWG,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,eAAfA,YAA2B;YACpCF,iBAAiB;YACjBF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO,CAAC;gBACR,SAASG,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,aAAfA,YACT,IAAKD,OAAO,aAAa,CAAC,GAAG,EAEtB;YACL,MAAMO,OAAOC,qBAAqBR,OAAO,aAAa,CAAC,GAAG;YAE1DH,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASU,KAAK,IAAI,CAAC;gBACrB;gBACA,SAASP,OAAO,OAAO,IAAI;YAC7B;QACF,OAXErC,QAAQ;aAYL,IAAIsC,AAAe,WAAfA,YACTJ,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASG,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,YAAY;YAErBH,iBAAiB,IAAI,CAAC;gBACpB,MAAMG;gBACN,SAASD,OAAO,OAAO,IAAI;YAC7B;YACAvC,MAAM,0BAA0BwC,YAAY,YAAYD,OAAO,OAAO;QACxE;IACF;IAEA,IAAIH,AAA4B,MAA5BA,iBAAiB,MAAM,EAAQ;QACjC,MAAMY,eAAyB,EAAE;QAGjC,IAAIxB,AAAkB,MAAlBA,OAAO,MAAM,EAAQ;YACvBwB,aAAa,IAAI,CAAC;YAGlB,IACE3B,IAAI,OAAO,CAAC,QAAQ,CAAC,eACrB,CAACA,IAAI,OAAO,CAAC,QAAQ,CAAC,YAEtB2B,aAAa,IAAI,CACf;iBAGFA,aAAa,IAAI,CAAC;QAEtB;QAGA,IAAIX,iBAAiB,MAAM,GAAG,GAAG;YAC/B,MAAMY,QAAQZ,iBAAiB,GAAG,CAAC,CAACa,IAAMA,EAAE,IAAI,EAAE,IAAI,CAAC;YACvDF,aAAa,IAAI,CAAC,CAAC,wBAAwB,EAAEC,OAAO;QACtD;QAEA,MAAMnB,eAAe;YACnB;eACGkB;SACJ,CAAC,IAAI,CAAC;QAGP,MAAM,IAAIf,qBACRH,cACAI,KAAK,SAAS,CAACb,IAAI,OAAO,EAAEc,QAAW,IACvCd,IAAI,KAAK;IAEb;IAEArB,MAAM,oBAAoBkC,KAAK,SAAS,CAACE,kBAAkB,MAAM;IACjE,MAAMe,MAAMC,WAAW/B,IAAI,OAAO;IAElCT,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAASuC;IACX;IAEA,OAAO;QACL,SAASf;QACTe;QACA,OAAO9B,IAAI,KAAK;QAChB,aAAaa,KAAK,SAAS,CAACb,IAAI,OAAO,EAAEc,QAAW;QACpD,wBAAwBG;IAC1B;AACF;AAOA,SAASb,yBAAyB4B,IAAY;IAE5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAGV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAGlC,MAAMM,IAAI1D,KAAK,KAAK,CAAEqD,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMG,IAAI3D,KAAK,KAAK,CAAEuD,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAGvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEC,EAAE,CAAC,CAAC;IACtB;IAGA,MAAMC,cAAcf,KAAK,OAAO,CAAC,YAAY;IAC7C,OAAOe,YAAY,OAAO,CAACd,SAASC,cAAc,IAAI;AACxD;AAEA,SAASb,SAAS2B,QAAgB,EAAEC,IAAuC;IACzE,MAAM,CAACJ,GAAGC,EAAE,GAAGjC,KAAK,KAAK,CAACmC;IAC1B,OAAO;QAACH,IAAII,KAAK,KAAK;QAAEH,IAAIG,KAAK,MAAM;KAAC;AAC1C"}

package/dist/es/common.mjs

@@ -0,0 +1,371 @@
+import { assert, isPlainObject } from "@midscene/shared/utils";
+import { isUITars } from "./ai-model/auto-glm/util.mjs";
+import { NodeType } from "@midscene/shared/constants";
+import { treeToList } from "@midscene/shared/extractor";
+import { compositeElementInfoImg } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+import { z } from "zod";
+const defaultBboxSize = 20;
+const debugInspectUtils = getDebug('ai:common');
+function pointToBbox(x, y, bboxSize = defaultBboxSize) {
+    const halfSize = bboxSize / 2;
+    const x1 = Math.max(x - halfSize, 0);
+    const y1 = Math.max(y - halfSize, 0);
+    const x2 = Math.min(x + halfSize, 1000);
+    const y2 = Math.min(y + halfSize, 1000);
+    return [
+        x1,
+        y1,
+        x2,
+        y2
+    ];
+}
+function fillBboxParam(locate, width, height, modelFamily) {
+    if (locate.bbox_2d && !locate?.bbox) {
+        locate.bbox = locate.bbox_2d;
+        delete locate.bbox_2d;
+    }
+    if (locate?.bbox) locate.bbox = adaptBbox(locate.bbox, width, height, modelFamily);
+    return locate;
+}
+function adaptQwen2_5Bbox(bbox) {
+    if (bbox.length < 2) {
+        const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;
+        throw new Error(msg);
+    }
+    const result = [
+        Math.round(bbox[0]),
+        Math.round(bbox[1]),
+        'number' == typeof bbox[2] ? Math.round(bbox[2]) : Math.round(bbox[0] + defaultBboxSize),
+        'number' == typeof bbox[3] ? Math.round(bbox[3]) : Math.round(bbox[1] + defaultBboxSize)
+    ];
+    return result;
+}
+function adaptGpt5Bbox(bbox) {
+    if (!Array.isArray(bbox) || 4 !== bbox.length || !bbox.every((value)=>'number' == typeof value && Number.isFinite(value))) {
+        const msg = `invalid bbox data for gpt-5 mode: ${JSON.stringify(bbox)} `;
+        throw new Error(msg);
+    }
+    const numericBbox = bbox;
+    return [
+        numericBbox[0],
+        numericBbox[1],
+        numericBbox[2],
+        numericBbox[3]
+    ];
+}
+function adaptDoubaoBbox(bbox, width, height) {
+    assert(width > 0 && height > 0, 'width and height must be greater than 0 in doubao mode');
+    if ('string' == typeof bbox) {
+        assert(/^(\d+)\s(\d+)\s(\d+)\s(\d+)$/.test(bbox.trim()), `invalid bbox data string for doubao-vision mode: ${bbox}`);
+        const splitted = bbox.split(' ');
+        if (4 === splitted.length) return [
+            Math.round(Number(splitted[0]) * width / 1000),
+            Math.round(Number(splitted[1]) * height / 1000),
+            Math.round(Number(splitted[2]) * width / 1000),
+            Math.round(Number(splitted[3]) * height / 1000)
+        ];
+        throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);
+    }
+    let bboxList = [];
+    if (Array.isArray(bbox) && 'string' == typeof bbox[0]) bbox.forEach((item)=>{
+        if ('string' == typeof item && item.includes(',')) {
+            const [x, y] = item.split(',');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else if ('string' == typeof item && item.includes(' ')) {
+            const [x, y] = item.split(' ');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else bboxList.push(Number(item));
+    });
+    else bboxList = bbox;
+    if (4 === bboxList.length || 5 === bboxList.length) return [
+        Math.round(bboxList[0] * width / 1000),
+        Math.round(bboxList[1] * height / 1000),
+        Math.round(bboxList[2] * width / 1000),
+        Math.round(bboxList[3] * height / 1000)
+    ];
+    if (6 === bboxList.length || 2 === bboxList.length || 3 === bboxList.length || 7 === bboxList.length) return [
+        Math.max(0, Math.round(bboxList[0] * width / 1000) - defaultBboxSize / 2),
+        Math.max(0, Math.round(bboxList[1] * height / 1000) - defaultBboxSize / 2),
+        Math.min(width, Math.round(bboxList[0] * width / 1000) + defaultBboxSize / 2),
+        Math.min(height, Math.round(bboxList[1] * height / 1000) + defaultBboxSize / 2)
+    ];
+    if (8 === bbox.length) return [
+        Math.round(bboxList[0] * width / 1000),
+        Math.round(bboxList[1] * height / 1000),
+        Math.round(bboxList[4] * width / 1000),
+        Math.round(bboxList[5] * height / 1000)
+    ];
+    const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;
+    throw new Error(msg);
+}
+function normalizeBboxInput(bbox) {
+    if (Array.isArray(bbox)) {
+        if (Array.isArray(bbox[0])) return bbox[0];
+    }
+    return bbox;
+}
+function adaptBbox(bbox, width, height, modelFamily) {
+    const normalizedBbox = normalizeBboxInput(bbox);
+    let result = [
+        0,
+        0,
+        0,
+        0
+    ];
+    result = 'doubao-vision' === modelFamily || 'doubao-seed' === modelFamily || isUITars(modelFamily) ? adaptDoubaoBbox(normalizedBbox, width, height) : 'gemini' === modelFamily ? adaptGeminiBbox(normalizedBbox, width, height) : 'qwen2.5-vl' === modelFamily ? adaptQwen2_5Bbox(normalizedBbox) : 'gpt-5' === modelFamily ? adaptGpt5Bbox(normalizedBbox) : normalized01000(normalizedBbox, width, height);
+    return result;
+}
+function normalized01000(bbox, width, height) {
+    return [
+        Math.round(bbox[0] * width / 1000),
+        Math.round(bbox[1] * height / 1000),
+        Math.round(bbox[2] * width / 1000),
+        Math.round(bbox[3] * height / 1000)
+    ];
+}
+function adaptGeminiBbox(bbox, width, height) {
+    const left = Math.round(bbox[1] * width / 1000);
+    const top = Math.round(bbox[0] * height / 1000);
+    const right = Math.round(bbox[3] * width / 1000);
+    const bottom = Math.round(bbox[2] * height / 1000);
+    return [
+        left,
+        top,
+        right,
+        bottom
+    ];
+}
+function adaptBboxToRect(bbox, width, height, offsetX = 0, offsetY = 0, rightLimit = width, bottomLimit = height, modelFamily, scale = 1) {
+    debugInspectUtils('adaptBboxToRect', bbox, width, height, 'offset', offsetX, offsetY, 'limit', rightLimit, bottomLimit, 'modelFamily', modelFamily, 'scale', scale);
+    const [left, top, right, bottom] = adaptBbox(bbox, width, height, modelFamily);
+    const rectLeft = Math.max(0, left);
+    const rectTop = Math.max(0, top);
+    const boundedRight = Math.min(right, rightLimit);
+    const boundedBottom = Math.min(bottom, bottomLimit);
+    const rectWidth = boundedRight - rectLeft + 1;
+    const rectHeight = boundedBottom - rectTop + 1;
+    const finalLeft = 1 !== scale ? Math.round(rectLeft / scale) : rectLeft;
+    const finalTop = 1 !== scale ? Math.round(rectTop / scale) : rectTop;
+    const finalWidth = 1 !== scale ? Math.round(rectWidth / scale) : rectWidth;
+    const finalHeight = 1 !== scale ? Math.round(rectHeight / scale) : rectHeight;
+    const rect = {
+        left: finalLeft + offsetX,
+        top: finalTop + offsetY,
+        width: finalWidth,
+        height: finalHeight
+    };
+    debugInspectUtils('adaptBboxToRect, result=', rect);
+    return rect;
+}
+function mergeRects(rects) {
+    const minLeft = Math.min(...rects.map((r)=>r.left));
+    const minTop = Math.min(...rects.map((r)=>r.top));
+    const maxRight = Math.max(...rects.map((r)=>r.left + r.width));
+    const maxBottom = Math.max(...rects.map((r)=>r.top + r.height));
+    return {
+        left: minLeft,
+        top: minTop,
+        width: maxRight - minLeft,
+        height: maxBottom - minTop
+    };
+}
+function expandSearchArea(rect, screenSize) {
+    const minArea = 160000;
+    const expandSize = 100;
+    const expandedLeft = Math.max(rect.left - expandSize, 0);
+    const expandedTop = Math.max(rect.top - expandSize, 0);
+    const expandRect = {
+        left: expandedLeft,
+        top: expandedTop,
+        width: Math.min(rect.left - expandedLeft + rect.width + expandSize, screenSize.width - expandedLeft),
+        height: Math.min(rect.top - expandedTop + rect.height + expandSize, screenSize.height - expandedTop)
+    };
+    const currentArea = expandRect.width * expandRect.height;
+    if (currentArea >= minArea) return expandRect;
+    const centerX = expandRect.left + expandRect.width / 2;
+    const centerY = expandRect.top + expandRect.height / 2;
+    const scaleFactor = Math.sqrt(minArea / currentArea);
+    const newWidth = Math.round(expandRect.width * scaleFactor);
+    const newHeight = Math.round(expandRect.height * scaleFactor);
+    const newLeft = Math.round(centerX - newWidth / 2);
+    const newTop = Math.round(centerY - newHeight / 2);
+    const left = Math.max(newLeft, 0);
+    const top = Math.max(newTop, 0);
+    return {
+        left,
+        top,
+        width: Math.min(newWidth, screenSize.width - left),
+        height: Math.min(newHeight, screenSize.height - top)
+    };
+}
+async function markupImageForLLM(screenshotBase64, tree, size) {
+    const elementsInfo = treeToList(tree);
+    const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo)=>{
+        if (elementInfo.attributes.nodeType === NodeType.TEXT) return false;
+        return true;
+    });
+    const imagePayload = await compositeElementInfoImg({
+        inputImgBase64: screenshotBase64,
+        elementsPositionInfo: elementsPositionInfoWithoutText,
+        size
+    });
+    return imagePayload;
+}
+function buildYamlFlowFromPlans(plans, actionSpace) {
+    const flow = [];
+    for (const plan of plans){
+        const verb = plan.type;
+        const action = actionSpace.find((action)=>action.name === verb);
+        if (!action) {
+            console.warn(`Cannot convert action ${verb} to yaml flow. Will ignore it.`);
+            continue;
+        }
+        const flowKey = action.interfaceAlias || verb;
+        const flowParam = action.paramSchema ? dumpActionParam(plan.param || {}, action.paramSchema) : {};
+        const flowItem = {
+            [flowKey]: '',
+            ...flowParam
+        };
+        flow.push(flowItem);
+    }
+    return flow;
+}
+const PointSchema = z.object({
+    left: z.number(),
+    top: z.number()
+});
+const SizeSchema = z.object({
+    width: z.number(),
+    height: z.number()
+});
+const RectSchema = PointSchema.and(SizeSchema).and(z.object({
+    zoom: z.number().optional()
+}));
+const TMultimodalPromptSchema = z.object({
+    images: z.array(z.object({
+        name: z.string(),
+        url: z.string()
+    })).optional(),
+    convertHttpImage2Base64: z.boolean().optional()
+});
+const TUserPromptSchema = z.union([
+    z.string(),
+    z.object({
+        prompt: z.string()
+    }).and(TMultimodalPromptSchema.partial())
+]);
+const locateFieldFlagName = 'midscene_location_field_flag';
+const MidsceneLocationInput = z.object({
+    prompt: TUserPromptSchema,
+    deepLocate: z.boolean().optional(),
+    deepThink: z.boolean().optional().describe('@deprecated Use `deepLocate` instead.'),
+    cacheable: z.boolean().optional(),
+    xpath: z.union([
+        z.string(),
+        z.boolean()
+    ]).optional()
+}).passthrough();
+const getMidsceneLocationSchema = ()=>MidsceneLocationInput;
+const ifMidsceneLocatorField = (field)=>{
+    let actualField = field;
+    if (actualField._def?.typeName === 'ZodOptional') actualField = actualField._def.innerType;
+    if (actualField._def?.typeName === 'ZodObject') {
+        const shape = actualField._def.shape();
+        if (locateFieldFlagName in shape) return true;
+        if ('prompt' in shape && shape.prompt) return true;
+    }
+    return false;
+};
+const dumpMidsceneLocatorField = (field)=>{
+    assert(ifMidsceneLocatorField(field), 'field is not a midscene locator field');
+    if ('string' == typeof field) return field;
+    if (field && 'object' == typeof field && field.prompt) {
+        if ('string' == typeof field.prompt) return field.prompt;
+        if ('object' == typeof field.prompt && field.prompt.prompt) return field.prompt.prompt;
+    }
+    return String(field);
+};
+const findAllMidsceneLocatorField = (zodType, requiredOnly)=>{
+    if (!zodType) return [];
+    const zodObject = zodType;
+    if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {
+        const keys = Object.keys(zodObject.shape);
+        return keys.filter((key)=>{
+            const field = zodObject.shape[key];
+            if (!ifMidsceneLocatorField(field)) return false;
+            if (requiredOnly) return field._def?.typeName !== 'ZodOptional';
+            return true;
+        });
+    }
+    return [];
+};
+const dumpActionParam = (jsonObject, zodSchema)=>{
+    if (!isPlainObject(jsonObject)) return {};
+    const locatorFields = findAllMidsceneLocatorField(zodSchema);
+    const result = {
+        ...jsonObject
+    };
+    for (const fieldName of locatorFields){
+        const fieldValue = result[fieldName];
+        if (fieldValue) {
+            if ('string' == typeof fieldValue) result[fieldName] = fieldValue;
+            else if ('object' == typeof fieldValue) {
+                if (fieldValue.prompt) {
+                    if ('string' == typeof fieldValue.prompt) result[fieldName] = fieldValue.prompt;
+                    else if ('object' == typeof fieldValue.prompt && fieldValue.prompt.prompt) result[fieldName] = fieldValue.prompt.prompt;
+                }
+            }
+        }
+    }
+    return result;
+};
+const parseActionParam = (rawParam, zodSchema, options)=>{
+    if (!zodSchema) return;
+    const param = rawParam ?? {};
+    const locateFields = findAllMidsceneLocatorField(zodSchema);
+    if (0 === locateFields.length) return zodSchema.parse(param);
+    const locateFieldValues = {};
+    for (const fieldName of locateFields)if (fieldName in param) locateFieldValues[fieldName] = param[fieldName];
+    const paramsForValidation = {};
+    for(const key in param)if (locateFields.includes(key)) paramsForValidation[key] = {
+        prompt: '_dummy_'
+    };
+    else paramsForValidation[key] = param[key];
+    const validated = zodSchema.parse(paramsForValidation);
+    const ratio = options?.shrunkShotToLogicalRatio;
+    for(const fieldName in locateFieldValues){
+        let value = locateFieldValues[fieldName];
+        if (void 0 !== ratio && 1 !== ratio && value && 'object' == typeof value && value.center && value.rect) value = {
+            ...value,
+            center: [
+                Math.round(value.center[0] / ratio),
+                Math.round(value.center[1] / ratio)
+            ],
+            rect: {
+                ...value.rect,
+                left: Math.round(value.rect.left / ratio),
+                top: Math.round(value.rect.top / ratio),
+                width: Math.round(value.rect.width / ratio),
+                height: Math.round(value.rect.height / ratio)
+            }
+        };
+        validated[fieldName] = value;
+    }
+    return validated;
+};
+const finalizeActionName = 'Finalize';
+const getReadableTimeString = (format = 'YYYY-MM-DD HH:mm:ss', timestamp)=>{
+    const now = void 0 !== timestamp ? new Date(timestamp) : new Date();
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, '0');
+    const day = String(now.getDate()).padStart(2, '0');
+    const hours = String(now.getHours()).padStart(2, '0');
+    const minutes = String(now.getMinutes()).padStart(2, '0');
+    const seconds = String(now.getSeconds()).padStart(2, '0');
+    const timeString = format.replace('YYYY', String(year)).replace('MM', month).replace('DD', day).replace('HH', hours).replace('mm', minutes).replace('ss', seconds);
+    return `${timeString} (${format})`;
+};
+export { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptGpt5Bbox, adaptQwen2_5Bbox, buildYamlFlowFromPlans, dumpActionParam, dumpMidsceneLocatorField, expandSearchArea, fillBboxParam, finalizeActionName, findAllMidsceneLocatorField, getMidsceneLocationSchema, getReadableTimeString, ifMidsceneLocatorField, markupImageForLLM, mergeRects, normalized01000, parseActionParam, pointToBbox };
+
+//# sourceMappingURL=common.mjs.map