@sqaitech/core 0.5.0

package/dist/es/ai-model/ui-tars-planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model\\ui-tars-planning.mjs","sources":["webpack://@sqaitech/core/./src/ai-model/ui-tars-planning.ts"],"sourcesContent":["import type {\r\n  PlanningAIResponse,\r\n  PlanningAction,\r\n  Size,\r\n  UIContext,\r\n} from '@/types';\r\nimport { type IModelConfig, UITarsModelVersion } from '@sqaitech/shared/env';\r\nimport { resizeImgBase64 } from '@sqaitech/shared/img';\r\nimport { getDebug } from '@sqaitech/shared/logger';\r\nimport { transformHotkeyInput } from '@sqaitech/shared/us-keyboard-layout';\r\nimport { assert } from '@sqaitech/shared/utils';\r\nimport { actionParser } from '@ui-tars/action-parser';\r\nimport { AIActionType } from './common';\r\nimport type { ConversationHistory } from './conversation-history';\r\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt/ui-tars-planning';\r\nimport { callAIWithStringResponse } from './service-caller/index';\r\ntype ActionType =\r\n  | 'click'\r\n  | 'drag'\r\n  | 'type'\r\n  | 'hotkey'\r\n  | 'finished'\r\n  | 'scroll'\r\n  | 'wait';\r\n\r\nconst debug = getDebug('ui-tars-planning');\r\nconst bboxSize = 10;\r\nconst pointToBbox = (\r\n  point: { x: number; y: number },\r\n  width: number,\r\n  height: number,\r\n): [number, number, number, number] => {\r\n  return [\r\n    Math.round(Math.max(point.x - bboxSize / 2, 0)),\r\n    Math.round(Math.max(point.y - bboxSize / 2, 0)),\r\n    Math.round(Math.min(point.x + bboxSize / 2, width)),\r\n    Math.round(Math.min(point.y + bboxSize / 2, height)),\r\n  ];\r\n};\r\n\r\nexport async function uiTarsPlanning(\r\n  userInstruction: string,\r\n  options: {\r\n    conversationHistory: ConversationHistory;\r\n    context: UIContext;\r\n    modelConfig: IModelConfig;\r\n  },\r\n): Promise<PlanningAIResponse> {\r\n  const { conversationHistory, context, modelConfig } = options;\r\n  const { uiTarsModelVersion } = modelConfig;\r\n  const systemPrompt = getUiTarsPlanningPrompt() + userInstruction;\r\n\r\n  const imagePayload = await resizeImageForUiTars(\r\n    context.screenshotBase64,\r\n    context.size,\r\n    uiTarsModelVersion,\r\n  );\r\n\r\n  conversationHistory.append({\r\n    role: 'user',\r\n    content: [\r\n      {\r\n        type: 'image_url',\r\n        image_url: {\r\n          url: imagePayload,\r\n        },\r\n      },\r\n    ],\r\n  });\r\n\r\n  const res = await callAIWithStringResponse(\r\n    [\r\n      {\r\n        role: 'user',\r\n        content: systemPrompt,\r\n      },\r\n      ...conversationHistory.snapshot(),\r\n    ],\r\n    AIActionType.INSPECT_ELEMENT,\r\n    modelConfig,\r\n  );\r\n  const convertedText = convertBboxToCoordinates(res.content);\r\n\r\n  const { size } = context;\r\n  const { parsed } = actionParser({\r\n    prediction: convertedText,\r\n    factor: [1000, 1000],\r\n    screenContext: {\r\n      width: size.width,\r\n      height: size.height,\r\n    },\r\n    modelVer: uiTarsModelVersion,\r\n  });\r\n\r\n  debug(\r\n    'ui-tars modelVer',\r\n    uiTarsModelVersion,\r\n    ', parsed',\r\n    JSON.stringify(parsed),\r\n  );\r\n\r\n  const transformActions: PlanningAction[] = [];\r\n  let shouldContinue = true;\r\n  parsed.forEach((action) => {\r\n    const actionType = (action.action_type || '').toLowerCase();\r\n    if (actionType === 'click') {\r\n      assert(action.action_inputs.start_box, 'start_box is required');\r\n      const point = getPoint(action.action_inputs.start_box, size);\r\n      transformActions.push({\r\n        type: 'Tap',\r\n        param: {\r\n          locate: {\r\n            prompt: action.thought || '',\r\n            bbox: pointToBbox(\r\n              { x: point[0], y: point[1] },\r\n              size.width,\r\n              size.height,\r\n            ),\r\n          },\r\n        },\r\n      });\r\n    } else if (actionType === 'drag') {\r\n      assert(action.action_inputs.start_box, 'start_box is required');\r\n      assert(action.action_inputs.end_box, 'end_box is required');\r\n      const startPoint = getPoint(action.action_inputs.start_box, size);\r\n      const endPoint = getPoint(action.action_inputs.end_box, size);\r\n      transformActions.push({\r\n        type: 'DragAndDrop',\r\n        param: {\r\n          from: {\r\n            prompt: action.thought || '',\r\n            bbox: pointToBbox(\r\n              { x: startPoint[0], y: startPoint[1] },\r\n              size.width,\r\n              size.height,\r\n            ),\r\n          },\r\n          to: {\r\n            prompt: action.thought || '',\r\n            bbox: pointToBbox(\r\n              { x: endPoint[0], y: endPoint[1] },\r\n              size.width,\r\n              size.height,\r\n            ),\r\n          },\r\n        },\r\n        thought: action.thought || '',\r\n      });\r\n    } else if (actionType === 'type') {\r\n      transformActions.push({\r\n        type: 'Input',\r\n        param: {\r\n          value: action.action_inputs.content,\r\n        },\r\n        thought: action.thought || '',\r\n      });\r\n    } else if (actionType === 'scroll') {\r\n      transformActions.push({\r\n        type: 'Scroll',\r\n        param: {\r\n          direction: action.action_inputs.direction,\r\n        },\r\n        thought: action.thought || '',\r\n      });\r\n    } else if (actionType === 'finished') {\r\n      shouldContinue = false;\r\n      transformActions.push({\r\n        type: 'Finished',\r\n        param: {},\r\n        thought: action.thought || '',\r\n      });\r\n    } else if (actionType === 'hotkey') {\r\n      if (!action.action_inputs.key) {\r\n        console.warn(\r\n          'No key found in action: hotkey. Will not perform action.',\r\n        );\r\n      } else {\r\n        const keys = transformHotkeyInput(action.action_inputs.key);\r\n\r\n        transformActions.push({\r\n          type: 'KeyboardPress',\r\n          param: {\r\n            keyName: keys,\r\n          },\r\n          thought: action.thought || '',\r\n        });\r\n      }\r\n    } else if (actionType === 'wait') {\r\n      transformActions.push({\r\n        type: 'Sleep',\r\n        param: {\r\n          timeMs: 1000,\r\n        },\r\n        thought: action.thought || '',\r\n      });\r\n    }\r\n  });\r\n\r\n  if (transformActions.length === 0) {\r\n    throw new Error(`No actions found, response: ${res.content}`, {\r\n      cause: {\r\n        prediction: res.content,\r\n        parsed,\r\n      },\r\n    });\r\n  }\r\n\r\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\r\n  const log = getSummary(res.content);\r\n\r\n  conversationHistory.append({\r\n    role: 'assistant',\r\n    content: log,\r\n  });\r\n\r\n  return {\r\n    actions: transformActions,\r\n    log,\r\n    usage: res.usage,\r\n    rawResponse: JSON.stringify(res.content, undefined, 2),\r\n    more_actions_needed_by_instruction: shouldContinue,\r\n  };\r\n}\r\n\r\n/**\r\n * Converts bounding box notation to coordinate points\r\n * @param text - The text containing bbox tags to be converted\r\n * @returns The text with bbox tags replaced by coordinate points\r\n */\r\nfunction convertBboxToCoordinates(text: string): string {\r\n  // Match the four numbers after <bbox>\r\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\r\n\r\n  function replaceMatch(\r\n    match: string,\r\n    x1: string,\r\n    y1: string,\r\n    x2: string,\r\n    y2: string,\r\n  ): string {\r\n    // Convert strings to numbers and calculate center point\r\n    const x1Num = Number.parseInt(x1, 10);\r\n    const y1Num = Number.parseInt(y1, 10);\r\n    const x2Num = Number.parseInt(x2, 10);\r\n    const y2Num = Number.parseInt(y2, 10);\r\n\r\n    // Use Math.floor to truncate and calculate center point\r\n    const x = Math.floor((x1Num + x2Num) / 2);\r\n    const y = Math.floor((y1Num + y2Num) / 2);\r\n\r\n    // Return formatted coordinate string\r\n    return `(${x},${y})`;\r\n  }\r\n\r\n  // Remove [EOS] and replace <bbox> coordinates\r\n  const cleanedText = text.replace(/\\[EOS\\]/g, '');\r\n  return cleanedText.replace(pattern, replaceMatch).trim();\r\n}\r\n\r\nfunction getPoint(startBox: string, size: { width: number; height: number }) {\r\n  const [x, y] = JSON.parse(startBox);\r\n  return [x * size.width, y * size.height];\r\n}\r\n\r\ninterface BaseAction {\r\n  action_type: ActionType;\r\n  action_inputs: Record<string, any>;\r\n  reflection: string | null;\r\n  thought: string | null;\r\n}\r\n\r\ninterface ClickAction extends BaseAction {\r\n  action_type: 'click';\r\n  action_inputs: {\r\n    start_box: string; // JSON string of [x, y] coordinates\r\n  };\r\n}\r\n\r\ninterface DragAction extends BaseAction {\r\n  action_type: 'drag';\r\n  action_inputs: {\r\n    start_box: string; // JSON string of [x, y] coordinates\r\n    end_box: string; // JSON string of [x, y] coordinates\r\n  };\r\n}\r\n\r\ninterface WaitAction extends BaseAction {\r\n  action_type: 'wait';\r\n  action_inputs: {\r\n    time: string; // JSON string of [x, y] coordinates\r\n  };\r\n}\r\n\r\ninterface TypeAction extends BaseAction {\r\n  action_type: 'type';\r\n  action_inputs: {\r\n    content: string;\r\n  };\r\n}\r\n\r\ninterface HotkeyAction extends BaseAction {\r\n  action_type: 'hotkey';\r\n  action_inputs: {\r\n    key: string;\r\n  };\r\n}\r\n\r\ninterface ScrollAction extends BaseAction {\r\n  action_type: 'scroll';\r\n  action_inputs: {\r\n    direction: 'up' | 'down';\r\n  };\r\n}\r\n\r\ninterface FinishedAction extends BaseAction {\r\n  action_type: 'finished';\r\n  action_inputs: Record<string, never>;\r\n}\r\n\r\nexport type Action =\r\n  | ClickAction\r\n  | DragAction\r\n  | TypeAction\r\n  | HotkeyAction\r\n  | ScrollAction\r\n  | FinishedAction\r\n  | WaitAction;\r\n\r\nexport async function resizeImageForUiTars(\r\n  imageBase64: string,\r\n  size: Size,\r\n  uiTarsVersion: UITarsModelVersion | undefined,\r\n) {\r\n  if (uiTarsVersion === UITarsModelVersion.V1_5) {\r\n    debug('ui-tars-v1.5, will check image size', size);\r\n    const currentPixels = size.width * size.height;\r\n    const maxPixels = 16384 * 28 * 28; //\r\n    if (currentPixels > maxPixels) {\r\n      const resizeFactor = Math.sqrt(maxPixels / currentPixels);\r\n      const newWidth = Math.floor(size.width * resizeFactor);\r\n      const newHeight = Math.floor(size.height * resizeFactor);\r\n      debug(\r\n        'resize image for ui-tars, new width: %s, new height: %s',\r\n        newWidth,\r\n        newHeight,\r\n      );\r\n      const resizedImage = await resizeImgBase64(imageBase64, {\r\n        width: newWidth,\r\n        height: newHeight,\r\n      });\r\n      return resizedImage;\r\n    }\r\n  }\r\n  return imageBase64;\r\n}\r\n"],"names":["debug","getDebug","bboxSize","pointToBbox","point","width","height","Math","uiTarsPlanning","userInstruction","options","conversationHistory","context","modelConfig","uiTarsModelVersion","systemPrompt","getUiTarsPlanningPrompt","imagePayload","resizeImageForUiTars","res","callAIWithStringResponse","AIActionType","convertedText","convertBboxToCoordinates","size","parsed","actionParser","JSON","transformActions","shouldContinue","action","actionType","assert","getPoint","startPoint","endPoint","keys","transformHotkeyInput","console","Error","log","getSummary","undefined","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","y","cleanedText","startBox","imageBase64","uiTarsVersion","UITarsModelVersion","currentPixels","maxPixels","resizeFactor","newWidth","newHeight","resizedImage","resizeImgBase64"],"mappings":";;;;;;;;;AAyBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,WAAW;AACjB,MAAMC,cAAc,CAClBC,OACAC,OACAC,SAEO;QACLC,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGG;QAC5CE,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGI;KAC7C;AAGI,eAAeE,eACpBC,eAAuB,EACvBC,OAIC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGH;IACtD,MAAM,EAAEI,kBAAkB,EAAE,GAAGD;IAC/B,MAAME,eAAeC,4BAA4BP;IAEjD,MAAMQ,eAAe,MAAMC,qBACzBN,QAAQ,gBAAgB,EACxBA,QAAQ,IAAI,EACZE;IAGFH,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKM;gBACP;YACF;SACD;IACH;IAEA,MAAME,MAAM,MAAMC,yBAChB;QACE;YACE,MAAM;YACN,SAASL;QACX;WACGJ,oBAAoB,QAAQ;KAChC,EACDU,aAAa,eAAe,EAC5BR;IAEF,MAAMS,gBAAgBC,yBAAyBJ,IAAI,OAAO;IAE1D,MAAM,EAAEK,IAAI,EAAE,GAAGZ;IACjB,MAAM,EAAEa,MAAM,EAAE,GAAGC,aAAa;QAC9B,YAAYJ;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOE,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACA,UAAUV;IACZ;IAEAd,MACE,oBACAc,oBACA,YACAa,KAAK,SAAS,CAACF;IAGjB,MAAMG,mBAAqC,EAAE;IAC7C,IAAIC,iBAAiB;IACrBJ,OAAO,OAAO,CAAC,CAACK;QACd,MAAMC,aAAcD,AAAAA,CAAAA,OAAO,WAAW,IAAI,EAAC,EAAG,WAAW;QACzD,IAAIC,AAAe,YAAfA,YAAwB;YAC1BC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAM1B,QAAQ6B,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEN;YACvDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQ;wBACN,QAAQE,OAAO,OAAO,IAAI;wBAC1B,MAAM3B,YACJ;4BAAE,GAAGC,KAAK,CAAC,EAAE;4BAAE,GAAGA,KAAK,CAAC,EAAE;wBAAC,GAC3BoB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;YACF;QACF,OAAO,IAAIO,AAAe,WAAfA,YAAuB;YAChCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCE,OAAOF,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMI,aAAaD,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEN;YAC5D,MAAMW,WAAWF,SAASH,OAAO,aAAa,CAAC,OAAO,EAAEN;YACxDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,QAAQE,OAAO,OAAO,IAAI;wBAC1B,MAAM3B,YACJ;4BAAE,GAAG+B,UAAU,CAAC,EAAE;4BAAE,GAAGA,UAAU,CAAC,EAAE;wBAAC,GACrCV,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;oBACA,IAAI;wBACF,QAAQM,OAAO,OAAO,IAAI;wBAC1B,MAAM3B,YACJ;4BAAE,GAAGgC,QAAQ,CAAC,EAAE;4BAAE,GAAGA,QAAQ,CAAC,EAAE;wBAAC,GACjCX,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASM,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOE,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,aAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWE,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,eAAfA,YAA2B;YACpCF,iBAAiB;YACjBD,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO,CAAC;gBACR,SAASE,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,aAAfA,YACT,IAAKD,OAAO,aAAa,CAAC,GAAG,EAItB;YACL,MAAMM,OAAOC,qBAAqBP,OAAO,aAAa,CAAC,GAAG;YAE1DF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASQ;gBACX;gBACA,SAASN,OAAO,OAAO,IAAI;YAC7B;QACF,OAbEQ,QAAQ,IAAI,CACV;aAaC,IAAIP,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASE,OAAO,OAAO,IAAI;QAC7B;IAEJ;IAEA,IAAIF,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIW,MAAM,CAAC,4BAA4B,EAAEpB,IAAI,OAAO,EAAE,EAAE;QAC5D,OAAO;YACL,YAAYA,IAAI,OAAO;YACvBM;QACF;IACF;IAGFzB,MAAM,oBAAoB2B,KAAK,SAAS,CAACC,kBAAkB,MAAM;IACjE,MAAMY,MAAMC,WAAWtB,IAAI,OAAO;IAElCR,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS6B;IACX;IAEA,OAAO;QACL,SAASZ;QACTY;QACA,OAAOrB,IAAI,KAAK;QAChB,aAAaQ,KAAK,SAAS,CAACR,IAAI,OAAO,EAAEuB,QAAW;QACpD,oCAAoCb;IACtC;AACF;AAOA,SAASN,yBAAyBoB,IAAY;IAE5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAGV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAGlC,MAAMM,IAAIjD,KAAK,KAAK,CAAE4C,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMG,IAAIlD,KAAK,KAAK,CAAE8C,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAGvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEC,EAAE,CAAC,CAAC;IACtB;IAGA,MAAMC,cAAcf,KAAK,OAAO,CAAC,YAAY;IAC7C,OAAOe,YAAY,OAAO,CAACd,SAASC,cAAc,IAAI;AACxD;AAEA,SAASZ,SAAS0B,QAAgB,EAAEnC,IAAuC;IACzE,MAAM,CAACgC,GAAGC,EAAE,GAAG9B,KAAK,KAAK,CAACgC;IAC1B,OAAO;QAACH,IAAIhC,KAAK,KAAK;QAAEiC,IAAIjC,KAAK,MAAM;KAAC;AAC1C;AAkEO,eAAeN,qBACpB0C,WAAmB,EACnBpC,IAAU,EACVqC,aAA6C;IAE7C,IAAIA,kBAAkBC,mBAAmB,IAAI,EAAE;QAC7C9D,MAAM,uCAAuCwB;QAC7C,MAAMuC,gBAAgBvC,KAAK,KAAK,GAAGA,KAAK,MAAM;QAC9C,MAAMwC,YAAY;QAClB,IAAID,gBAAgBC,WAAW;YAC7B,MAAMC,eAAe1D,KAAK,IAAI,CAACyD,YAAYD;YAC3C,MAAMG,WAAW3D,KAAK,KAAK,CAACiB,KAAK,KAAK,GAAGyC;YACzC,MAAME,YAAY5D,KAAK,KAAK,CAACiB,KAAK,MAAM,GAAGyC;YAC3CjE,MACE,2DACAkE,UACAC;YAEF,MAAMC,eAAe,MAAMC,gBAAgBT,aAAa;gBACtD,OAAOM;gBACP,QAAQC;YACV;YACA,OAAOC;QACT;IACF;IACA,OAAOR;AACT"}

package/dist/es/device/index.mjs

@@ -0,0 +1,152 @@
+import { getMidsceneLocationSchema } from "../ai-model/index.mjs";
+import { z } from "zod";
+class AbstractInterface {
+}
+const defineAction = (config)=>config;
+const actionTapParamSchema = z.object({
+    locate: getMidsceneLocationSchema().describe('The element to be tapped')
+});
+const defineActionTap = (call)=>defineAction({
+        name: 'Tap',
+        description: 'Tap the element',
+        interfaceAlias: 'aiTap',
+        paramSchema: actionTapParamSchema,
+        call
+    });
+const actionRightClickParamSchema = z.object({
+    locate: getMidsceneLocationSchema().describe('The element to be right clicked')
+});
+const defineActionRightClick = (call)=>defineAction({
+        name: 'RightClick',
+        description: 'Right click the element',
+        interfaceAlias: 'aiRightClick',
+        paramSchema: actionRightClickParamSchema,
+        call
+    });
+const actionDoubleClickParamSchema = z.object({
+    locate: getMidsceneLocationSchema().describe('The element to be double clicked')
+});
+const defineActionDoubleClick = (call)=>defineAction({
+        name: 'DoubleClick',
+        description: 'Double click the element',
+        interfaceAlias: 'aiDoubleClick',
+        paramSchema: actionDoubleClickParamSchema,
+        call
+    });
+const actionHoverParamSchema = z.object({
+    locate: getMidsceneLocationSchema().describe('The element to be hovered')
+});
+const defineActionHover = (call)=>defineAction({
+        name: 'Hover',
+        description: 'Move the mouse to the element',
+        interfaceAlias: 'aiHover',
+        paramSchema: actionHoverParamSchema,
+        call
+    });
+const actionInputParamSchema = z.object({
+    value: z.union([
+        z.string(),
+        z.number()
+    ]).transform((val)=>String(val)).describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
+    locate: getMidsceneLocationSchema().describe('The element to be input').optional(),
+    mode: z["enum"]([
+        'replace',
+        'clear',
+        'append'
+    ]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "append" - append the value to existing content; "clear" - clear the field without inputting new text.')
+});
+const defineActionInput = (call)=>defineAction({
+        name: 'Input',
+        description: 'Input the value into the element',
+        interfaceAlias: 'aiInput',
+        paramSchema: actionInputParamSchema,
+        call
+    });
+const actionKeyboardPressParamSchema = z.object({
+    locate: getMidsceneLocationSchema().describe('The element to be clicked before pressing the key').optional(),
+    keyName: z.string().describe("The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'")
+});
+const defineActionKeyboardPress = (call)=>defineAction({
+        name: 'KeyboardPress',
+        description: 'Press a key or key combination, like "Enter", "Tab", "Escape", or "Control+A", "Shift+Enter". Do not use this to type text.',
+        interfaceAlias: 'aiKeyboardPress',
+        paramSchema: actionKeyboardPressParamSchema,
+        call
+    });
+const actionScrollParamSchema = z.object({
+    direction: z["enum"]([
+        'down',
+        'up',
+        'right',
+        'left'
+    ]).default('down').describe('The direction to scroll'),
+    scrollType: z["enum"]([
+        'once',
+        'untilBottom',
+        'untilTop',
+        'untilRight',
+        'untilLeft'
+    ]).default('once').describe('The scroll type'),
+    distance: z.number().nullable().optional().describe('The distance in pixels to scroll'),
+    locate: getMidsceneLocationSchema().optional().describe('The element to be scrolled')
+});
+const defineActionScroll = (call)=>defineAction({
+        name: 'Scroll',
+        description: 'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',
+        interfaceAlias: 'aiScroll',
+        paramSchema: actionScrollParamSchema,
+        call
+    });
+const actionDragAndDropParamSchema = z.object({
+    from: getMidsceneLocationSchema().describe('The position to be dragged'),
+    to: getMidsceneLocationSchema().describe('The position to be dropped')
+});
+const defineActionDragAndDrop = (call)=>defineAction({
+        name: 'DragAndDrop',
+        description: 'Drag and drop the element',
+        interfaceAlias: 'aiDragAndDrop',
+        paramSchema: actionDragAndDropParamSchema,
+        call
+    });
+const ActionLongPressParamSchema = z.object({
+    locate: getMidsceneLocationSchema().describe('The element to be long pressed'),
+    duration: z.number().default(500).optional().describe('Long press duration in milliseconds')
+});
+const defineActionLongPress = (call)=>defineAction({
+        name: 'LongPress',
+        description: 'Long press the element',
+        paramSchema: ActionLongPressParamSchema,
+        call
+    });
+const ActionSwipeParamSchema = z.object({
+    start: getMidsceneLocationSchema().optional().describe('Starting point of the swipe gesture, if not specified, the center of the page will be used'),
+    direction: z["enum"]([
+        'up',
+        'down',
+        'left',
+        'right'
+    ]).optional().describe('The direction to swipe (required when using distance). The direction means the direction of the finger swipe.'),
+    distance: z.number().optional().describe('The distance in pixels to swipe (mutually exclusive with end)'),
+    end: getMidsceneLocationSchema().optional().describe('Ending point of the swipe gesture (mutually exclusive with distance)'),
+    duration: z.number().default(300).describe('Duration of the swipe gesture in milliseconds'),
+    repeat: z.number().optional().describe('The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)')
+});
+const defineActionSwipe = (call)=>defineAction({
+        name: 'Swipe',
+        description: 'Perform a swipe gesture. You must specify either "end" (target location) or "distance" + "direction" - they are mutually exclusive. Use "end" for precise location-based swipes, or "distance" + "direction" for relative movement.',
+        paramSchema: ActionSwipeParamSchema,
+        call
+    });
+const actionClearInputParamSchema = z.object({
+    locate: getMidsceneLocationSchema().describe('The input field to be cleared')
+});
+const defineActionClearInput = (call)=>defineAction({
+        name: 'ClearInput',
+        description: 'Clear the text content of an input field',
+        interfaceAlias: 'aiClearInput',
+        paramSchema: actionClearInputParamSchema,
+        call
+    });
+export { AbstractInterface, ActionLongPressParamSchema, ActionSwipeParamSchema, actionClearInputParamSchema, actionDoubleClickParamSchema, actionDragAndDropParamSchema, actionHoverParamSchema, actionInputParamSchema, actionKeyboardPressParamSchema, actionRightClickParamSchema, actionScrollParamSchema, actionTapParamSchema, defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionHover, defineActionInput, defineActionKeyboardPress, defineActionLongPress, defineActionRightClick, defineActionScroll, defineActionSwipe, defineActionTap };
+
+//# sourceMappingURL=index.mjs.map

package/dist/es/device/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"device\\index.mjs","sources":["webpack://@sqaitech/core/./src/device/index.ts"],"sourcesContent":["import { getMidsceneLocationSchema } from '@/ai-model';\r\nimport type { DeviceAction, LocateResultElement } from '@/types';\r\nimport type { ElementNode } from '@sqaitech/shared/extractor';\r\nimport { _keyDefinitions } from '@sqaitech/shared/us-keyboard-layout';\r\nimport { z } from 'zod';\r\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\r\n\r\nexport abstract class AbstractInterface {\r\n  abstract interfaceType: string;\r\n\r\n  abstract screenshotBase64(): Promise<string>;\r\n  abstract size(): Promise<Size>;\r\n  abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;\r\n\r\n  abstract cacheFeatureForRect?(\r\n    rect: Rect,\r\n    opt?: { _orderSensitive: boolean },\r\n  ): Promise<ElementCacheFeature>;\r\n  abstract rectMatchesCacheFeature?(\r\n    feature: ElementCacheFeature,\r\n  ): Promise<Rect>;\r\n\r\n  abstract destroy?(): Promise<void>;\r\n\r\n  abstract describe?(): string;\r\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\r\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\r\n\r\n  // @deprecated do NOT extend this method\r\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\r\n\r\n  // @deprecated do NOT extend this method\r\n  abstract url?: () => string | Promise<string>;\r\n\r\n  // @deprecated do NOT extend this method\r\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\r\n\r\n  // @deprecated do NOT extend this method\r\n  abstract getContext?(): Promise<UIContext>;\r\n}\r\n\r\n// Generic function to define actions with proper type inference\r\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\r\nexport const defineAction = <\r\n  TSchema extends z.ZodType,\r\n  TRuntime = z.infer<TSchema>,\r\n>(\r\n  config: {\r\n    name: string;\r\n    description: string;\r\n    interfaceAlias?: string;\r\n    paramSchema: TSchema;\r\n    call: (param: TRuntime) => Promise<void>;\r\n  } & Partial<\r\n    Omit<\r\n      DeviceAction<TRuntime>,\r\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\r\n    >\r\n  >,\r\n): DeviceAction<TRuntime> => {\r\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\r\n};\r\n\r\n// Tap\r\nexport const actionTapParamSchema = z.object({\r\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\r\n});\r\n// Override the inferred type to use LocateResultElement for the runtime locate field\r\nexport type ActionTapParam = {\r\n  locate: LocateResultElement;\r\n};\r\n\r\nexport const defineActionTap = (\r\n  call: (param: ActionTapParam) => Promise<void>,\r\n): DeviceAction<ActionTapParam> => {\r\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\r\n    name: 'Tap',\r\n    description: 'Tap the element',\r\n    interfaceAlias: 'aiTap',\r\n    paramSchema: actionTapParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\n// RightClick\r\nexport const actionRightClickParamSchema = z.object({\r\n  locate: getMidsceneLocationSchema().describe(\r\n    'The element to be right clicked',\r\n  ),\r\n});\r\nexport type ActionRightClickParam = {\r\n  locate: LocateResultElement;\r\n};\r\n\r\nexport const defineActionRightClick = (\r\n  call: (param: ActionRightClickParam) => Promise<void>,\r\n): DeviceAction<ActionRightClickParam> => {\r\n  return defineAction<\r\n    typeof actionRightClickParamSchema,\r\n    ActionRightClickParam\r\n  >({\r\n    name: 'RightClick',\r\n    description: 'Right click the element',\r\n    interfaceAlias: 'aiRightClick',\r\n    paramSchema: actionRightClickParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\n// DoubleClick\r\nexport const actionDoubleClickParamSchema = z.object({\r\n  locate: getMidsceneLocationSchema().describe(\r\n    'The element to be double clicked',\r\n  ),\r\n});\r\nexport type ActionDoubleClickParam = {\r\n  locate: LocateResultElement;\r\n};\r\n\r\nexport const defineActionDoubleClick = (\r\n  call: (param: ActionDoubleClickParam) => Promise<void>,\r\n): DeviceAction<ActionDoubleClickParam> => {\r\n  return defineAction<\r\n    typeof actionDoubleClickParamSchema,\r\n    ActionDoubleClickParam\r\n  >({\r\n    name: 'DoubleClick',\r\n    description: 'Double click the element',\r\n    interfaceAlias: 'aiDoubleClick',\r\n    paramSchema: actionDoubleClickParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\n// Hover\r\nexport const actionHoverParamSchema = z.object({\r\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\r\n});\r\nexport type ActionHoverParam = {\r\n  locate: LocateResultElement;\r\n};\r\n\r\nexport const defineActionHover = (\r\n  call: (param: ActionHoverParam) => Promise<void>,\r\n): DeviceAction<ActionHoverParam> => {\r\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\r\n    name: 'Hover',\r\n    description: 'Move the mouse to the element',\r\n    interfaceAlias: 'aiHover',\r\n    paramSchema: actionHoverParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\n// Input\r\nexport const actionInputParamSchema = z.object({\r\n  value: z\r\n    .union([z.string(), z.number()])\r\n    .transform((val) => String(val))\r\n    .describe(\r\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\r\n    ),\r\n  locate: getMidsceneLocationSchema()\r\n    .describe('The element to be input')\r\n    .optional(),\r\n  mode: z\r\n    .enum(['replace', 'clear', 'append'])\r\n    .default('replace')\r\n    .optional()\r\n    .describe(\r\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"append\" - append the value to existing content; \"clear\" - clear the field without inputting new text.',\r\n    ),\r\n});\r\nexport type ActionInputParam = {\r\n  value: string;\r\n  locate?: LocateResultElement;\r\n  mode?: 'replace' | 'clear' | 'append';\r\n};\r\n\r\nexport const defineActionInput = (\r\n  call: (param: ActionInputParam) => Promise<void>,\r\n): DeviceAction<ActionInputParam> => {\r\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\r\n    name: 'Input',\r\n    description: 'Input the value into the element',\r\n    interfaceAlias: 'aiInput',\r\n    paramSchema: actionInputParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\n// KeyboardPress\r\nexport const actionKeyboardPressParamSchema = z.object({\r\n  locate: getMidsceneLocationSchema()\r\n    .describe('The element to be clicked before pressing the key')\r\n    .optional(),\r\n  keyName: z\r\n    .string()\r\n    .describe(\r\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\r\n    ),\r\n});\r\nexport type ActionKeyboardPressParam = {\r\n  locate?: LocateResultElement;\r\n  keyName: string;\r\n};\r\n\r\nexport const defineActionKeyboardPress = (\r\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\r\n): DeviceAction<ActionKeyboardPressParam> => {\r\n  return defineAction<\r\n    typeof actionKeyboardPressParamSchema,\r\n    ActionKeyboardPressParam\r\n  >({\r\n    name: 'KeyboardPress',\r\n    description:\r\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\r\n    interfaceAlias: 'aiKeyboardPress',\r\n    paramSchema: actionKeyboardPressParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\n// Scroll\r\nexport const actionScrollParamSchema = z.object({\r\n  direction: z\r\n    .enum(['down', 'up', 'right', 'left'])\r\n    .default('down')\r\n    .describe('The direction to scroll'),\r\n  scrollType: z\r\n    .enum(['once', 'untilBottom', 'untilTop', 'untilRight', 'untilLeft'])\r\n    .default('once')\r\n    .describe('The scroll type'),\r\n  distance: z\r\n    .number()\r\n    .nullable()\r\n    .optional()\r\n    .describe('The distance in pixels to scroll'),\r\n  locate: getMidsceneLocationSchema()\r\n    .optional()\r\n    .describe('The element to be scrolled'),\r\n});\r\nexport type ActionScrollParam = {\r\n  direction?: 'down' | 'up' | 'right' | 'left';\r\n  scrollType?: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';\r\n  distance?: number | null;\r\n  locate?: LocateResultElement;\r\n};\r\n\r\nexport const defineActionScroll = (\r\n  call: (param: ActionScrollParam) => Promise<void>,\r\n): DeviceAction<ActionScrollParam> => {\r\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\r\n    name: 'Scroll',\r\n    description:\r\n      'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',\r\n    interfaceAlias: 'aiScroll',\r\n    paramSchema: actionScrollParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\n// DragAndDrop\r\nexport const actionDragAndDropParamSchema = z.object({\r\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\r\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\r\n});\r\nexport type ActionDragAndDropParam = {\r\n  from: LocateResultElement;\r\n  to: LocateResultElement;\r\n};\r\n\r\nexport const defineActionDragAndDrop = (\r\n  call: (param: ActionDragAndDropParam) => Promise<void>,\r\n): DeviceAction<ActionDragAndDropParam> => {\r\n  return defineAction<\r\n    typeof actionDragAndDropParamSchema,\r\n    ActionDragAndDropParam\r\n  >({\r\n    name: 'DragAndDrop',\r\n    description: 'Drag and drop the element',\r\n    interfaceAlias: 'aiDragAndDrop',\r\n    paramSchema: actionDragAndDropParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\nexport const ActionLongPressParamSchema = z.object({\r\n  locate: getMidsceneLocationSchema().describe(\r\n    'The element to be long pressed',\r\n  ),\r\n  duration: z\r\n    .number()\r\n    .default(500)\r\n    .optional()\r\n    .describe('Long press duration in milliseconds'),\r\n});\r\n\r\nexport type ActionLongPressParam = {\r\n  locate: LocateResultElement;\r\n  duration?: number;\r\n};\r\nexport const defineActionLongPress = (\r\n  call: (param: ActionLongPressParam) => Promise<void>,\r\n): DeviceAction<ActionLongPressParam> => {\r\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\r\n    name: 'LongPress',\r\n    description: 'Long press the element',\r\n    paramSchema: ActionLongPressParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\nexport const ActionSwipeParamSchema = z.object({\r\n  start: getMidsceneLocationSchema()\r\n    .optional()\r\n    .describe(\r\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\r\n    ),\r\n  direction: z\r\n    .enum(['up', 'down', 'left', 'right'])\r\n    .optional()\r\n    .describe(\r\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\r\n    ),\r\n  distance: z\r\n    .number()\r\n    .optional()\r\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\r\n  end: getMidsceneLocationSchema()\r\n    .optional()\r\n    .describe(\r\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\r\n    ),\r\n  duration: z\r\n    .number()\r\n    .default(300)\r\n    .describe('Duration of the swipe gesture in milliseconds'),\r\n  repeat: z\r\n    .number()\r\n    .optional()\r\n    .describe(\r\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\r\n    ),\r\n});\r\n\r\nexport type ActionSwipeParam = {\r\n  start?: LocateResultElement;\r\n  direction?: 'up' | 'down' | 'left' | 'right';\r\n  distance?: number;\r\n  end?: LocateResultElement;\r\n  duration?: number;\r\n  repeat?: number;\r\n};\r\n\r\nexport const defineActionSwipe = (\r\n  call: (param: ActionSwipeParam) => Promise<void>,\r\n): DeviceAction<ActionSwipeParam> => {\r\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\r\n    name: 'Swipe',\r\n    description:\r\n      'Perform a swipe gesture. You must specify either \"end\" (target location) or \"distance\" + \"direction\" - they are mutually exclusive. Use \"end\" for precise location-based swipes, or \"distance\" + \"direction\" for relative movement.',\r\n    paramSchema: ActionSwipeParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\n// ClearInput\r\nexport const actionClearInputParamSchema = z.object({\r\n  locate: getMidsceneLocationSchema().describe('The input field to be cleared'),\r\n});\r\nexport type ActionClearInputParam = {\r\n  locate: LocateResultElement;\r\n};\r\n\r\nexport const defineActionClearInput = (\r\n  call: (param: ActionClearInputParam) => Promise<void>,\r\n): DeviceAction<ActionClearInputParam> => {\r\n  return defineAction<\r\n    typeof actionClearInputParamSchema,\r\n    ActionClearInputParam\r\n  >({\r\n    name: 'ClearInput',\r\n    description: 'Clear the text content of an input field',\r\n    interfaceAlias: 'aiClearInput',\r\n    paramSchema: actionClearInputParamSchema,\r\n    call,\r\n  });\r\n};\r\n\r\nexport type { DeviceAction } from '../types';\r\n"],"names":["AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","actionInputParamSchema","val","String","defineActionInput","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput"],"mappings":";;AAOO,MAAeA;AAgCtB;AAIO,MAAMC,eAAe,CAI1BC,SAaOA;AAIF,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAMO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACbI;IACF;AAIK,MAAMC,8BAA8BJ,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACbD;IACF;AAIK,MAAMG,+BAA+BN,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACbH;IACF;AAIK,MAAMK,yBAAyBR,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACbL;IACF;AAIK,MAAMO,yBAAyBV,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACW,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQV,4BACL,QAAQ,CAAC,2BACT,QAAQ;IACX,MAAMD,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAS,EACnC,OAAO,CAAC,WACR,QAAQ,GACR,QAAQ,CACP;AAEN;AAOO,MAAMa,oBAAoB,CAC/BV,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaa;QACbP;IACF;AAIK,MAAMW,iCAAiCd,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMe,4BAA4B,CACvCZ,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaiB;QACbX;IACF;AAIK,MAAMa,0BAA0BhB,EAAE,MAAM,CAAC;IAC9C,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,YAAYA,CAAC,CAADA,OACL,CAAC;QAAC;QAAQ;QAAe;QAAY;QAAc;KAAY,EACnE,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CAAC;AACd;AAQO,MAAMgB,qBAAqB,CAChCd,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACbb;IACF;AAIK,MAAMe,+BAA+BlB,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMkB,0BAA0B,CACrChB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaqB;QACbf;IACF;AAGK,MAAMiB,6BAA6BpB,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMqB,wBAAwB,CACnClB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAauB;QACbjB;IACF;AAGK,MAAMmB,yBAAyBtB,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,MAAMuB,oBAAoB,CAC/BpB,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAayB;QACbnB;IACF;AAIK,MAAMqB,8BAA8BxB,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMwB,yBAAyB,CACpCtB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAa2B;QACbrB;IACF"}

package/dist/es/image/index.mjs

@@ -0,0 +1,2 @@
+import { httpImg2Base64, imageInfo, imageInfoOfBase64, localImg2Base64, resizeAndConvertImgBuffer, saveBase64Image, zoomForGPT4o } from "@sqaitech/shared/img";
+export { httpImg2Base64, imageInfo, imageInfoOfBase64, localImg2Base64, resizeAndConvertImgBuffer, saveBase64Image, zoomForGPT4o };

package/dist/es/index.mjs

@@ -0,0 +1,11 @@
+import { z } from "zod";
+import { Executor } from "./ai-model/action-executor.mjs";
+import insight from "./insight/index.mjs";
+import { getVersion } from "./utils.mjs";
+import { AiLocateElement, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, describeUserPage, getMidsceneLocationSchema, plan } from "./ai-model/index.mjs";
+import { SQAI_MODEL_NAME } from "@sqaitech/shared/env";
+import { Agent, createAgent } from "./agent/index.mjs";
+const src = insight;
+export { Agent, AiLocateElement, Executor, insight as Insight, PointSchema, RectSchema, SQAI_MODEL_NAME, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, createAgent, src as default, describeUserPage, getMidsceneLocationSchema, getVersion, plan, z };
+
+//# sourceMappingURL=index.mjs.map

package/dist/es/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","sources":["webpack://@sqaitech/core/./src/index.ts"],"sourcesContent":["import { z } from 'zod';\r\nimport { Executor } from './ai-model/action-executor';\r\nimport Insight from './insight/index';\r\nimport { getVersion } from './utils';\r\n\r\nexport {\r\n  plan,\r\n  describeUserPage,\r\n  AiLocateElement,\r\n  getMidsceneLocationSchema,\r\n  type MidsceneLocationResultType,\r\n  PointSchema,\r\n  SizeSchema,\r\n  RectSchema,\r\n  TMultimodalPromptSchema,\r\n  TUserPromptSchema,\r\n  type TMultimodalPrompt,\r\n  type TUserPrompt,\r\n} from './ai-model/index';\r\n\r\nexport { SQAI_MODEL_NAME } from '@sqaitech/shared/env';\r\n\r\nexport type * from './types';\r\n\r\nexport { z };\r\n\r\nexport default Insight;\r\nexport { Executor, Insight, getVersion };\r\n\r\nexport type {\r\n  MidsceneYamlScript,\r\n  MidsceneYamlTask,\r\n  MidsceneYamlFlowItem,\r\n  MidsceneYamlConfigResult,\r\n  MidsceneYamlConfig,\r\n  MidsceneYamlScriptWebEnv,\r\n  MidsceneYamlScriptAndroidEnv,\r\n  MidsceneYamlScriptIOSEnv,\r\n  MidsceneYamlScriptEnv,\r\n  LocateOption,\r\n  DetailedLocateParam,\r\n} from './yaml';\r\n\r\nexport { Agent, type AgentOpt, createAgent } from './agent';\r\n"],"names":["Insight"],"mappings":";;;;;;;AA0BA,YAAeA"}

package/dist/es/insight/index.mjs

@@ -0,0 +1,231 @@
+import { AIActionType, expandSearchArea } from "../ai-model/common.mjs";
+import { AiExtractElementInfo, AiLocateElement, callAIWithObjectResponse } from "../ai-model/index.mjs";
+import { AiLocateSection } from "../ai-model/inspect.mjs";
+import { elementDescriberInstruction } from "../ai-model/prompt/describe.mjs";
+import { SQAI_FORCE_DEEP_THINK, globalConfigManager } from "@sqaitech/shared/env";
+import { compositeElementInfoImg, cropByRect } from "@sqaitech/shared/img";
+import { getDebug } from "@sqaitech/shared/logger";
+import { assert } from "@sqaitech/shared/utils";
+import { emitInsightDump } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('ai:insight');
+class Insight {
+    async locate(query, opt, modelConfig) {
+        var _parseResult_errors;
+        const queryPrompt = 'string' == typeof query ? query : query.prompt;
+        assert(queryPrompt, 'query is required for locate');
+        const dumpSubscriber = this.onceDumpUpdatedFn;
+        this.onceDumpUpdatedFn = void 0;
+        assert('object' == typeof query, 'query should be an object for locate');
+        const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(SQAI_FORCE_DEEP_THINK);
+        if (globalDeepThinkSwitch) debug('globalDeepThinkSwitch', globalDeepThinkSwitch);
+        let searchAreaPrompt;
+        if (query.deepThink || globalDeepThinkSwitch) searchAreaPrompt = query.prompt;
+        const { vlMode } = modelConfig;
+        if (searchAreaPrompt && !vlMode) {
+            console.warn('The "deepThink" feature is not supported with multimodal LLM. Please config VL model for SQAI. https://sqai.tech/choose-a-model');
+            searchAreaPrompt = void 0;
+        }
+        const context = (null == opt ? void 0 : opt.context) || await this.contextRetrieverFn('locate');
+        let searchArea;
+        let searchAreaRawResponse;
+        let searchAreaUsage;
+        let searchAreaResponse;
+        if (searchAreaPrompt) {
+            searchAreaResponse = await AiLocateSection({
+                context,
+                sectionDescription: searchAreaPrompt,
+                modelConfig
+            });
+            assert(searchAreaResponse.rect, `cannot find search area for "${searchAreaPrompt}"${searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''}`);
+            searchAreaRawResponse = searchAreaResponse.rawResponse;
+            searchAreaUsage = searchAreaResponse.usage;
+            searchArea = searchAreaResponse.rect;
+        }
+        const startTime = Date.now();
+        const { parseResult, rect, elementById, rawResponse, usage, isOrderSensitive } = await AiLocateElement({
+            callAIFn: this.aiVendorFn,
+            context,
+            targetElementDescription: queryPrompt,
+            searchConfig: searchAreaResponse,
+            modelConfig
+        });
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse: JSON.stringify(rawResponse),
+            formatResponse: JSON.stringify(parseResult),
+            usage,
+            searchArea,
+            searchAreaRawResponse,
+            searchAreaUsage
+        };
+        let errorLog;
+        if (null == (_parseResult_errors = parseResult.errors) ? void 0 : _parseResult_errors.length) errorLog = `AI model failed to locate: \n${parseResult.errors.join('\n')}`;
+        const dumpData = {
+            type: 'locate',
+            userQuery: {
+                element: queryPrompt
+            },
+            matchedElement: [],
+            matchedRect: rect,
+            data: null,
+            taskInfo,
+            deepThink: !!searchArea,
+            error: errorLog
+        };
+        const elements = [];
+        (parseResult.elements || []).forEach((item)=>{
+            if ('id' in item) {
+                const element = elementById(null == item ? void 0 : item.id);
+                if (!element) return void console.warn(`locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`);
+                elements.push(element);
+            }
+        });
+        emitInsightDump({
+            ...dumpData,
+            matchedElement: elements
+        }, dumpSubscriber);
+        if (errorLog) throw new Error(errorLog);
+        assert(elements.length <= 1, `locate: multiple elements found, length = ${elements.length}`);
+        if (1 === elements.length) return {
+            element: {
+                id: elements[0].id,
+                indexId: elements[0].indexId,
+                center: elements[0].center,
+                rect: elements[0].rect,
+                xpaths: elements[0].xpaths || [],
+                attributes: elements[0].attributes,
+                isOrderSensitive
+            },
+            rect
+        };
+        return {
+            element: null,
+            rect
+        };
+    }
+    async extract(dataDemand, modelConfig, opt, multimodalPrompt) {
+        var _parseResult_errors;
+        assert('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
+        const dumpSubscriber = this.onceDumpUpdatedFn;
+        this.onceDumpUpdatedFn = void 0;
+        const context = await this.contextRetrieverFn('extract');
+        const startTime = Date.now();
+        const { parseResult, usage } = await AiExtractElementInfo({
+            context,
+            dataQuery: dataDemand,
+            multimodalPrompt,
+            extractOption: opt,
+            modelConfig
+        });
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse: JSON.stringify(parseResult)
+        };
+        let errorLog;
+        if (null == (_parseResult_errors = parseResult.errors) ? void 0 : _parseResult_errors.length) errorLog = `AI response error: \n${parseResult.errors.join('\n')}`;
+        const dumpData = {
+            type: 'extract',
+            userQuery: {
+                dataDemand
+            },
+            matchedElement: [],
+            data: null,
+            taskInfo,
+            error: errorLog
+        };
+        const { data, thought } = parseResult || {};
+        emitInsightDump({
+            ...dumpData,
+            data
+        }, dumpSubscriber);
+        if (errorLog && !data && !(null == opt ? void 0 : opt.doNotThrowError)) throw new Error(errorLog);
+        return {
+            data,
+            thought,
+            usage
+        };
+    }
+    async describe(target, modelConfig, opt) {
+        assert(target, 'target is required for insight.describe');
+        const context = await this.contextRetrieverFn('describe');
+        const { screenshotBase64, size } = context;
+        assert(screenshotBase64, 'screenshot is required for insight.describe');
+        const { vlMode } = modelConfig;
+        const systemPrompt = elementDescriberInstruction();
+        const defaultRectSize = 30;
+        const targetRect = Array.isArray(target) ? {
+            left: Math.floor(target[0] - defaultRectSize / 2),
+            top: Math.floor(target[1] - defaultRectSize / 2),
+            width: defaultRectSize,
+            height: defaultRectSize
+        } : target;
+        let imagePayload = await compositeElementInfoImg({
+            inputImgBase64: screenshotBase64,
+            size,
+            elementsPositionInfo: [
+                {
+                    rect: targetRect
+                }
+            ],
+            borderThickness: 3
+        });
+        if (null == opt ? void 0 : opt.deepThink) {
+            const searchArea = expandSearchArea(targetRect, context.size, vlMode);
+            debug('describe: set searchArea', searchArea);
+            const croppedResult = await cropByRect(imagePayload, searchArea, 'qwen-vl' === vlMode);
+            imagePayload = croppedResult.imageBase64;
+        }
+        const msgs = [
+            {
+                role: 'system',
+                content: systemPrompt
+            },
+            {
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: imagePayload,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            }
+        ];
+        const callAIFn = this.aiVendorFn;
+        const res = await callAIFn(msgs, AIActionType.DESCRIBE_ELEMENT, modelConfig);
+        const { content } = res;
+        assert(!content.error, `describe failed: ${content.error}`);
+        assert(content.description, 'failed to describe the element');
+        return content;
+    }
+    constructor(context, opt){
+        _define_property(this, "contextRetrieverFn", void 0);
+        _define_property(this, "aiVendorFn", callAIWithObjectResponse);
+        _define_property(this, "onceDumpUpdatedFn", void 0);
+        _define_property(this, "taskInfo", void 0);
+        assert(context, 'context is required for Insight');
+        if ('function' == typeof context) this.contextRetrieverFn = context;
+        else this.contextRetrieverFn = ()=>Promise.resolve(context);
+        if (void 0 !== (null == opt ? void 0 : opt.aiVendorFn)) this.aiVendorFn = opt.aiVendorFn;
+        if (void 0 !== (null == opt ? void 0 : opt.taskInfo)) this.taskInfo = opt.taskInfo;
+    }
+}
+export { Insight as default };
+
+//# sourceMappingURL=index.mjs.map

package/dist/es/insight/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"insight\\index.mjs","sources":["webpack://@sqaitech/core/./src/insight/index.ts"],"sourcesContent":["import { AIActionType, type AIArgs, expandSearchArea } from '@/ai-model/common';\r\nimport {\r\n  AiExtractElementInfo,\r\n  AiLocateElement,\r\n  callAIWithObjectResponse,\r\n} from '@/ai-model/index';\r\nimport { AiLocateSection } from '@/ai-model/inspect';\r\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\r\nimport type {\r\n  AIDescribeElementResponse,\r\n  AIUsageInfo,\r\n  BaseElement,\r\n  DetailedLocateParam,\r\n  DumpSubscriber,\r\n  InsightAction,\r\n  InsightExtractOption,\r\n  InsightExtractParam,\r\n  InsightTaskInfo,\r\n  LocateResult,\r\n  PartialInsightDumpFromSDK,\r\n  Rect,\r\n  UIContext,\r\n} from '@/types';\r\nimport {\r\n  type IModelConfig,\r\n  SQAI_FORCE_DEEP_THINK,\r\n  globalConfigManager,\r\n} from '@sqaitech/shared/env';\r\nimport { compositeElementInfoImg, cropByRect } from '@sqaitech/shared/img';\r\nimport { getDebug } from '@sqaitech/shared/logger';\r\nimport { assert } from '@sqaitech/shared/utils';\r\nimport type { TMultimodalPrompt } from '../ai-model/common';\r\nimport { emitInsightDump } from './utils';\r\n\r\nexport interface LocateOpts {\r\n  context?: UIContext<BaseElement>;\r\n}\r\n\r\nexport type AnyValue<T> = {\r\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\r\n};\r\n\r\ninterface InsightOptions {\r\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\r\n  aiVendorFn?: typeof callAIWithObjectResponse;\r\n}\r\n\r\nconst debug = getDebug('ai:insight');\r\nexport default class Insight<\r\n  ElementType extends BaseElement = BaseElement,\r\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\r\n> {\r\n  contextRetrieverFn: (\r\n    action: InsightAction,\r\n  ) => Promise<ContextType> | ContextType;\r\n\r\n  aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined> =\r\n    callAIWithObjectResponse;\r\n\r\n  onceDumpUpdatedFn?: DumpSubscriber;\r\n\r\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\r\n\r\n  constructor(\r\n    context:\r\n      | ContextType\r\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\r\n    opt?: InsightOptions,\r\n  ) {\r\n    assert(context, 'context is required for Insight');\r\n    if (typeof context === 'function') {\r\n      this.contextRetrieverFn = context;\r\n    } else {\r\n      this.contextRetrieverFn = () => Promise.resolve(context);\r\n    }\r\n\r\n    // just for unit test, aiVendorFn is callAIWithObjectResponse by default\r\n    if (typeof opt?.aiVendorFn !== 'undefined') {\r\n      this.aiVendorFn = opt.aiVendorFn;\r\n    }\r\n    if (typeof opt?.taskInfo !== 'undefined') {\r\n      this.taskInfo = opt.taskInfo;\r\n    }\r\n  }\r\n\r\n  async locate(\r\n    query: DetailedLocateParam,\r\n    opt: LocateOpts,\r\n    modelConfig: IModelConfig,\r\n  ): Promise<LocateResult> {\r\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\r\n    assert(queryPrompt, 'query is required for locate');\r\n    const dumpSubscriber = this.onceDumpUpdatedFn;\r\n    this.onceDumpUpdatedFn = undefined;\r\n\r\n    assert(typeof query === 'object', 'query should be an object for locate');\r\n\r\n    const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(\r\n      SQAI_FORCE_DEEP_THINK,\r\n    );\r\n    if (globalDeepThinkSwitch) {\r\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\r\n    }\r\n    let searchAreaPrompt;\r\n    if (query.deepThink || globalDeepThinkSwitch) {\r\n      searchAreaPrompt = query.prompt;\r\n    }\r\n\r\n    const { vlMode } = modelConfig;\r\n\r\n    if (searchAreaPrompt && !vlMode) {\r\n      console.warn(\r\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for SQAI. https://sqai.tech/choose-a-model',\r\n      );\r\n      searchAreaPrompt = undefined;\r\n    }\r\n\r\n    const context = opt?.context || (await this.contextRetrieverFn('locate'));\r\n\r\n    let searchArea: Rect | undefined = undefined;\r\n    let searchAreaRawResponse: string | undefined = undefined;\r\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\r\n    let searchAreaResponse:\r\n      | Awaited<ReturnType<typeof AiLocateSection>>\r\n      | undefined = undefined;\r\n    if (searchAreaPrompt) {\r\n      searchAreaResponse = await AiLocateSection({\r\n        context,\r\n        sectionDescription: searchAreaPrompt,\r\n        modelConfig,\r\n      });\r\n      assert(\r\n        searchAreaResponse.rect,\r\n        `cannot find search area for \"${searchAreaPrompt}\"${\r\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\r\n        }`,\r\n      );\r\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\r\n      searchAreaUsage = searchAreaResponse.usage;\r\n      searchArea = searchAreaResponse.rect;\r\n    }\r\n\r\n    const startTime = Date.now();\r\n    const {\r\n      parseResult,\r\n      rect,\r\n      elementById,\r\n      rawResponse,\r\n      usage,\r\n      isOrderSensitive,\r\n    } = await AiLocateElement({\r\n      callAIFn: this.aiVendorFn,\r\n      context,\r\n      targetElementDescription: queryPrompt,\r\n      searchConfig: searchAreaResponse,\r\n      modelConfig,\r\n    });\r\n\r\n    const timeCost = Date.now() - startTime;\r\n    const taskInfo: InsightTaskInfo = {\r\n      ...(this.taskInfo ? this.taskInfo : {}),\r\n      durationMs: timeCost,\r\n      rawResponse: JSON.stringify(rawResponse),\r\n      formatResponse: JSON.stringify(parseResult),\r\n      usage,\r\n      searchArea,\r\n      searchAreaRawResponse,\r\n      searchAreaUsage,\r\n    };\r\n\r\n    let errorLog: string | undefined;\r\n    if (parseResult.errors?.length) {\r\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\r\n    }\r\n\r\n    const dumpData: PartialInsightDumpFromSDK = {\r\n      type: 'locate',\r\n      userQuery: {\r\n        element: queryPrompt,\r\n      },\r\n      matchedElement: [],\r\n      matchedRect: rect,\r\n      data: null,\r\n      taskInfo,\r\n      deepThink: !!searchArea,\r\n      error: errorLog,\r\n    };\r\n\r\n    const elements: BaseElement[] = [];\r\n    (parseResult.elements || []).forEach((item) => {\r\n      if ('id' in item) {\r\n        const element = elementById(item?.id);\r\n\r\n        if (!element) {\r\n          console.warn(\r\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\r\n          );\r\n          return;\r\n        }\r\n        elements.push(element);\r\n      }\r\n    });\r\n\r\n    emitInsightDump(\r\n      {\r\n        ...dumpData,\r\n        matchedElement: elements,\r\n      },\r\n      dumpSubscriber,\r\n    );\r\n\r\n    if (errorLog) {\r\n      throw new Error(errorLog);\r\n    }\r\n\r\n    assert(\r\n      elements.length <= 1,\r\n      `locate: multiple elements found, length = ${elements.length}`,\r\n    );\r\n\r\n    if (elements.length === 1) {\r\n      return {\r\n        element: {\r\n          id: elements[0]!.id,\r\n          indexId: elements[0]!.indexId,\r\n          center: elements[0]!.center,\r\n          rect: elements[0]!.rect,\r\n          xpaths: elements[0]!.xpaths || [],\r\n          attributes: elements[0]!.attributes,\r\n          isOrderSensitive,\r\n        },\r\n        rect,\r\n      };\r\n    }\r\n    return {\r\n      element: null,\r\n      rect,\r\n    };\r\n  }\r\n\r\n  async extract<T>(\r\n    dataDemand: InsightExtractParam,\r\n    modelConfig: IModelConfig,\r\n    opt?: InsightExtractOption,\r\n    multimodalPrompt?: TMultimodalPrompt,\r\n  ): Promise<{\r\n    data: T;\r\n    thought?: string;\r\n    usage?: AIUsageInfo;\r\n  }> {\r\n    assert(\r\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\r\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\r\n    );\r\n    const dumpSubscriber = this.onceDumpUpdatedFn;\r\n    this.onceDumpUpdatedFn = undefined;\r\n\r\n    const context = await this.contextRetrieverFn('extract');\r\n\r\n    const startTime = Date.now();\r\n\r\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\r\n      context,\r\n      dataQuery: dataDemand,\r\n      multimodalPrompt,\r\n      extractOption: opt,\r\n      modelConfig,\r\n    });\r\n\r\n    const timeCost = Date.now() - startTime;\r\n    const taskInfo: InsightTaskInfo = {\r\n      ...(this.taskInfo ? this.taskInfo : {}),\r\n      durationMs: timeCost,\r\n      rawResponse: JSON.stringify(parseResult),\r\n    };\r\n\r\n    let errorLog: string | undefined;\r\n    if (parseResult.errors?.length) {\r\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\r\n    }\r\n\r\n    const dumpData: PartialInsightDumpFromSDK = {\r\n      type: 'extract',\r\n      userQuery: {\r\n        dataDemand,\r\n      },\r\n      matchedElement: [],\r\n      data: null,\r\n      taskInfo,\r\n      error: errorLog,\r\n    };\r\n\r\n    const { data, thought } = parseResult || {};\r\n\r\n    // 4\r\n    emitInsightDump(\r\n      {\r\n        ...dumpData,\r\n        data,\r\n      },\r\n      dumpSubscriber,\r\n    );\r\n\r\n    if (errorLog && !data && !opt?.doNotThrowError) {\r\n      throw new Error(errorLog);\r\n    }\r\n\r\n    return {\r\n      data,\r\n      thought,\r\n      usage,\r\n    };\r\n  }\r\n\r\n  async describe(\r\n    target: Rect | [number, number],\r\n    modelConfig: IModelConfig,\r\n    opt?: {\r\n      deepThink?: boolean;\r\n    },\r\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\r\n    assert(target, 'target is required for insight.describe');\r\n    const context = await this.contextRetrieverFn('describe');\r\n    const { screenshotBase64, size } = context;\r\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\r\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\r\n    const { vlMode } = modelConfig;\r\n    const systemPrompt = elementDescriberInstruction();\r\n\r\n    // Convert [x,y] center point to Rect if needed\r\n    const defaultRectSize = 30;\r\n    const targetRect: Rect = Array.isArray(target)\r\n      ? {\r\n          left: Math.floor(target[0] - defaultRectSize / 2),\r\n          top: Math.floor(target[1] - defaultRectSize / 2),\r\n          width: defaultRectSize,\r\n          height: defaultRectSize,\r\n        }\r\n      : target;\r\n\r\n    let imagePayload = await compositeElementInfoImg({\r\n      inputImgBase64: screenshotBase64,\r\n      size,\r\n      elementsPositionInfo: [\r\n        {\r\n          rect: targetRect,\r\n        },\r\n      ],\r\n      borderThickness: 3,\r\n    });\r\n\r\n    if (opt?.deepThink) {\r\n      const searchArea = expandSearchArea(targetRect, context.size, vlMode);\r\n      debug('describe: set searchArea', searchArea);\r\n      const croppedResult = await cropByRect(\r\n        imagePayload,\r\n        searchArea,\r\n        vlMode === 'qwen-vl',\r\n      );\r\n      imagePayload = croppedResult.imageBase64;\r\n    }\r\n\r\n    const msgs: AIArgs = [\r\n      { role: 'system', content: systemPrompt },\r\n      {\r\n        role: 'user',\r\n        content: [\r\n          {\r\n            type: 'image_url',\r\n            image_url: {\r\n              url: imagePayload,\r\n              detail: 'high',\r\n            },\r\n          },\r\n        ],\r\n      },\r\n    ];\r\n\r\n    const callAIFn = this\r\n      .aiVendorFn as typeof callAIWithObjectResponse<AIDescribeElementResponse>;\r\n\r\n    const res = await callAIFn(\r\n      msgs,\r\n      AIActionType.DESCRIBE_ELEMENT,\r\n      modelConfig,\r\n    );\r\n\r\n    const { content } = res;\r\n    assert(!content.error, `describe failed: ${content.error}`);\r\n    assert(content.description, 'failed to describe the element');\r\n    return content;\r\n  }\r\n}\r\n"],"names":["debug","getDebug","Insight","query","opt","modelConfig","_parseResult_errors","queryPrompt","assert","dumpSubscriber","undefined","globalDeepThinkSwitch","globalConfigManager","SQAI_FORCE_DEEP_THINK","searchAreaPrompt","vlMode","console","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","elementById","rawResponse","usage","isOrderSensitive","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","item","element","emitInsightDump","Error","dataDemand","multimodalPrompt","AiExtractElementInfo","data","thought","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","callAIFn","res","AIActionType","content","callAIWithObjectResponse","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;AA+CA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IAqCnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACF;YAkFnBC;QAjFJ,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QACpB,MAAME,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzBF,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMQ,wBAAwBC,oBAAoB,qBAAqB,CACrEC;QAEF,IAAIF,uBACFX,MAAM,yBAAyBW;QAEjC,IAAIG;QACJ,IAAIX,MAAM,SAAS,IAAIQ,uBACrBG,mBAAmBX,MAAM,MAAM;QAGjC,MAAM,EAAEY,MAAM,EAAE,GAAGV;QAEnB,IAAIS,oBAAoB,CAACC,QAAQ;YAC/BC,QAAQ,IAAI,CACV;YAEFF,mBAAmBJ;QACrB;QAEA,MAAMO,UAAUb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAM,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAE/D,IAAIc;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIP,kBAAkB;YACpBO,qBAAqB,MAAMC,gBAAgB;gBACzCL;gBACA,oBAAoBH;gBACpBT;YACF;YACAG,OACEa,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAEP,iBAAiB,CAAC,EAChDO,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,WAAW,EACXC,KAAK,EACLC,gBAAgB,EACjB,GAAG,MAAMC,gBAAgB;YACxB,UAAU,IAAI,CAAC,UAAU;YACzBd;YACA,0BAA0BV;YAC1B,cAAcc;YACdhB;QACF;QAEA,MAAM2B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAX;YACAC;YACAC;QACF;QAEA,IAAIe;QACJ,IAAI,QAAA7B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B6B,WAAW,CAAC,6BAA6B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG5E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS7B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAamB;YACb,MAAM;YACNO;YACA,WAAW,CAAC,CAACf;YACb,OAAOiB;QACT;QAEA,MAAME,WAA0B,EAAE;QACjCZ,CAAAA,YAAY,QAAQ,IAAI,EAAC,EAAG,OAAO,CAAC,CAACa;YACpC,IAAI,QAAQA,MAAM;gBAChB,MAAMC,UAAUZ,YAAYW,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE;gBAEpC,IAAI,CAACC,SAAS,YACZvB,QAAQ,IAAI,CACV,CAAC,+BAA+B,EAAEsB,KAAK,EAAE,CAAC,0CAA0C,CAAC;gBAIzFD,SAAS,IAAI,CAACE;YAChB;QACF;QAEAC,gBACE;YACE,GAAGJ,QAAQ;YACX,gBAAgBC;QAClB,GACA5B;QAGF,IAAI0B,UACF,MAAM,IAAIM,MAAMN;QAGlB3B,OACE6B,SAAS,MAAM,IAAI,GACnB,CAAC,0CAA0C,EAAEA,SAAS,MAAM,EAAE;QAGhE,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,IAAIA,QAAQ,CAAC,EAAE,CAAE,EAAE;gBACnB,SAASA,QAAQ,CAAC,EAAE,CAAE,OAAO;gBAC7B,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM,IAAI,EAAE;gBACjC,YAAYA,QAAQ,CAAC,EAAE,CAAE,UAAU;gBACnCP;YACF;YACAJ;QACF;QAEF,OAAO;YACL,SAAS;YACTA;QACF;IACF;IAEA,MAAM,QACJgB,UAA+B,EAC/BrC,WAAyB,EACzBD,GAA0B,EAC1BuC,gBAAoC,EAKnC;YA4BGrC;QA3BJE,OACE,AAAsB,YAAtB,OAAOkC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAMjC,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzB,MAAMO,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAE9C,MAAMM,YAAYC,KAAK,GAAG;QAE1B,MAAM,EAAEC,WAAW,EAAEI,KAAK,EAAE,GAAG,MAAMe,qBAAwB;YAC3D3B;YACA,WAAWyB;YACXC;YACA,eAAevC;YACfC;QACF;QAEA,MAAM2B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACT;QAC9B;QAEA,IAAIU;QACJ,IAAI,QAAA7B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B6B,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTM;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNT;YACA,OAAOE;QACT;QAEA,MAAM,EAAEU,IAAI,EAAEC,OAAO,EAAE,GAAGrB,eAAe,CAAC;QAG1Ce,gBACE;YACE,GAAGJ,QAAQ;YACXS;QACF,GACApC;QAGF,IAAI0B,YAAY,CAACU,QAAQ,CAACzC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,GAC3C,MAAM,IAAIqC,MAAMN;QAGlB,OAAO;YACLU;YACAC;YACAjB;QACF;IACF;IAEA,MAAM,SACJkB,MAA+B,EAC/B1C,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOuC,QAAQ;QACf,MAAM9B,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAC9C,MAAM,EAAE+B,gBAAgB,EAAEC,IAAI,EAAE,GAAGhC;QACnCT,OAAOwC,kBAAkB;QAEzB,MAAM,EAAEjC,MAAM,EAAE,GAAGV;QACnB,MAAM6C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIjD,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,EAAE;YAClB,MAAMc,aAAawC,iBAAiBL,YAAYpC,QAAQ,IAAI,EAAEF;YAC9Df,MAAM,4BAA4BkB;YAClC,MAAMyC,gBAAgB,MAAMC,WAC1BJ,cACAtC,YACAH,AAAW,cAAXA;YAEFyC,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,WAAW,IAAI,CAClB,UAAU;QAEb,MAAMC,MAAM,MAAMD,SAChBD,MACAG,aAAa,gBAAgB,EAC7B3D;QAGF,MAAM,EAAE4D,OAAO,EAAE,GAAGF;QACpBvD,OAAO,CAACyD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DzD,OAAOyD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAxUA,YACEhD,OAEmE,EACnEb,GAAoB,CACpB;QAhBF;QAIA,qCACE8D;QAEF;QAEA;QAQE1D,OAAOS,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMkD,QAAQ,OAAO,CAAClD;QAIlD,IAAI,AAA2B,WAApBb,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,UAAU,AAAD,GACvB,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,QAAQ,AAAD,GACrB,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAqTF"}

package/dist/es/insight/utils.mjs

@@ -0,0 +1,15 @@
+import { uuid } from "@sqaitech/shared/utils";
+function emitInsightDump(data, dumpSubscriber) {
+    const baseData = {
+        logTime: Date.now()
+    };
+    const finalData = {
+        logId: uuid(),
+        ...baseData,
+        ...data
+    };
+    null == dumpSubscriber || dumpSubscriber(finalData);
+}
+export { emitInsightDump };
+
+//# sourceMappingURL=utils.mjs.map

package/dist/es/insight/utils.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"insight\\utils.mjs","sources":["webpack://@sqaitech/core/./src/insight/utils.ts"],"sourcesContent":["import type {\r\n  DumpMeta,\r\n  DumpSubscriber,\r\n  InsightDump,\r\n  PartialInsightDumpFromSDK,\r\n} from '@/types';\r\nimport { uuid } from '@sqaitech/shared/utils';\r\n\r\nexport function emitInsightDump(\r\n  data: PartialInsightDumpFromSDK,\r\n  dumpSubscriber?: DumpSubscriber,\r\n) {\r\n  const baseData: DumpMeta = {\r\n    logTime: Date.now(),\r\n  };\r\n  const finalData: InsightDump = {\r\n    logId: uuid(),\r\n    ...baseData,\r\n    ...data,\r\n  };\r\n\r\n  dumpSubscriber?.(finalData);\r\n}\r\n"],"names":["emitInsightDump","data","dumpSubscriber","baseData","Date","finalData","uuid"],"mappings":";AAQO,SAASA,gBACdC,IAA+B,EAC/BC,cAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC;QACP,GAAGH,QAAQ;QACX,GAAGF,IAAI;IACT;IAEAC,QAAAA,kBAAAA,eAAiBG;AACnB"}

package/dist/es/report.mjs

@@ -0,0 +1,88 @@
+import { appendFileSync, existsSync, readFileSync, unlinkSync } from "node:fs";
+import { resolve } from "node:path";
+import { getMidsceneRunSubDir } from "@sqaitech/shared/common";
+import { getReportFileName } from "./agent/index.mjs";
+import { getReportTpl, reportHTMLContent } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class ReportMergingTool {
+    append(reportInfo) {
+        this.reportInfos.push(reportInfo);
+    }
+    clear() {
+        this.reportInfos = [];
+    }
+    extractScriptContent(filePath) {
+        const scriptRegex = /\n<script type="SQAI_web_dump" type="application\/json"[^>]*>([\s\S]*?)\n<\/script>/;
+        const fileContent = readFileSync(filePath, 'utf-8');
+        const match = scriptRegex.exec(fileContent);
+        return match ? match[1].trim() : '';
+    }
+    mergeReports(reportFileName = 'AUTO', opts) {
+        if (this.reportInfos.length <= 1) {
+            console.log('Not enough report to merge');
+            return null;
+        }
+        opts = Object.assign({
+            rmOriginalReports: false,
+            overwrite: false
+        }, opts || {});
+        const { rmOriginalReports, overwrite } = opts;
+        let outputFilePath;
+        const targetDir = `${getMidsceneRunSubDir('report')}`;
+        if ('AUTO' === reportFileName) outputFilePath = resolve(targetDir, `${getReportFileName('merged-report')}.html`);
+        else {
+            outputFilePath = resolve(targetDir, `${reportFileName}.html`);
+            if (existsSync(outputFilePath) && !overwrite) throw Error(`report file already existed: ${outputFilePath}\nset override to true to overwrite this file.`);
+            if (existsSync(outputFilePath) && overwrite) unlinkSync(outputFilePath);
+        }
+        console.log(`Start merging ${this.reportInfos.length} reports...\nCreating template file...`);
+        try {
+            appendFileSync(outputFilePath, getReportTpl());
+            for(let i = 0; i < this.reportInfos.length; i++){
+                const reportInfo = this.reportInfos[i];
+                console.log(`Processing report ${i + 1}/${this.reportInfos.length}`);
+                const dumpString = this.extractScriptContent(reportInfo.reportFilePath);
+                const reportAttributes = reportInfo.reportAttributes;
+                const reportHtmlStr = `${reportHTMLContent({
+                    dumpString,
+                    attributes: {
+                        playwright_test_duration: reportAttributes.testDuration,
+                        playwright_test_status: reportAttributes.testStatus,
+                        playwright_test_title: reportAttributes.testTitle,
+                        playwright_test_id: reportAttributes.testId,
+                        playwright_test_description: reportAttributes.testDescription
+                    }
+                }, void 0, void 0, false)}\n`;
+                appendFileSync(outputFilePath, reportHtmlStr);
+            }
+            console.log(`Successfully merged new report: ${outputFilePath}`);
+            if (rmOriginalReports) {
+                for (const info of this.reportInfos)try {
+                    unlinkSync(info.reportFilePath);
+                } catch (error) {
+                    console.error(`Error deleting report ${info.reportFilePath}:`, error);
+                }
+                console.log(`Removed ${this.reportInfos.length} original reports`);
+            }
+            return outputFilePath;
+        } catch (error) {
+            console.error('Error in mergeReports:', error);
+            throw error;
+        }
+    }
+    constructor(){
+        _define_property(this, "reportInfos", []);
+    }
+}
+export { ReportMergingTool };
+
+//# sourceMappingURL=report.mjs.map