@midscene/core 1.0.1-beta-20251022061922.0 → 1.0.1-beta-20251024064637.0

package/dist/es/ai-model/inspect.mjs

@@ -1,3 +1,4 @@
+import { generateElementByPosition } from "@midscene/shared/extractor/dom-util";
 import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
@@ -5,7 +6,7 @@ import { AIActionType, adaptBboxToRect, expandSearchArea, mergeRects } from "./c
 import { extractDataQueryPrompt, systemPromptToExtract } from "./prompt/extraction.mjs";
 import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
-import { describeUserPage, distance, distanceThreshold, elementByPositionWithElementInfo } from "./prompt/util.mjs";
+import { describeUserPage } from "./prompt/util.mjs";
 import { callAIWithObjectResponse } from "./service-caller/index.mjs";
 const debugInspect = getDebug('ai:inspect');
 const debugSection = getDebug('ai:section');
@@ -57,12 +58,9 @@ async function AiLocateElement(options) {
     const { context, targetElementDescription, callAIFn, modelConfig } = options;
     const { vlMode } = modelConfig;
     const { screenshotBase64 } = context;
-    const { description, elementById, insertElementByPosition } = await describeUserPage(context, {
-        vlMode
-    });
     assert(targetElementDescription, "cannot find the target element description");
     const userInstructionPrompt = await findElementPrompt.format({
-        pageDescription: description,
+        pageDescription: await describeUserPage(context),
         targetElementDescription: extraTextFromUserPrompt(targetElementDescription)
     });
     const systemPrompt = systemPromptToLocateElement(vlMode);
@@ -129,18 +127,12 @@ async function AiLocateElement(options) {
                 x: resRect.left + resRect.width / 2,
                 y: resRect.top + resRect.height / 2
             };
-            let element = elementByPositionWithElementInfo(context.tree, rectCenter);
-            const distanceToCenter = element ? distance({
-                x: element.center[0],
-                y: element.center[1]
-            }, rectCenter) : 0;
-            if (!element || distanceToCenter > distanceThreshold) element = insertElementByPosition(rectCenter);
-            if (element) {
-                matchedElements = [
-                    element
-                ];
-                errors = [];
-            }
+            const element = generateElementByPosition(rectCenter);
+            errors = [];
+            element.isOrderSensitive = 'object' == typeof res.content && null !== res.content && 'isOrderSensitive' in res.content ? res.content.isOrderSensitive : void 0;
+            if (element) matchedElements = [
+                element
+            ];
         }
     } catch (e) {
         const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
@@ -153,12 +145,10 @@ async function AiLocateElement(options) {
         rect: resRect,
         parseResult: {
             elements: matchedElements,
-            errors
+            errors: errors
         },
         rawResponse,
-        elementById,
-        usage: res.usage,
-        isOrderSensitive: 'object' == typeof res.content && null !== res.content && 'isOrderSensitive' in res.content ? res.content.isOrderSensitive : void 0
+        usage: res.usage
     };
 }
 async function AiLocateSection(options) {
@@ -233,17 +223,9 @@ async function AiLocateSection(options) {
 }
 async function AiExtractElementInfo(options) {
     const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } = options;
-    const { vlMode } = modelConfig;
     const systemPrompt = systemPromptToExtract();
     const { screenshotBase64 } = context;
-    const { description, elementById } = await describeUserPage(context, {
-        truncateTextLength: 200,
-        filterNonTextContent: false,
-        visibleOnly: false,
-        domIncluded: null == extractOption ? void 0 : extractOption.domIncluded,
-        vlMode
-    });
-    const extractDataPromptText = await extractDataQueryPrompt(description, dataQuery);
+    const extractDataPromptText = await extractDataQueryPrompt(options.pageDescription || '', dataQuery);
     const userContent = [];
     if ((null == extractOption ? void 0 : extractOption.screenshotIncluded) !== false) userContent.push({
         type: 'image_url',
@@ -276,7 +258,6 @@ async function AiExtractElementInfo(options) {
     const result = await callAIWithObjectResponse(msgs, AIActionType.EXTRACT_DATA, modelConfig);
     return {
         parseResult: result.content,
-        elementById,
         usage: result.usage
     };
 }

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocatorResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  BaseElement,\n  ElementById,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from './common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  mergeRects,\n} from './common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  describeUserPage,\n  distance,\n  distanceThreshold,\n  elementByPositionWithElementInfo,\n} from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement<\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  context: UIContext<ElementType>;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: AIElementLocatorResponse;\n  rect?: Rect;\n  rawResponse: string;\n  elementById: ElementById;\n  usage?: AIUsageInfo;\n  isOrderSensitive?: boolean;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const { description, elementById, insertElementByPosition } =\n    await describeUserPage(context, { vlMode });\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: description,\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: AIElementLocatorResponse['elements'] =\n    'elements' in res.content ? res.content.elements : [];\n  let errors: AIElementLocatorResponse['errors'] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n      let element = elementByPositionWithElementInfo(context.tree, rectCenter);\n\n      const distanceToCenter = element\n        ? distance({ x: element.center[0], y: element.center[1] }, rectCenter)\n        : 0;\n\n      if (!element || distanceToCenter > distanceThreshold) {\n        element = insertElementByPosition(rectCenter);\n      }\n\n      if (element) {\n        matchedElements = [element];\n        errors = [];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements,\n      errors,\n    },\n    rawResponse,\n    elementById,\n    usage: res.usage,\n    isOrderSensitive:\n      typeof res.content === 'object' &&\n      res.content !== null &&\n      'isOrderSensitive' in res.content\n        ? (res.content as any).isOrderSensitive\n        : undefined,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext<BaseElement>;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<\n  T,\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext<ElementType>;\n  extractOption?: InsightExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const { vlMode } = modelConfig;\n  const systemPrompt = systemPromptToExtract();\n\n  const { screenshotBase64 } = context;\n\n  const { description, elementById } = await describeUserPage(context, {\n    truncateTextLength: 200,\n    filterNonTextContent: false,\n    visibleOnly: false,\n    domIncluded: extractOption?.domIncluded,\n    vlMode,\n  });\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    description,\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    elementById,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","description","elementById","insertElementByPosition","describeUserPage","assert","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","_options_searchConfig_rect","_options_searchConfig_rect1","paddedResult","paddingToMatchBlockByBase64","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","elementByPositionWithElementInfo","distanceToCenter","distance","distanceThreshold","e","msg","Error","undefined","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent"],"mappings":";;;;;;;;;AAyDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAEpBC,OASD;IAQC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAM,EAAEM,WAAW,EAAEC,WAAW,EAAEC,uBAAuB,EAAE,GACzD,MAAMC,iBAAiBT,SAAS;QAAEI;IAAO;IAE3CM,OACET,0BACA;IAEF,MAAMU,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiBN;QACjB,0BAA0BjB,wBAAwBY;IACpD;IACA,MAAMY,eAAeC,4BAA4BV;IAEjD,IAAIW,eAAeV;IACnB,IAAIW,aAAahB,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIiB,cAAcjB,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIkB,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIlB,QAAQ,YAAY,EAAE;YAWXqB,4BACCC;QAXdX,OACEX,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFW,OACEX,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFgB,eAAehB,QAAQ,YAAY,CAAC,WAAW;QAC/CiB,aAAa,QAAAI,CAAAA,6BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,KAAK;QAC7CH,cAAc,QAAAI,CAAAA,8BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,MAAM;QAC/CH,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIb,AAAW,cAAXA,QAAsB;QAC/B,MAAMkB,eAAe,MAAMC,4BAA4BR;QACvDC,aAAaM,aAAa,KAAK;QAC/BL,cAAcK,aAAa,MAAM;QACjCP,eAAeO,aAAa,WAAW;IACzC;IAEA,MAAM5B,OAAe;QACnB;YAAE,MAAM;YAAU,SAASmB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMJ;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOV,0BAAuC;QAChD,MAAMuB,SAAS,MAAMjC,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI8B;IACf;IAEA,MAAMC,MAAM,MAAMvB,SAASR,MAAMgC,aAAa,eAAe,EAAEvB;IAE/D,MAAMwB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBACF,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IACvD,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAK1DQ,6BAAAA,uBACAC,6BAAAA;YALFL,UAAUM,gBACRV,IAAI,OAAO,CAAC,IAAI,EAChBT,YACAC,aAAAA,QACAgB,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG,EAC/BhB,oBACAC,qBACAf;YAGFlB,aAAa,WAAW2C;YAExB,MAAMO,aAAa;gBACjB,GAAGP,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YACA,IAAIQ,UAAUC,iCAAiCtC,QAAQ,IAAI,EAAEoC;YAE7D,MAAMG,mBAAmBF,UACrBG,SAAS;gBAAE,GAAGH,QAAQ,MAAM,CAAC,EAAE;gBAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;YAAC,GAAGD,cACzD;YAEJ,IAAI,CAACC,WAAWE,mBAAmBE,mBACjCJ,UAAU7B,wBAAwB4B;YAGpC,IAAIC,SAAS;gBACXP,kBAAkB;oBAACO;iBAAQ;gBAC3BN,SAAS,EAAE;YACb;QACF;IACF,EAAE,OAAOW,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACX,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEY,IAAI,CAAC,CAAC;aAFtBZ,SAAS;YAACY;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMd;QACN,aAAa;YACX,UAAUC;YACVC;QACF;QACAJ;QACApB;QACA,OAAOkB,IAAI,KAAK;QAChB,kBACE,AAAuB,YAAvB,OAAOA,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCoB;IACR;AACF;AAEO,eAAeC,gBAAgB/C,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAE+C,kBAAkB,EAAE5C,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMa,eAAemC,4BAA4B5C;IACjD,MAAM6C,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoB7D,wBAAwB0D;IAC9C;IACA,MAAMrD,OAAe;QACnB;YAAE,MAAM;YAAU,SAASmB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM4C;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMvB,SAAS,MAAMjC,mBAAmB;YACtC,QAAQwD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACArD,KAAK,IAAI,IAAI8B;IACf;IAEA,MAAM2B,SAAS,MAAMC,yBACnB1D,MACAgC,aAAa,YAAY,EACzBvB;IAGF,IAAIkD;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAapB,gBACjBmB,aACAtD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0BmE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9D/D,aAAa,wBAAwBoE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAS1B,MAAM,OAAO,CAAC0B,OAC/B,GAAG,CAAC,CAACA,OACGvB,gBACLuB,MACA1D,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqBqE;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DrE,aAAa,iBAAiBuE;QAG9BN,cAAcQ,iBAAiBF,YAAY3D,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2BiE;IAC1C;IAEA,IAAIS,cAAczD;IAClB,IAAIgD,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1B3D,kBACAgD,aACAjD,AAAW,cAAXA;QAEF0D,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAavB,KAAK,SAAS,CAACuB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAGpBlE,OAMD;IACC,MAAM,EAAEmE,SAAS,EAAElE,OAAO,EAAEmE,aAAa,EAAE3E,gBAAgB,EAAEW,WAAW,EAAE,GACxEJ;IACF,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAMU,eAAeuD;IAErB,MAAM,EAAE/D,gBAAgB,EAAE,GAAGL;IAE7B,MAAM,EAAEM,WAAW,EAAEC,WAAW,EAAE,GAAG,MAAME,iBAAiBT,SAAS;QACnE,oBAAoB;QACpB,sBAAsB;QACtB,aAAa;QACb,aAAamE,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,WAAW;QACvC/D;IACF;IAEA,MAAMiE,wBAAwB,MAAMC,uBAClChE,aACA4D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKlE;YACL,QAAQ;QACV;IACF;IAGFkE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM3E,OAAe;QACnB;YAAE,MAAM;YAAU,SAASmB;QAAa;QACxC;YACE,MAAM;YACN,SAAS0D;QACX;KACD;IAED,IAAI/E,kBAAkB;QACpB,MAAMgC,SAAS,MAAMjC,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAI8B;IACf;IAEA,MAAM2B,SAAS,MAAMC,yBACnB1D,MACAgC,aAAa,YAAY,EACzBvB;IAEF,OAAO;QACL,aAAagD,OAAO,OAAO;QAC3B5C;QACA,OAAO4C,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from './common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  mergeRects,\n} from './common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport { describeUserPage } from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: await describeUserPage(context),\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements = 'elements' in res.content ? res.content.elements : [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement =\n        generateElementByPosition(rectCenter);\n      errors = [];\n\n      element.isOrderSensitive =\n        typeof res.content === 'object' &&\n        res.content !== null &&\n        'isOrderSensitive' in res.content\n          ? (res.content as any).isOrderSensitive\n          : undefined;\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: InsightExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","userInstructionPrompt","findElementPrompt","describeUserPage","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","_options_searchConfig_rect","_options_searchConfig_rect1","paddedResult","paddingToMatchBlockByBase64","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","generateElementByPosition","undefined","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent"],"mappings":";;;;;;;;;;AAmDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IASC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiB,MAAMC,iBAAiBT;QACxC,0BAA0BX,wBAAwBY;IACpD;IACA,MAAMS,eAAeC,4BAA4BP;IAEjD,IAAIQ,eAAeP;IACnB,IAAIQ,aAAab,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIc,cAAcd,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIe,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIf,QAAQ,YAAY,EAAE;YAWXkB,4BACCC;QAXdZ,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFa,eAAeb,QAAQ,YAAY,CAAC,WAAW;QAC/Cc,aAAa,QAAAI,CAAAA,6BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,KAAK;QAC7CH,cAAc,QAAAI,CAAAA,8BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,MAAM;QAC/CH,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIV,AAAW,cAAXA,QAAsB;QAC/B,MAAMe,eAAe,MAAMC,4BAA4BR;QACvDC,aAAaM,aAAa,KAAK;QAC/BL,cAAcK,aAAa,MAAM;QACjCP,eAAeO,aAAa,WAAW;IACzC;IAEA,MAAMzB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAML;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAON,0BAAuC;QAChD,MAAMoB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMC,MAAM,MAAMpB,SAASR,MAAM6B,aAAa,eAAe,EAAEpB;IAE/D,MAAMqB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBAAkB,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IAC3E,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAK1DQ,6BAAAA,uBACAC,6BAAAA;YALFL,UAAUM,gBACRV,IAAI,OAAO,CAAC,IAAI,EAChBT,YACAC,aAAAA,QACAgB,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG,EAC/BhB,oBACAC,qBACAZ;YAGFlB,aAAa,WAAWwC;YAExB,MAAMO,aAAa;gBACjB,GAAGP,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMQ,UACJC,0BAA0BF;YAC5BL,SAAS,EAAE;YAEXM,QAAQ,gBAAgB,GACtB,AAAuB,YAAvB,OAAOZ,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCc;YAEN,IAAIF,SACFP,kBAAkB;gBAACO;aAAQ;QAE/B;IACF,EAAE,OAAOG,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACT,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEU,IAAI,CAAC,CAAC;aAFtBV,SAAS;YAACU;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMZ;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAJ;QACA,OAAOF,IAAI,KAAK;IAClB;AACF;AAEO,eAAekB,gBAAgBzC,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEyC,kBAAkB,EAAEtC,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAegC,4BAA4BtC;IACjD,MAAMuC,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoBvD,wBAAwBoD;IAC9C;IACA,MAAM/C,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMsC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMpB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQkD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA/C,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMwB,SAAS,MAAMC,yBACnBpD,MACA6B,aAAa,YAAY,EACzBpB;IAGF,IAAI4C;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAajB,gBACjBgB,aACAhD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0B6D;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DzD,aAAa,wBAAwB8D;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASvB,MAAM,OAAO,CAACuB,OAC/B,GAAG,CAAC,CAACA,OACGpB,gBACLoB,MACApD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqB+D;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7D/D,aAAa,iBAAiBiE;QAG9BN,cAAcQ,iBAAiBF,YAAYrD,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2B2D;IAC1C;IAEA,IAAIS,cAAcnD;IAClB,IAAI0C,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1BrD,kBACA0C,aACA3C,AAAW,cAAXA;QAEFoD,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAapB,KAAK,SAAS,CAACoB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwB5D,OAO7C;IACC,MAAM,EAAE6D,SAAS,EAAE5D,OAAO,EAAE6D,aAAa,EAAErE,gBAAgB,EAAEW,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAeoD;IACrB,MAAM,EAAEzD,gBAAgB,EAAE,GAAGL;IAE7B,MAAM+D,wBAAwB,MAAMC,uBAClCjE,QAAQ,eAAe,IAAI,IAC3B6D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAK5D;YACL,QAAQ;QACV;IACF;IAGF4D,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMrE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASuD;QACX;KACD;IAED,IAAIzE,kBAAkB;QACpB,MAAM6B,SAAS,MAAM9B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMwB,SAAS,MAAMC,yBACnBpD,MACA6B,aAAa,YAAY,EACzBpB;IAEF,OAAO;QACL,aAAa0C,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/prompt/util.mjs

@@ -1,7 +1,3 @@
-import { imageInfoOfBase64 } from "../../image/index.mjs";
-import { NodeType } from "@midscene/shared/constants";
-import { descriptionOfTree, generateElementByPosition, treeToList } from "@midscene/shared/extractor";
-import { assert } from "@midscene/shared/utils";
 function describeSize(size) {
     return `${size.width} x ${size.height}`;
 }
@@ -17,35 +13,6 @@ function describeElement(elements) {
         ].join(', ')).join('\n');
 }
 const distanceThreshold = 16;
-function elementByPositionWithElementInfo(treeRoot, position, options) {
-    const requireStrictDistance = (null == options ? void 0 : options.requireStrictDistance) ?? true;
-    const filterPositionElements = (null == options ? void 0 : options.filterPositionElements) ?? false;
-    assert(void 0 !== position, 'position is required for query');
-    const matchingElements = [];
-    function dfs(node) {
-        if (null == node ? void 0 : node.node) {
-            const item = node.node;
-            if (item.rect.left <= position.x && position.x <= item.rect.left + item.rect.width && item.rect.top <= position.y && position.y <= item.rect.top + item.rect.height) {
-                var _item_attributes;
-                if (!(filterPositionElements && (null == (_item_attributes = item.attributes) ? void 0 : _item_attributes.nodeType) === NodeType.POSITION) && item.isVisible) matchingElements.push(item);
-            }
-        }
-        for (const child of node.children)dfs(child);
-    }
-    dfs(treeRoot);
-    if (0 === matchingElements.length) return;
-    const element = matchingElements.reduce((smallest, current)=>{
-        const smallestArea = smallest.rect.width * smallest.rect.height;
-        const currentArea = current.rect.width * current.rect.height;
-        return currentArea < smallestArea ? current : smallest;
-    });
-    const distanceToCenter = distance({
-        x: element.center[0],
-        y: element.center[1]
-    }, position);
-    if (requireStrictDistance) return distanceToCenter <= distanceThreshold ? element : void 0;
-    return element;
-}
 function distance(point1, point2) {
     return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);
 }
@@ -64,61 +31,9 @@ Description of all the elements in screenshot:
 </div>
 ====================
 `;
-async function describeUserPage(context, opt) {
-    const { screenshotBase64 } = context;
-    let width;
-    let height;
-    if (context.size) ({ width, height } = context.size);
-    else {
-        const imgSize = await imageInfoOfBase64(screenshotBase64);
-        ({ width, height } = imgSize);
-    }
-    const treeRoot = context.tree;
-    const idElementMap = {};
-    const flatElements = treeToList(treeRoot);
-    if ((null == opt ? void 0 : opt.domIncluded) === true && flatElements.length >= 5000) console.warn('The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: "visible-only" to reduce the number of elements');
-    flatElements.forEach((element)=>{
-        idElementMap[element.id] = element;
-        if (void 0 !== element.indexId) idElementMap[`${element.indexId}`] = element;
-    });
-    let pageDescription = '';
-    const visibleOnly = (null == opt ? void 0 : opt.domIncluded) === 'visible-only' ? true : (null == opt ? void 0 : opt.visibleOnly) ?? false;
-    const resolvedVlMode = null == opt ? void 0 : opt.vlMode;
-    const shouldIncludeDOM = (null == opt ? void 0 : opt.domIncluded) || !resolvedVlMode;
-    if (shouldIncludeDOM) {
-        const contentTree = await descriptionOfTree(treeRoot, null == opt ? void 0 : opt.truncateTextLength, null == opt ? void 0 : opt.filterNonTextContent, visibleOnly);
-        const sizeDescription = describeSize({
-            width,
-            height
-        });
-        pageDescription = `The size of the page: ${sizeDescription} \n The page elements tree:\n${contentTree}`;
-    }
-    return {
-        description: pageDescription,
-        elementById (idOrIndexId) {
-            assert(void 0 !== idOrIndexId, 'id is required for query');
-            const item = idElementMap[`${idOrIndexId}`];
-            return item;
-        },
-        elementByPosition (position, size) {
-            return elementByPositionWithElementInfo(treeRoot, position);
-        },
-        insertElementByPosition (position) {
-            const element = generateElementByPosition(position);
-            treeRoot.children.push({
-                node: element,
-                children: []
-            });
-            flatElements.push(element);
-            idElementMap[element.id] = element;
-            return element;
-        },
-        size: {
-            width,
-            height
-        }
-    };
+async function describeUserPage(context) {
+    return `The size of the page: ${describeSize(context.size)}`;
 }
-export { describeElement, describeSize, describeUserPage, distance, distanceThreshold, elementByPositionWithElementInfo, samplePageDescription };
+export { describeElement, describeSize, describeUserPage, distance, distanceThreshold, samplePageDescription };
 
 //# sourceMappingURL=util.mjs.map

package/dist/es/ai-model/prompt/util.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/util.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/util.ts"],"sourcesContent":["import { imageInfoOfBase64 } from '@/image/index';\nimport type { BaseElement, ElementTreeNode, Size, UIContext } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport {\n  descriptionOfTree,\n  generateElementByPosition,\n  treeToList,\n} from '@midscene/shared/extractor';\nimport { assert } from '@midscene/shared/utils';\n\nexport function describeSize(size: Size) {\n  return `${size.width} x ${size.height}`;\n}\n\nexport function describeElement(\n  elements: (Pick<BaseElement, 'rect' | 'content'> & { id: string })[],\n) {\n  const sliceLength = 80;\n  return elements\n    .map((item) =>\n      [\n        item.id,\n        item.rect.left,\n        item.rect.top,\n        item.rect.left + item.rect.width,\n        item.rect.top + item.rect.height,\n        item.content.length > sliceLength\n          ? `${item.content.slice(0, sliceLength)}...`\n          : item.content,\n      ].join(', '),\n    )\n    .join('\\n');\n}\nexport const distanceThreshold = 16;\n\nexport function elementByPositionWithElementInfo(\n  treeRoot: ElementTreeNode<BaseElement>,\n  position: {\n    x: number;\n    y: number;\n  },\n  options?: {\n    requireStrictDistance?: boolean;\n    filterPositionElements?: boolean;\n  },\n) {\n  const requireStrictDistance = options?.requireStrictDistance ?? true;\n  const filterPositionElements = options?.filterPositionElements ?? false;\n\n  assert(typeof position !== 'undefined', 'position is required for query');\n\n  const matchingElements: BaseElement[] = [];\n\n  function dfs(node: ElementTreeNode<BaseElement>) {\n    if (node?.node) {\n      const item = node.node;\n      if (\n        item.rect.left <= position.x &&\n        position.x <= item.rect.left + item.rect.width &&\n        item.rect.top <= position.y &&\n        position.y <= item.rect.top + item.rect.height\n      ) {\n        if (\n          !(\n            filterPositionElements &&\n            item.attributes?.nodeType === NodeType.POSITION\n          ) &&\n          item.isVisible\n        ) {\n          matchingElements.push(item);\n        }\n      }\n    }\n\n    for (const child of node.children) {\n      dfs(child);\n    }\n  }\n\n  dfs(treeRoot);\n\n  if (matchingElements.length === 0) {\n    return undefined;\n  }\n\n  // Find the smallest element by area\n  const element = matchingElements.reduce((smallest, current) => {\n    const smallestArea = smallest.rect.width * smallest.rect.height;\n    const currentArea = current.rect.width * current.rect.height;\n    return currentArea < smallestArea ? current : smallest;\n  });\n\n  const distanceToCenter = distance(\n    { x: element.center[0], y: element.center[1] },\n    position,\n  );\n\n  if (requireStrictDistance) {\n    return distanceToCenter <= distanceThreshold ? element : undefined;\n  }\n\n  return element;\n}\n\nexport function distance(\n  point1: { x: number; y: number },\n  point2: { x: number; y: number },\n) {\n  return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);\n}\n\nexport const samplePageDescription = `\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n`;\n\nexport async function describeUserPage<\n  ElementType extends BaseElement = BaseElement,\n>(\n  context: Omit<UIContext<ElementType>, 'describer'>,\n  opt: {\n    truncateTextLength?: number;\n    filterNonTextContent?: boolean;\n    domIncluded?: boolean | 'visible-only';\n    visibleOnly?: boolean;\n    vlMode: TVlModeTypes | undefined;\n  },\n) {\n  const { screenshotBase64 } = context;\n  let width: number;\n  let height: number;\n\n  if (context.size) {\n    ({ width, height } = context.size);\n  } else {\n    const imgSize = await imageInfoOfBase64(screenshotBase64);\n    ({ width, height } = imgSize);\n  }\n\n  const treeRoot = context.tree;\n  // dfs tree, save the id and element info\n  const idElementMap: Record<string, ElementType> = {};\n  const flatElements: ElementType[] = treeToList(treeRoot);\n\n  if (opt?.domIncluded === true && flatElements.length >= 5000) {\n    console.warn(\n      'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: \"visible-only\" to reduce the number of elements',\n    );\n  }\n\n  flatElements.forEach((element) => {\n    idElementMap[element.id] = element;\n    if (typeof element.indexId !== 'undefined') {\n      idElementMap[`${element.indexId}`] = element;\n    }\n  });\n\n  let pageDescription = '';\n\n  const visibleOnly =\n    opt?.domIncluded === 'visible-only' ? true : (opt?.visibleOnly ?? false);\n  const resolvedVlMode = opt?.vlMode;\n  const shouldIncludeDOM = opt?.domIncluded || !resolvedVlMode;\n\n  if (shouldIncludeDOM) {\n    // non-vl mode must provide the page description\n    const contentTree = await descriptionOfTree(\n      treeRoot,\n      opt?.truncateTextLength,\n      opt?.filterNonTextContent,\n      visibleOnly,\n    );\n\n    // if match by position, don't need to provide the page description\n    const sizeDescription = describeSize({ width, height });\n    pageDescription = `The size of the page: ${sizeDescription} \\n The page elements tree:\\n${contentTree}`;\n  }\n\n  return {\n    description: pageDescription,\n    elementById(idOrIndexId: string) {\n      assert(typeof idOrIndexId !== 'undefined', 'id is required for query');\n      const item = idElementMap[`${idOrIndexId}`];\n      return item;\n    },\n    elementByPosition(\n      position: { x: number; y: number },\n      size: { width: number; height: number },\n    ) {\n      return elementByPositionWithElementInfo(treeRoot, position);\n    },\n    insertElementByPosition(position: { x: number; y: number }) {\n      const element = generateElementByPosition(position) as ElementType;\n\n      treeRoot.children.push({\n        node: element,\n        children: [],\n      });\n      flatElements.push(element);\n      idElementMap[element.id] = element;\n      return element;\n    },\n    size: { width, height },\n  };\n}\n"],"names":["describeSize","size","describeElement","elements","sliceLength","item","distanceThreshold","elementByPositionWithElementInfo","treeRoot","position","options","requireStrictDistance","filterPositionElements","assert","matchingElements","dfs","node","_item_attributes","NodeType","child","element","smallest","current","smallestArea","currentArea","distanceToCenter","distance","undefined","point1","point2","Math","samplePageDescription","describeUserPage","context","opt","screenshotBase64","width","height","imgSize","imageInfoOfBase64","idElementMap","flatElements","treeToList","console","pageDescription","visibleOnly","resolvedVlMode","shouldIncludeDOM","contentTree","descriptionOfTree","sizeDescription","idOrIndexId","generateElementByPosition"],"mappings":";;;;AAWO,SAASA,aAAaC,IAAU;IACrC,OAAO,GAAGA,KAAK,KAAK,CAAC,GAAG,EAAEA,KAAK,MAAM,EAAE;AACzC;AAEO,SAASC,gBACdC,QAAoE;IAEpE,MAAMC,cAAc;IACpB,OAAOD,SACJ,GAAG,CAAC,CAACE,OACJ;YACEA,KAAK,EAAE;YACPA,KAAK,IAAI,CAAC,IAAI;YACdA,KAAK,IAAI,CAAC,GAAG;YACbA,KAAK,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI,CAAC,KAAK;YAChCA,KAAK,IAAI,CAAC,GAAG,GAAGA,KAAK,IAAI,CAAC,MAAM;YAChCA,KAAK,OAAO,CAAC,MAAM,GAAGD,cAClB,GAAGC,KAAK,OAAO,CAAC,KAAK,CAAC,GAAGD,aAAa,GAAG,CAAC,GAC1CC,KAAK,OAAO;SACjB,CAAC,IAAI,CAAC,OAER,IAAI,CAAC;AACV;AACO,MAAMC,oBAAoB;AAE1B,SAASC,iCACdC,QAAsC,EACtCC,QAGC,EACDC,OAGC;IAED,MAAMC,wBAAwBD,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,qBAAqB,AAAD,KAAK;IAChE,MAAME,yBAAyBF,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,sBAAsB,AAAD,KAAK;IAElEG,OAAO,AAAoB,WAAbJ,UAA0B;IAExC,MAAMK,mBAAkC,EAAE;IAE1C,SAASC,IAAIC,IAAkC;QAC7C,IAAIA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,EAAE;YACd,MAAMX,OAAOW,KAAK,IAAI;YACtB,IACEX,KAAK,IAAI,CAAC,IAAI,IAAII,SAAS,CAAC,IAC5BA,SAAS,CAAC,IAAIJ,KAAK,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI,CAAC,KAAK,IAC9CA,KAAK,IAAI,CAAC,GAAG,IAAII,SAAS,CAAC,IAC3BA,SAAS,CAAC,IAAIJ,KAAK,IAAI,CAAC,GAAG,GAAGA,KAAK,IAAI,CAAC,MAAM,EAC9C;oBAIIY;gBAHJ,IACE,CACEL,CAAAA,0BACAK,AAAAA,SAAAA,CAAAA,mBAAAA,KAAK,UAAU,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,QAAQ,AAAD,MAAMC,SAAS,QAAO,KAEhDb,KAAK,SAAS,EAEdS,iBAAiB,IAAI,CAACT;YAE1B;QACF;QAEA,KAAK,MAAMc,SAASH,KAAK,QAAQ,CAC/BD,IAAII;IAER;IAEAJ,IAAIP;IAEJ,IAAIM,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB;IAIF,MAAMM,UAAUN,iBAAiB,MAAM,CAAC,CAACO,UAAUC;QACjD,MAAMC,eAAeF,SAAS,IAAI,CAAC,KAAK,GAAGA,SAAS,IAAI,CAAC,MAAM;QAC/D,MAAMG,cAAcF,QAAQ,IAAI,CAAC,KAAK,GAAGA,QAAQ,IAAI,CAAC,MAAM;QAC5D,OAAOE,cAAcD,eAAeD,UAAUD;IAChD;IAEA,MAAMI,mBAAmBC,SACvB;QAAE,GAAGN,QAAQ,MAAM,CAAC,EAAE;QAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;IAAC,GAC7CX;IAGF,IAAIE,uBACF,OAAOc,oBAAoBnB,oBAAoBc,UAAUO;IAG3D,OAAOP;AACT;AAEO,SAASM,SACdE,MAAgC,EAChCC,MAAgC;IAEhC,OAAOC,KAAK,IAAI,CAAEF,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM,IAAKD,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM;AACzE;AAEO,MAAME,wBAAwB,CAAC;;;;;;;;;;;;;;AActC,CAAC;AAEM,eAAeC,iBAGpBC,OAAkD,EAClDC,GAMC;IAED,MAAM,EAAEC,gBAAgB,EAAE,GAAGF;IAC7B,IAAIG;IACJ,IAAIC;IAEJ,IAAIJ,QAAQ,IAAI,EACb,GAAEG,KAAK,EAAEC,MAAM,EAAE,GAAGJ,QAAQ,IAAG;SAC3B;QACL,MAAMK,UAAU,MAAMC,kBAAkBJ;QACvC,GAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGC,OAAM;IAC7B;IAEA,MAAM9B,WAAWyB,QAAQ,IAAI;IAE7B,MAAMO,eAA4C,CAAC;IACnD,MAAMC,eAA8BC,WAAWlC;IAE/C,IAAI0B,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM,QAAQO,aAAa,MAAM,IAAI,MACtDE,QAAQ,IAAI,CACV;IAIJF,aAAa,OAAO,CAAC,CAACrB;QACpBoB,YAAY,CAACpB,QAAQ,EAAE,CAAC,GAAGA;QAC3B,IAAI,AAA2B,WAApBA,QAAQ,OAAO,EACxBoB,YAAY,CAAC,GAAGpB,QAAQ,OAAO,EAAE,CAAC,GAAGA;IAEzC;IAEA,IAAIwB,kBAAkB;IAEtB,MAAMC,cACJX,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM,iBAAiB,OAAQA,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK;IACpE,MAAMY,iBAAiBZ,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,MAAM;IAClC,MAAMa,mBAAmBb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK,CAACY;IAE9C,IAAIC,kBAAkB;QAEpB,MAAMC,cAAc,MAAMC,kBACxBzC,UACA0B,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,kBAAkB,EACvBA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,oBAAoB,EACzBW;QAIF,MAAMK,kBAAkBlD,aAAa;YAAEoC;YAAOC;QAAO;QACrDO,kBAAkB,CAAC,sBAAsB,EAAEM,gBAAgB,6BAA6B,EAAEF,aAAa;IACzG;IAEA,OAAO;QACL,aAAaJ;QACb,aAAYO,WAAmB;YAC7BtC,OAAO,AAAuB,WAAhBsC,aAA6B;YAC3C,MAAM9C,OAAOmC,YAAY,CAAC,GAAGW,aAAa,CAAC;YAC3C,OAAO9C;QACT;QACA,mBACEI,QAAkC,EAClCR,IAAuC;YAEvC,OAAOM,iCAAiCC,UAAUC;QACpD;QACA,yBAAwBA,QAAkC;YACxD,MAAMW,UAAUgC,0BAA0B3C;YAE1CD,SAAS,QAAQ,CAAC,IAAI,CAAC;gBACrB,MAAMY;gBACN,UAAU,EAAE;YACd;YACAqB,aAAa,IAAI,CAACrB;YAClBoB,YAAY,CAACpB,QAAQ,EAAE,CAAC,GAAGA;YAC3B,OAAOA;QACT;QACA,MAAM;YAAEgB;YAAOC;QAAO;IACxB;AACF"}
+{"version":3,"file":"ai-model/prompt/util.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/util.ts"],"sourcesContent":["import type { BaseElement, ElementTreeNode, Size, UIContext } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport { assert } from '@midscene/shared/utils';\n\nexport function describeSize(size: Size) {\n  return `${size.width} x ${size.height}`;\n}\n\nexport function describeElement(\n  elements: (Pick<BaseElement, 'rect' | 'content'> & { id: string })[],\n) {\n  const sliceLength = 80;\n  return elements\n    .map((item) =>\n      [\n        item.id,\n        item.rect.left,\n        item.rect.top,\n        item.rect.left + item.rect.width,\n        item.rect.top + item.rect.height,\n        item.content.length > sliceLength\n          ? `${item.content.slice(0, sliceLength)}...`\n          : item.content,\n      ].join(', '),\n    )\n    .join('\\n');\n}\nexport const distanceThreshold = 16;\n\n// export function elementByPositionWithElementInfo(\n//   treeRoot: ElementTreeNode<BaseElement>,\n//   position: {\n//     x: number;\n//     y: number;\n//   },\n//   options?: {\n//     requireStrictDistance?: boolean;\n//     filterPositionElements?: boolean;\n//   },\n// ) {\n//   const requireStrictDistance = options?.requireStrictDistance ?? true;\n//   const filterPositionElements = options?.filterPositionElements ?? false;\n\n//   assert(typeof position !== 'undefined', 'position is required for query');\n\n//   const matchingElements: BaseElement[] = [];\n\n//   function dfs(node: ElementTreeNode<BaseElement>) {\n//     if (node?.node) {\n//       const item = node.node;\n//       if (\n//         item.rect.left <= position.x &&\n//         position.x <= item.rect.left + item.rect.width &&\n//         item.rect.top <= position.y &&\n//         position.y <= item.rect.top + item.rect.height\n//       ) {\n//         if (\n//           !(\n//             filterPositionElements &&\n//             item.attributes?.nodeType === NodeType.POSITION\n//           ) &&\n//           item.isVisible\n//         ) {\n//           matchingElements.push(item);\n//         }\n//       }\n//     }\n\n//     for (const child of node.children) {\n//       dfs(child);\n//     }\n//   }\n\n//   dfs(treeRoot);\n\n//   if (matchingElements.length === 0) {\n//     return undefined;\n//   }\n\n//   // Find the smallest element by area\n//   const element = matchingElements.reduce((smallest, current) => {\n//     const smallestArea = smallest.rect.width * smallest.rect.height;\n//     const currentArea = current.rect.width * current.rect.height;\n//     return currentArea < smallestArea ? current : smallest;\n//   });\n\n//   const distanceToCenter = distance(\n//     { x: element.center[0], y: element.center[1] },\n//     position,\n//   );\n\n//   if (requireStrictDistance) {\n//     return distanceToCenter <= distanceThreshold ? element : undefined;\n//   }\n\n//   return element;\n// }\n\nexport function distance(\n  point1: { x: number; y: number },\n  point2: { x: number; y: number },\n) {\n  return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);\n}\n\nexport const samplePageDescription = `\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n`;\n\nexport async function describeUserPage(context: UIContext) {\n  return `The size of the page: ${describeSize(context.size)}`;\n}\n"],"names":["describeSize","size","describeElement","elements","sliceLength","item","distanceThreshold","distance","point1","point2","Math","samplePageDescription","describeUserPage","context"],"mappings":"AAIO,SAASA,aAAaC,IAAU;IACrC,OAAO,GAAGA,KAAK,KAAK,CAAC,GAAG,EAAEA,KAAK,MAAM,EAAE;AACzC;AAEO,SAASC,gBACdC,QAAoE;IAEpE,MAAMC,cAAc;IACpB,OAAOD,SACJ,GAAG,CAAC,CAACE,OACJ;YACEA,KAAK,EAAE;YACPA,KAAK,IAAI,CAAC,IAAI;YACdA,KAAK,IAAI,CAAC,GAAG;YACbA,KAAK,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI,CAAC,KAAK;YAChCA,KAAK,IAAI,CAAC,GAAG,GAAGA,KAAK,IAAI,CAAC,MAAM;YAChCA,KAAK,OAAO,CAAC,MAAM,GAAGD,cAClB,GAAGC,KAAK,OAAO,CAAC,KAAK,CAAC,GAAGD,aAAa,GAAG,CAAC,GAC1CC,KAAK,OAAO;SACjB,CAAC,IAAI,CAAC,OAER,IAAI,CAAC;AACV;AACO,MAAMC,oBAAoB;AAuE1B,SAASC,SACdC,MAAgC,EAChCC,MAAgC;IAEhC,OAAOC,KAAK,IAAI,CAAEF,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM,IAAKD,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM;AACzE;AAEO,MAAME,wBAAwB,CAAC;;;;;;;;;;;;;;AActC,CAAC;AAEM,eAAeC,iBAAiBC,OAAkB;IACvD,OAAO,CAAC,sBAAsB,EAAEb,aAAaa,QAAQ,IAAI,GAAG;AAC9D"}

package/dist/es/device/index.mjs

@@ -64,11 +64,11 @@ const defineActionInput = (call)=>defineAction({
     });
 const actionKeyboardPressParamSchema = z.object({
     locate: getMidsceneLocationSchema().describe('The element to be clicked before pressing the key').optional(),
-    keyName: z.string().describe('The key to be pressed')
+    keyName: z.string().describe("The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'")
 });
 const defineActionKeyboardPress = (call)=>defineAction({
         name: 'KeyboardPress',
-        description: 'Press a function key, like "Enter", "Tab", "Escape". Do not use this to type text.',
+        description: 'Press a key or key combination, like "Enter", "Tab", "Escape", or "Control+A", "Shift+Enter". Do not use this to type text.',
         interfaceAlias: 'aiKeyboardPress',
         paramSchema: actionKeyboardPressParamSchema,
         call

package/dist/es/device/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"device/index.mjs","sources":["webpack://@midscene/core/./src/device/index.ts"],"sourcesContent":["import { getMidsceneLocationSchema } from '@/ai-model';\nimport type { DeviceAction, LocateResultElement } from '@/types';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;\n\n  abstract cacheFeatureForRect?(\n    rect: Rect,\n    opt?: { _orderSensitive: boolean },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  // @deprecated do NOT extend this method\n  abstract getContext?(): Promise<UIContext>;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\nexport const defineAction = <\n  TSchema extends z.ZodType,\n  TRuntime = z.infer<TSchema>,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema: TSchema;\n    call: (param: TRuntime) => Promise<void>;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\n// Override the inferred type to use LocateResultElement for the runtime locate field\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    call,\n  });\n};\n\n// Input\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be input')\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'append'])\n    .default('replace')\n    .optional()\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"append\" - append the value to existing content; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    call,\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z.string().describe('The key to be pressed'),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a function key, like \"Enter\", \"Tab\", \"Escape\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe('The direction to scroll'),\n  scrollType: z\n    .enum(['once', 'untilBottom', 'untilTop', 'untilRight', 'untilLeft'])\n    .default('once')\n    .describe('The scroll type'),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe('The element to be scrolled'),\n});\nexport type ActionScrollParam = {\n  direction?: 'down' | 'up' | 'right' | 'left';\n  scrollType?: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';\n  distance?: number | null;\n  locate?: LocateResultElement;\n};\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description: 'Drag and drop the element',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a swipe gesture. You must specify either \"end\" (target location) or \"distance\" + \"direction\" - they are mutually exclusive. Use \"end\" for precise location-based swipes, or \"distance\" + \"direction\" for relative movement.',\n    paramSchema: ActionSwipeParamSchema,\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The input field to be cleared'),\n});\nexport type ActionClearInputParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: 'Clear the text content of an input field',\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    call,\n  });\n};\n\nexport type { DeviceAction } from '../types';\n"],"names":["AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","actionInputParamSchema","val","String","defineActionInput","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput"],"mappings":";;AAOO,MAAeA;AAgCtB;AAIO,MAAMC,eAAe,CAI1BC,SAaOA;AAIF,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAMO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACbI;IACF;AAIK,MAAMC,8BAA8BJ,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACbD;IACF;AAIK,MAAMG,+BAA+BN,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACbH;IACF;AAIK,MAAMK,yBAAyBR,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACbL;IACF;AAIK,MAAMO,yBAAyBV,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACW,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQV,4BACL,QAAQ,CAAC,2BACT,QAAQ;IACX,MAAMD,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAS,EACnC,OAAO,CAAC,WACR,QAAQ,GACR,QAAQ,CACP;AAEN;AAOO,MAAMa,oBAAoB,CAC/BV,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaa;QACbP;IACF;AAIK,MAAMW,iCAAiCd,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAE,MAAM,GAAG,QAAQ,CAAC;AAC/B;AAMO,MAAMe,4BAA4B,CACvCZ,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaiB;QACbX;IACF;AAIK,MAAMa,0BAA0BhB,EAAE,MAAM,CAAC;IAC9C,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,YAAYA,CAAC,CAADA,OACL,CAAC;QAAC;QAAQ;QAAe;QAAY;QAAc;KAAY,EACnE,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CAAC;AACd;AAQO,MAAMgB,qBAAqB,CAChCd,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACbb;IACF;AAIK,MAAMe,+BAA+BlB,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMkB,0BAA0B,CACrChB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaqB;QACbf;IACF;AAGK,MAAMiB,6BAA6BpB,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMqB,wBAAwB,CACnClB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAauB;QACbjB;IACF;AAGK,MAAMmB,yBAAyBtB,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,MAAMuB,oBAAoB,CAC/BpB,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAayB;QACbnB;IACF;AAIK,MAAMqB,8BAA8BxB,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMwB,yBAAyB,CACpCtB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAa2B;QACbrB;IACF"}
+{"version":3,"file":"device/index.mjs","sources":["webpack://@midscene/core/./src/device/index.ts"],"sourcesContent":["import { getMidsceneLocationSchema } from '@/ai-model';\nimport type { DeviceAction, LocateResultElement } from '@/types';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;\n\n  abstract cacheFeatureForRect?(\n    rect: Rect,\n    opt?: { _orderSensitive: boolean },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  // @deprecated do NOT extend this method\n  abstract getContext?(): Promise<UIContext>;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\nexport const defineAction = <\n  TSchema extends z.ZodType,\n  TRuntime = z.infer<TSchema>,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema: TSchema;\n    call: (param: TRuntime) => Promise<void>;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\n// Override the inferred type to use LocateResultElement for the runtime locate field\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    call,\n  });\n};\n\n// Input\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be input')\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'append'])\n    .default('replace')\n    .optional()\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"append\" - append the value to existing content; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    call,\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe('The direction to scroll'),\n  scrollType: z\n    .enum(['once', 'untilBottom', 'untilTop', 'untilRight', 'untilLeft'])\n    .default('once')\n    .describe('The scroll type'),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe('The element to be scrolled'),\n});\nexport type ActionScrollParam = {\n  direction?: 'down' | 'up' | 'right' | 'left';\n  scrollType?: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';\n  distance?: number | null;\n  locate?: LocateResultElement;\n};\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description: 'Drag and drop the element',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a swipe gesture. You must specify either \"end\" (target location) or \"distance\" + \"direction\" - they are mutually exclusive. Use \"end\" for precise location-based swipes, or \"distance\" + \"direction\" for relative movement.',\n    paramSchema: ActionSwipeParamSchema,\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The input field to be cleared'),\n});\nexport type ActionClearInputParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: 'Clear the text content of an input field',\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    call,\n  });\n};\n\nexport type { DeviceAction } from '../types';\n"],"names":["AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","actionInputParamSchema","val","String","defineActionInput","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput"],"mappings":";;AAOO,MAAeA;AAgCtB;AAIO,MAAMC,eAAe,CAI1BC,SAaOA;AAIF,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAMO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACbI;IACF;AAIK,MAAMC,8BAA8BJ,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACbD;IACF;AAIK,MAAMG,+BAA+BN,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACbH;IACF;AAIK,MAAMK,yBAAyBR,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACbL;IACF;AAIK,MAAMO,yBAAyBV,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACW,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQV,4BACL,QAAQ,CAAC,2BACT,QAAQ;IACX,MAAMD,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAS,EACnC,OAAO,CAAC,WACR,QAAQ,GACR,QAAQ,CACP;AAEN;AAOO,MAAMa,oBAAoB,CAC/BV,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaa;QACbP;IACF;AAIK,MAAMW,iCAAiCd,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMe,4BAA4B,CACvCZ,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaiB;QACbX;IACF;AAIK,MAAMa,0BAA0BhB,EAAE,MAAM,CAAC;IAC9C,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,YAAYA,CAAC,CAADA,OACL,CAAC;QAAC;QAAQ;QAAe;QAAY;QAAc;KAAY,EACnE,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CAAC;AACd;AAQO,MAAMgB,qBAAqB,CAChCd,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACbb;IACF;AAIK,MAAMe,+BAA+BlB,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMkB,0BAA0B,CACrChB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaqB;QACbf;IACF;AAGK,MAAMiB,6BAA6BpB,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMqB,wBAAwB,CACnClB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAauB;QACbjB;IACF;AAGK,MAAMmB,yBAAyBtB,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,MAAMuB,oBAAoB,CAC/BpB,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAayB;QACbnB;IACF;AAIK,MAAMqB,8BAA8BxB,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMwB,yBAAyB,CACpCtB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAa2B;QACbrB;IACF"}

package/dist/es/insight/index.mjs

@@ -51,7 +51,7 @@ class Insight {
             searchArea = searchAreaResponse.rect;
         }
         const startTime = Date.now();
-        const { parseResult, rect, elementById, rawResponse, usage, isOrderSensitive } = await AiLocateElement({
+        const { parseResult, rect, rawResponse, usage } = await AiLocateElement({
             callAIFn: this.aiVendorFn,
             context,
             targetElementDescription: queryPrompt,
@@ -83,14 +83,7 @@ class Insight {
             deepThink: !!searchArea,
             error: errorLog
         };
-        const elements = [];
-        (parseResult.elements || []).forEach((item)=>{
-            if ('id' in item) {
-                const element = elementById(null == item ? void 0 : item.id);
-                if (!element) return void console.warn(`locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`);
-                elements.push(element);
-            }
-        });
+        const elements = parseResult.elements || [];
         const dump = createInsightDump({
             ...dumpData,
             matchedElement: elements
@@ -100,12 +93,9 @@ class Insight {
         if (1 === elements.length) return {
             element: {
                 id: elements[0].id,
-                indexId: elements[0].indexId,
                 center: elements[0].center,
                 rect: elements[0].rect,
-                xpaths: elements[0].xpaths || [],
-                attributes: elements[0].attributes,
-                isOrderSensitive
+                isOrderSensitive: elements[0].isOrderSensitive
             },
             rect,
             dump
@@ -116,7 +106,7 @@ class Insight {
             dump
         };
     }
-    async extract(dataDemand, modelConfig, opt, multimodalPrompt) {
+    async extract(dataDemand, modelConfig, opt, pageDescription, multimodalPrompt) {
         var _parseResult_errors;
         assert('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
         const context = await this.contextRetrieverFn();
@@ -126,7 +116,8 @@ class Insight {
             dataQuery: dataDemand,
             multimodalPrompt,
             extractOption: opt,
-            modelConfig
+            modelConfig,
+            pageDescription
         });
         const timeCost = Date.now() - startTime;
         const taskInfo = {

package/dist/es/insight/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"insight/index.mjs","sources":["webpack://@midscene/core/./src/insight/index.ts"],"sourcesContent":["import { AIActionType, type AIArgs, expandSearchArea } from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  InsightAction,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightExtractResult,\n  InsightTaskInfo,\n  LocateResultWithDump,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { InsightError } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_FORCE_DEEP_THINK,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../ai-model/common';\nimport { createInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext<BaseElement>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface InsightOptions {\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n  aiVendorFn?: typeof callAIWithObjectResponse;\n}\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: () => Promise<ContextType> | ContextType;\n\n  aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined> =\n    callAIWithObjectResponse;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: ContextType | (() => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    // just for unit test, aiVendorFn is callAIWithObjectResponse by default\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { vlMode } = modelConfig;\n\n    if (searchAreaPrompt && !vlMode) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      elementById,\n      rawResponse,\n      usage,\n      isOrderSensitive,\n    } = await AiLocateElement({\n      callAIFn: this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    const dump = createInsightDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new InsightError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new InsightError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n          isOrderSensitive,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<InsightExtractResult<T>> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const context = await this.contextRetrieverFn();\n\n    const startTime = Date.now();\n\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    const dump = createInsightDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data && !opt?.doNotThrowError) {\n      throw new InsightError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn();\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { vlMode } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size, vlMode);\n      debug('describe: set searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        vlMode === 'qwen-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn = this\n      .aiVendorFn as typeof callAIWithObjectResponse<AIDescribeElementResponse>;\n\n    const res = await callAIFn(\n      msgs,\n      AIActionType.DESCRIBE_ELEMENT,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Insight","query","opt","modelConfig","_parseResult_errors","queryPrompt","assert","globalDeepThinkSwitch","globalConfigManager","MIDSCENE_FORCE_DEEP_THINK","searchAreaPrompt","vlMode","console","undefined","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","elementById","rawResponse","usage","isOrderSensitive","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","item","element","dump","createInsightDump","InsightError","dataDemand","multimodalPrompt","AiExtractElementInfo","data","thought","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","callAIFn","res","AIActionType","content","callAIWithObjectResponse","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;;AAgDA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IA+BnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACM;YAgF3BC;QA/EJ,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMM,wBAAwBC,oBAAoB,qBAAqB,CACrEC;QAEF,IAAIF,uBACFT,MAAM,yBAAyBS;QAEjC,IAAIG;QACJ,IAAIT,MAAM,SAAS,IAAIM,uBACrBG,mBAAmBT,MAAM,MAAM;QAGjC,MAAM,EAAEU,MAAM,EAAE,GAAGR;QAEnB,IAAIO,oBAAoB,CAACC,QAAQ;YAC/BC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,MAAMC,UAAUZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAM,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIa;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,gBAAgB;gBACzCL;gBACA,oBAAoBJ;gBACpBP;YACF;YACAG,OACEY,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,WAAW,EACXC,KAAK,EACLC,gBAAgB,EACjB,GAAG,MAAMC,gBAAgB;YACxB,UAAU,IAAI,CAAC,UAAU;YACzBd;YACA,0BAA0BT;YAC1B,cAAca;YACdf;QACF;QAEA,MAAM0B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAX;YACAC;YACAC;QACF;QAEA,IAAIe;QACJ,IAAI,QAAA5B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B4B,WAAW,CAAC,6BAA6B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG5E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS5B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAakB;YACb,MAAM;YACNO;YACA,WAAW,CAAC,CAACf;YACb,OAAOiB;QACT;QAEA,MAAME,WAA0B,EAAE;QACjCZ,CAAAA,YAAY,QAAQ,IAAI,EAAC,EAAG,OAAO,CAAC,CAACa;YACpC,IAAI,QAAQA,MAAM;gBAChB,MAAMC,UAAUZ,YAAYW,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE;gBAEpC,IAAI,CAACC,SAAS,YACZxB,QAAQ,IAAI,CACV,CAAC,+BAA+B,EAAEuB,KAAK,EAAE,CAAC,0CAA0C,CAAC;gBAIzFD,SAAS,IAAI,CAACE;YAChB;QACF;QAEA,MAAMC,OAAOC,kBAAkB;YAC7B,GAAGL,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIO,aAAaP,UAAUK;QAGnC,IAAIH,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIK,aACR,CAAC,0CAA0C,EAAEL,SAAS,MAAM,EAAE,EAC9DG;QAIJ,IAAIH,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,IAAIA,QAAQ,CAAC,EAAE,CAAE,EAAE;gBACnB,SAASA,QAAQ,CAAC,EAAE,CAAE,OAAO;gBAC7B,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM,IAAI,EAAE;gBACjC,YAAYA,QAAQ,CAAC,EAAE,CAAE,UAAU;gBACnCP;YACF;YACAJ;YACAc;QACF;QAGF,OAAO;YACL,SAAS;YACTd;YACAc;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BrC,WAAyB,EACzBD,GAA0B,EAC1BuC,gBAAoC,EACF;YAyB9BrC;QAxBJE,OACE,AAAsB,YAAtB,OAAOkC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAM1B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAE7C,MAAMM,YAAYC,KAAK,GAAG;QAE1B,MAAM,EAAEC,WAAW,EAAEI,KAAK,EAAE,GAAG,MAAMgB,qBAAwB;YAC3D5B;YACA,WAAW0B;YACXC;YACA,eAAevC;YACfC;QACF;QAEA,MAAM0B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACT;QAC9B;QAEA,IAAIU;QACJ,IAAI,QAAA5B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B4B,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTO;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNV;YACA,OAAOE;QACT;QAEA,MAAM,EAAEW,IAAI,EAAEC,OAAO,EAAE,GAAGtB,eAAe,CAAC;QAG1C,MAAMe,OAAOC,kBAAkB;YAC7B,GAAGL,QAAQ;YACXU;QACF;QAEA,IAAIX,YAAY,CAACW,QAAQ,CAACzC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,GAC3C,MAAM,IAAIqC,aAAaP,UAAUK;QAGnC,OAAO;YACLM;YACAC;YACAlB;YACAW;QACF;IACF;IAEA,MAAM,SACJQ,MAA+B,EAC/B1C,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOuC,QAAQ;QACf,MAAM/B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEgC,gBAAgB,EAAEC,IAAI,EAAE,GAAGjC;QACnCR,OAAOwC,kBAAkB;QAEzB,MAAM,EAAEnC,MAAM,EAAE,GAAGR;QACnB,MAAM6C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIjD,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,EAAE;YAClB,MAAMa,aAAayC,iBAAiBL,YAAYrC,QAAQ,IAAI,EAAEH;YAC9Db,MAAM,4BAA4BiB;YAClC,MAAM0C,gBAAgB,MAAMC,WAC1BJ,cACAvC,YACAJ,AAAW,cAAXA;YAEF2C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,WAAW,IAAI,CAClB,UAAU;QAEb,MAAMC,MAAM,MAAMD,SAChBD,MACAG,aAAa,gBAAgB,EAC7B3D;QAGF,MAAM,EAAE4D,OAAO,EAAE,GAAGF;QACpBvD,OAAO,CAACyD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DzD,OAAOyD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IA7TA,YACEjD,OAAiE,EACjEZ,GAAoB,CACpB;QAVF;QAEA,qCACE8D;QAEF;QAME1D,OAAOQ,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMmD,QAAQ,OAAO,CAACnD;QAIlD,IAAI,AAA2B,WAApBZ,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,UAAU,AAAD,GACvB,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,QAAQ,AAAD,GACrB,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AA4SF"}
+{"version":3,"file":"insight/index.mjs","sources":["webpack://@midscene/core/./src/insight/index.ts"],"sourcesContent":["import { AIActionType, type AIArgs, expandSearchArea } from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightExtractResult,\n  InsightTaskInfo,\n  LocateResultWithDump,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { InsightError } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_FORCE_DEEP_THINK,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../ai-model/common';\nimport { createInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface InsightOptions {\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n  aiVendorFn?: typeof callAIWithObjectResponse;\n}\n\nconst debug = getDebug('ai:insight');\nexport default class Insight {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined> =\n    callAIWithObjectResponse;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    // just for unit test, aiVendorFn is callAIWithObjectResponse by default\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { vlMode } = modelConfig;\n\n    if (searchAreaPrompt && !vlMode) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage } = await AiLocateElement({\n      callAIFn: this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createInsightDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new InsightError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new InsightError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          isOrderSensitive: elements[0]!.isOrderSensitive,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<InsightExtractResult<T>> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const context = await this.contextRetrieverFn();\n\n    const startTime = Date.now();\n\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n      modelConfig,\n      pageDescription,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    const dump = createInsightDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data && !opt?.doNotThrowError) {\n      throw new InsightError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn();\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { vlMode } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size, vlMode);\n      debug('describe: set searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        vlMode === 'qwen-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn = this\n      .aiVendorFn as typeof callAIWithObjectResponse<AIDescribeElementResponse>;\n\n    const res = await callAIFn(\n      msgs,\n      AIActionType.DESCRIBE_ELEMENT,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Insight","query","opt","modelConfig","_parseResult_errors","queryPrompt","assert","globalDeepThinkSwitch","globalConfigManager","MIDSCENE_FORCE_DEEP_THINK","searchAreaPrompt","vlMode","console","undefined","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","dump","createInsightDump","InsightError","dataDemand","pageDescription","multimodalPrompt","AiExtractElementInfo","data","thought","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","callAIFn","res","AIActionType","content","callAIWithObjectResponse","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;;AA8CA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IA4BnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACM;YAyE3BC;QAxEJ,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMM,wBAAwBC,oBAAoB,qBAAqB,CACrEC;QAEF,IAAIF,uBACFT,MAAM,yBAAyBS;QAEjC,IAAIG;QACJ,IAAIT,MAAM,SAAS,IAAIM,uBACrBG,mBAAmBT,MAAM,MAAM;QAGjC,MAAM,EAAEU,MAAM,EAAE,GAAGR;QAEnB,IAAIO,oBAAoB,CAACC,QAAQ;YAC/BC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,MAAMC,UAAUZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAM,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIa;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,gBAAgB;gBACzCL;gBACA,oBAAoBJ;gBACpBP;YACF;YACAG,OACEY,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,gBAAgB;YACtE,UAAU,IAAI,CAAC,UAAU;YACzBZ;YACA,0BAA0BT;YAC1B,cAAca;YACdf;QACF;QAEA,MAAMwB,WAAWN,KAAK,GAAG,KAAKD;QAC9B,MAAMQ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACL;YAC5B,gBAAgBK,KAAK,SAAS,CAACP;YAC/BG;YACAV;YACAC;YACAC;QACF;QAEA,IAAIa;QACJ,IAAI,QAAA1B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B0B,WAAW,CAAC,6BAA6B,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG5E,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS1B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAakB;YACb,MAAM;YACNK;YACA,WAAW,CAAC,CAACb;YACb,OAAOe;QACT;QAEA,MAAME,WAAWV,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMW,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,aAAaL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,aACR,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,IAAIA,QAAQ,CAAC,EAAE,CAAE,EAAE;gBACnB,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,kBAAkBA,QAAQ,CAAC,EAAE,CAAE,gBAAgB;YACjD;YACAT;YACAU;QACF;QAGF,OAAO;YACL,SAAS;YACTV;YACAU;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BjC,WAAyB,EACzBD,GAA0B,EAC1BmC,eAAwB,EACxBC,gBAAoC,EACF;YA0B9BlC;QAzBJE,OACE,AAAsB,YAAtB,OAAO8B,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAMtB,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAE7C,MAAMM,YAAYC,KAAK,GAAG;QAE1B,MAAM,EAAEC,WAAW,EAAEG,KAAK,EAAE,GAAG,MAAMc,qBAAwB;YAC3DzB;YACA,WAAWsB;YACXE;YACA,eAAepC;YACfC;YACAkC;QACF;QAEA,MAAMV,WAAWN,KAAK,GAAG,KAAKD;QAC9B,MAAMQ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACP;QAC9B;QAEA,IAAIQ;QACJ,IAAI,QAAA1B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B0B,WAAW,CAAC,qBAAqB,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNR;YACA,OAAOE;QACT;QAEA,MAAM,EAAEU,IAAI,EAAEC,OAAO,EAAE,GAAGnB,eAAe,CAAC;QAG1C,MAAMW,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACXS;QACF;QAEA,IAAIV,YAAY,CAACU,QAAQ,CAACtC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,GAC3C,MAAM,IAAIiC,aAAaL,UAAUG;QAGnC,OAAO;YACLO;YACAC;YACAhB;YACAQ;QACF;IACF;IAEA,MAAM,SACJS,MAA+B,EAC/BvC,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOoC,QAAQ;QACf,MAAM5B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAE6B,gBAAgB,EAAEC,IAAI,EAAE,GAAG9B;QACnCR,OAAOqC,kBAAkB;QAEzB,MAAM,EAAEhC,MAAM,EAAE,GAAGR;QACnB,MAAM0C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAI9C,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,EAAE;YAClB,MAAMa,aAAasC,iBAAiBL,YAAYlC,QAAQ,IAAI,EAAEH;YAC9Db,MAAM,4BAA4BiB;YAClC,MAAMuC,gBAAgB,MAAMC,WAC1BJ,cACApC,YACAJ,AAAW,cAAXA;YAEFwC,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,WAAW,IAAI,CAClB,UAAU;QAEb,MAAMC,MAAM,MAAMD,SAChBD,MACAG,aAAa,gBAAgB,EAC7BxD;QAGF,MAAM,EAAEyD,OAAO,EAAE,GAAGF;QACpBpD,OAAO,CAACsD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DtD,OAAOsD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAxSA,YACE9C,OAA2D,EAC3DZ,GAAoB,CACpB;QAVF;QAEA,qCACE2D;QAEF;QAMEvD,OAAOQ,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMgD,QAAQ,OAAO,CAAChD;QAIlD,IAAI,AAA2B,WAApBZ,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,UAAU,AAAD,GACvB,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,QAAQ,AAAD,GACrB,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAuRF"}

package/dist/es/tree.mjs

@@ -1,2 +1,13 @@
-import { descriptionOfTree, trimAttributes, truncateText } from "@midscene/shared/extractor";
-export { descriptionOfTree, trimAttributes, truncateText };
+import { descriptionOfTree, treeToList, trimAttributes, truncateText } from "@midscene/shared/extractor";
+const ELEMENT_COUNT_WARNING_THRESHOLD = 5000;
+const TREE_SIZE_WARNING_MESSAGE = 'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: "visible-only" to reduce the number of elements';
+function tree_descriptionOfTree(tree, truncateTextLength, filterNonTextContent = false, visibleOnly = true) {
+    if (!visibleOnly) {
+        const flatElements = treeToList(tree);
+        if (flatElements.length >= ELEMENT_COUNT_WARNING_THRESHOLD) console.warn(TREE_SIZE_WARNING_MESSAGE);
+    }
+    return descriptionOfTree(tree, truncateTextLength, filterNonTextContent, visibleOnly);
+}
+export { tree_descriptionOfTree as descriptionOfTree, trimAttributes, truncateText };
+
+//# sourceMappingURL=tree.mjs.map

package/dist/es/tree.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"tree.mjs","sources":["webpack://@midscene/core/./src/tree.ts"],"sourcesContent":["import type { BaseElement, ElementTreeNode } from '@midscene/shared/types';\n\nimport {\n  descriptionOfTree as sharedDescriptionOfTree,\n  treeToList,\n  trimAttributes,\n  truncateText,\n} from '@midscene/shared/extractor';\n\nconst ELEMENT_COUNT_WARNING_THRESHOLD = 5000;\nconst TREE_SIZE_WARNING_MESSAGE =\n  'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: \"visible-only\" to reduce the number of elements';\n\nexport { trimAttributes, truncateText };\n\nexport function descriptionOfTree<\n  ElementType extends BaseElement = BaseElement,\n>(\n  tree: ElementTreeNode<ElementType>,\n  truncateTextLength?: number,\n  filterNonTextContent = false,\n  visibleOnly = true,\n) {\n  if (!visibleOnly) {\n    const flatElements = treeToList(tree);\n    if (flatElements.length >= ELEMENT_COUNT_WARNING_THRESHOLD) {\n      console.warn(TREE_SIZE_WARNING_MESSAGE);\n    }\n  }\n\n  return sharedDescriptionOfTree(\n    tree,\n    truncateTextLength,\n    filterNonTextContent,\n    visibleOnly,\n  );\n}\n"],"names":["ELEMENT_COUNT_WARNING_THRESHOLD","TREE_SIZE_WARNING_MESSAGE","descriptionOfTree","tree","truncateTextLength","filterNonTextContent","visibleOnly","flatElements","treeToList","console","sharedDescriptionOfTree"],"mappings":";AASA,MAAMA,kCAAkC;AACxC,MAAMC,4BACJ;AAIK,SAASC,uBAGdC,IAAkC,EAClCC,kBAA2B,EAC3BC,uBAAuB,KAAK,EAC5BC,cAAc,IAAI;IAElB,IAAI,CAACA,aAAa;QAChB,MAAMC,eAAeC,WAAWL;QAChC,IAAII,aAAa,MAAM,IAAIP,iCACzBS,QAAQ,IAAI,CAACR;IAEjB;IAEA,OAAOS,kBACLP,MACAC,oBACAC,sBACAC;AAEJ"}