@midscene/core 0.26.5-beta-20250814095614.0 → 0.26.5-beta-20250814125155.0

package/dist/es/ai-model/inspect.mjs

@@ -0,0 +1,317 @@
+import { MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, getAIConfigInBoolean, vlLocateMode } from "@midscene/shared/env";
+import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+import { assert } from "@midscene/shared/utils";
+import { AIActionType, adaptBboxToRect, callAiFn, expandSearchArea, markupImageForLLM, mergeRects } from "./common.mjs";
+import { systemPromptToAssert } from "./prompt/assertion.mjs";
+import { extractDataQueryPrompt, systemPromptToExtract } from "./prompt/extraction.mjs";
+import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
+import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
+import { describeUserPage, distance, distanceThreshold, elementByPositionWithElementInfo } from "./prompt/util.mjs";
+import { callToGetJSONObject } from "./service-caller/index.mjs";
+const debugInspect = getDebug('ai:inspect');
+const debugSection = getDebug('ai:section');
+const extraTextFromUserPrompt = (prompt)=>{
+    if ('string' == typeof prompt) return prompt;
+    return prompt.prompt;
+};
+const promptsToChatParam = async (multimodalPrompt)=>{
+    var _multimodalPrompt_images;
+    const msgs = [];
+    if (null == multimodalPrompt ? void 0 : null == (_multimodalPrompt_images = multimodalPrompt.images) ? void 0 : _multimodalPrompt_images.length) {
+        msgs.push({
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: 'Next, I will provide all the reference images.'
+                }
+            ]
+        });
+        for (const item of multimodalPrompt.images){
+            const base64 = await preProcessImageUrl(item.url, !!multimodalPrompt.convertHttpImage2Base64);
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'text',
+                        text: `reference image ${item.name}:`
+                    }
+                ]
+            });
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: base64,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            });
+        }
+    }
+    return msgs;
+};
+async function AiLocateElement(options) {
+    const { context, targetElementDescription, callAI } = options;
+    const { screenshotBase64 } = context;
+    const { description, elementById, insertElementByPosition } = await describeUserPage(context);
+    assert(targetElementDescription, "cannot find the target element description");
+    const userInstructionPrompt = await findElementPrompt.format({
+        pageDescription: description,
+        targetElementDescription: extraTextFromUserPrompt(targetElementDescription)
+    });
+    const systemPrompt = systemPromptToLocateElement(vlLocateMode());
+    let imagePayload = screenshotBase64;
+    if (options.searchConfig) {
+        assert(options.searchConfig.rect, 'searchArea is provided but its rect cannot be found. Failed to locate element');
+        assert(options.searchConfig.imageBase64, 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element');
+        imagePayload = options.searchConfig.imageBase64;
+    } else if ('qwen-vl' === vlLocateMode()) imagePayload = await paddingToMatchBlockByBase64(imagePayload);
+    else if (!vlLocateMode()) imagePayload = await markupImageForLLM(screenshotBase64, context.tree, context.size);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: userInstructionPrompt
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof targetElementDescription) {
+        const addOns = await promptsToChatParam({
+            images: targetElementDescription.images,
+            convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const callAIFn = callAI || callToGetJSONObject;
+    const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT);
+    const rawResponse = JSON.stringify(res.content);
+    let resRect;
+    let matchedElements = 'elements' in res.content ? res.content.elements : [];
+    let errors = 'errors' in res.content ? res.content.errors : [];
+    try {
+        if ('bbox' in res.content && Array.isArray(res.content.bbox)) {
+            var _options_searchConfig_rect, _options_searchConfig, _options_searchConfig_rect1, _options_searchConfig1, _options_searchConfig_rect2, _options_searchConfig2, _options_searchConfig_rect3, _options_searchConfig3;
+            resRect = adaptBboxToRect(res.content.bbox, (null == (_options_searchConfig = options.searchConfig) ? void 0 : null == (_options_searchConfig_rect = _options_searchConfig.rect) ? void 0 : _options_searchConfig_rect.width) || context.size.width, (null == (_options_searchConfig1 = options.searchConfig) ? void 0 : null == (_options_searchConfig_rect1 = _options_searchConfig1.rect) ? void 0 : _options_searchConfig_rect1.height) || context.size.height, null == (_options_searchConfig2 = options.searchConfig) ? void 0 : null == (_options_searchConfig_rect2 = _options_searchConfig2.rect) ? void 0 : _options_searchConfig_rect2.left, null == (_options_searchConfig3 = options.searchConfig) ? void 0 : null == (_options_searchConfig_rect3 = _options_searchConfig3.rect) ? void 0 : _options_searchConfig_rect3.top);
+            debugInspect('resRect', resRect);
+            const rectCenter = {
+                x: resRect.left + resRect.width / 2,
+                y: resRect.top + resRect.height / 2
+            };
+            let element = elementByPositionWithElementInfo(context.tree, rectCenter);
+            const distanceToCenter = element ? distance({
+                x: element.center[0],
+                y: element.center[1]
+            }, rectCenter) : 0;
+            if (!element || distanceToCenter > distanceThreshold) element = insertElementByPosition(rectCenter);
+            if (element) {
+                matchedElements = [
+                    element
+                ];
+                errors = [];
+            }
+        }
+    } catch (e) {
+        const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
+        if (errors && (null == errors ? void 0 : errors.length) !== 0) errors.push(`(${msg})`);
+        else errors = [
+            msg
+        ];
+    }
+    return {
+        rect: resRect,
+        parseResult: {
+            elements: matchedElements,
+            errors
+        },
+        rawResponse,
+        elementById,
+        usage: res.usage,
+        isOrderSensitive: 'object' == typeof res.content && null !== res.content && 'isOrderSensitive' in res.content ? res.content.isOrderSensitive : void 0
+    };
+}
+async function AiLocateSection(options) {
+    const { context, sectionDescription } = options;
+    const { screenshotBase64 } = context;
+    const systemPrompt = systemPromptToLocateSection(vlLocateMode());
+    const sectionLocatorInstructionText = await sectionLocatorInstruction.format({
+        sectionDescription: extraTextFromUserPrompt(sectionDescription)
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: screenshotBase64,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: sectionLocatorInstructionText
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof sectionDescription) {
+        const addOns = await promptsToChatParam({
+            images: sectionDescription.images,
+            convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const result = await callAiFn(msgs, AIActionType.EXTRACT_DATA);
+    let sectionRect;
+    const sectionBbox = result.content.bbox;
+    if (sectionBbox) {
+        const targetRect = adaptBboxToRect(sectionBbox, context.size.width, context.size.height);
+        debugSection('original targetRect %j', targetRect);
+        const referenceBboxList = result.content.references_bbox || [];
+        debugSection('referenceBboxList %j', referenceBboxList);
+        const referenceRects = referenceBboxList.filter((bbox)=>Array.isArray(bbox)).map((bbox)=>adaptBboxToRect(bbox, context.size.width, context.size.height));
+        debugSection('referenceRects %j', referenceRects);
+        const mergedRect = mergeRects([
+            targetRect,
+            ...referenceRects
+        ]);
+        debugSection('mergedRect %j', mergedRect);
+        sectionRect = expandSearchArea(mergedRect, context.size);
+        debugSection('expanded sectionRect %j', sectionRect);
+    }
+    let imageBase64 = screenshotBase64;
+    if (sectionRect) imageBase64 = await cropByRect(screenshotBase64, sectionRect, getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL));
+    return {
+        rect: sectionRect,
+        imageBase64,
+        error: result.content.error,
+        rawResponse: JSON.stringify(result.content),
+        usage: result.usage
+    };
+}
+async function AiExtractElementInfo(options) {
+    var _options_extractOption;
+    const { dataQuery, context, extractOption, multimodalPrompt } = options;
+    const systemPrompt = systemPromptToExtract();
+    const { screenshotBase64 } = context;
+    const { description, elementById } = await describeUserPage(context, {
+        truncateTextLength: 200,
+        filterNonTextContent: false,
+        visibleOnly: false,
+        domIncluded: null == extractOption ? void 0 : extractOption.domIncluded
+    });
+    const extractDataPromptText = await extractDataQueryPrompt(description, dataQuery);
+    const userContent = [];
+    if ((null == extractOption ? void 0 : extractOption.screenshotIncluded) !== false) userContent.push({
+        type: 'image_url',
+        image_url: {
+            url: screenshotBase64,
+            detail: 'high'
+        }
+    });
+    userContent.push({
+        type: 'text',
+        text: extractDataPromptText
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userContent
+        }
+    ];
+    if (null == (_options_extractOption = options.extractOption) ? void 0 : _options_extractOption.returnThought) msgs.push({
+        role: 'user',
+        content: 'Please provide reasons.'
+    });
+    if (multimodalPrompt) {
+        const addOns = await promptsToChatParam({
+            images: multimodalPrompt.images,
+            convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const result = await callAiFn(msgs, AIActionType.EXTRACT_DATA);
+    return {
+        parseResult: result.content,
+        elementById,
+        usage: result.usage
+    };
+}
+async function AiAssert(options) {
+    const { assertion, context } = options;
+    assert(assertion, 'assertion should not be empty');
+    const { screenshotBase64 } = context;
+    const systemPrompt = systemPromptToAssert({
+        isUITars: getAIConfigInBoolean(MIDSCENE_USE_VLM_UI_TARS)
+    });
+    const assertionText = extraTextFromUserPrompt(assertion);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: screenshotBase64,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: `
+Here is the assertion. Please tell whether it is truthy according to the screenshot.
+=====================================
+${assertionText}
+=====================================
+  `
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof assertion) {
+        const addOns = await promptsToChatParam({
+            images: assertion.images,
+            convertHttpImage2Base64: assertion.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const { content: assertResult, usage } = await callAiFn(msgs, AIActionType.ASSERT);
+    return {
+        content: assertResult,
+        usage
+    };
+}
+export { AiAssert, AiExtractElementInfo, AiLocateElement, AiLocateSection };
+
+//# sourceMappingURL=inspect.mjs.map

package/dist/es/ai-model/inspect.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIAssertionResponse,\n  AIDataExtractionResponse,\n  AIElementLocatorResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  BaseElement,\n  ElementById,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  TMultimodalPrompt,\n  TUserPrompt,\n  UIContext,\n} from '@/types';\nimport {\n  MIDSCENE_USE_QWEN_VL,\n  MIDSCENE_USE_VLM_UI_TARS,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  callAiFn,\n  expandSearchArea,\n  markupImageForLLM,\n  mergeRects,\n} from './common';\nimport { systemPromptToAssert } from './prompt/assertion';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  describeUserPage,\n  distance,\n  distanceThreshold,\n  elementByPositionWithElementInfo,\n} from './prompt/util';\nimport { callToGetJSONObject } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement<\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  context: UIContext<ElementType>;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAI?: typeof callAiFn<AIElementResponse | [number, number]>;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n}): Promise<{\n  parseResult: AIElementLocatorResponse;\n  rect?: Rect;\n  rawResponse: string;\n  elementById: ElementById;\n  usage?: AIUsageInfo;\n  isOrderSensitive?: boolean;\n}> {\n  const { context, targetElementDescription, callAI } = options;\n  const { screenshotBase64 } = context;\n  const { description, elementById, insertElementByPosition } =\n    await describeUserPage(context);\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: description,\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(vlLocateMode());\n\n  let imagePayload = screenshotBase64;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n  } else if (vlLocateMode() === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlLocateMode()) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const callAIFn =\n    callAI || callToGetJSONObject<AIElementResponse | [number, number]>;\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: AIElementLocatorResponse['elements'] =\n    'elements' in res.content ? res.content.elements : [];\n  let errors: AIElementLocatorResponse['errors'] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        options.searchConfig?.rect?.width || context.size.width,\n        options.searchConfig?.rect?.height || context.size.height,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n      );\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n      let element = elementByPositionWithElementInfo(context.tree, rectCenter);\n\n      const distanceToCenter = element\n        ? distance({ x: element.center[0], y: element.center[1] }, rectCenter)\n        : 0;\n\n      if (!element || distanceToCenter > distanceThreshold) {\n        element = insertElementByPosition(rectCenter);\n      }\n\n      if (element) {\n        matchedElements = [element];\n        errors = [];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements,\n      errors,\n    },\n    rawResponse,\n    elementById,\n    usage: res.usage,\n    isOrderSensitive:\n      typeof res.content === 'object' &&\n      res.content !== null &&\n      'isOrderSensitive' in res.content\n        ? (res.content as any).isOrderSensitive\n        : undefined,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext<BaseElement>;\n  sectionDescription: TUserPrompt;\n  callAI?: typeof callAiFn<AISectionLocatorResponse>;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlLocateMode());\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAiFn<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(bbox, context.size.width, context.size.height);\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    imageBase64 = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL),\n    );\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<\n  T,\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext<ElementType>;\n  extractOption?: InsightExtractOption;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt } = options;\n  const systemPrompt = systemPromptToExtract();\n\n  const { screenshotBase64 } = context;\n  const { description, elementById } = await describeUserPage(context, {\n    truncateTextLength: 200,\n    filterNonTextContent: false,\n    visibleOnly: false,\n    domIncluded: extractOption?.domIncluded,\n  });\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    description,\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (options.extractOption?.returnThought) {\n    msgs.push({\n      role: 'user',\n      content: 'Please provide reasons.',\n    });\n  }\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAiFn<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n  );\n  return {\n    parseResult: result.content,\n    elementById,\n    usage: result.usage,\n  };\n}\n\nexport async function AiAssert<\n  ElementType extends BaseElement = BaseElement,\n>(options: { assertion: TUserPrompt; context: UIContext<ElementType> }) {\n  const { assertion, context } = options;\n\n  assert(assertion, 'assertion should not be empty');\n\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToAssert({\n    isUITars: getAIConfigInBoolean(MIDSCENE_USE_VLM_UI_TARS),\n  });\n\n  const assertionText = extraTextFromUserPrompt(assertion);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: `\nHere is the assertion. Please tell whether it is truthy according to the screenshot.\n=====================================\n${assertionText}\n=====================================\n  `,\n        },\n      ],\n    },\n  ];\n\n  if (typeof assertion !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: assertion.images,\n      convertHttpImage2Base64: assertion.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const { content: assertResult, usage } = await callAiFn<AIAssertionResponse>(\n    msgs,\n    AIActionType.ASSERT,\n  );\n  return {\n    content: assertResult,\n    usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAI","screenshotBase64","description","elementById","insertElementByPosition","describeUserPage","assert","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","vlLocateMode","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","addOns","callAIFn","callToGetJSONObject","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect","_options_searchConfig_rect1","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","elementByPositionWithElementInfo","distanceToCenter","distance","distanceThreshold","e","msg","Error","undefined","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAiFn","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","cropByRect","getAIConfigInBoolean","MIDSCENE_USE_QWEN_VL","AiExtractElementInfo","_options_extractOption","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiAssert","assertion","systemPromptToAssert","MIDSCENE_USE_VLM_UI_TARS","assertionText","assertResult","usage"],"mappings":";;;;;;;;;;;AAmEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAEpBC,OAMD;IAQC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,MAAM,EAAE,GAAGH;IACtD,MAAM,EAAEI,gBAAgB,EAAE,GAAGH;IAC7B,MAAM,EAAEI,WAAW,EAAEC,WAAW,EAAEC,uBAAuB,EAAE,GACzD,MAAMC,iBAAiBP;IAEzBQ,OACEP,0BACA;IAGF,MAAMQ,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiBN;QACjB,0BAA0Bf,wBAAwBY;IACpD;IACA,MAAMU,eAAeC,4BAA4BC;IAEjD,IAAIC,eAAeX;IAEnB,IAAIJ,QAAQ,YAAY,EAAE;QACxBS,OACET,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFS,OACET,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFe,eAAef,QAAQ,YAAY,CAAC,WAAW;IACjD,OAAO,IAAIc,AAAmB,cAAnBA,gBACTC,eAAe,MAAMC,4BAA4BD;SAC5C,IAAI,CAACD,gBACVC,eAAe,MAAME,kBACnBb,kBACAH,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhB,MAAMN,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKG;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAML;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOR,0BAAuC;QAChD,MAAMgB,SAAS,MAAM1B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAMC,WACJhB,UAAUiB;IAEZ,MAAMC,MAAM,MAAMF,SAASxB,MAAM2B,aAAa,eAAe;IAE7D,MAAMC,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBACF,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IACvD,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAG1DQ,4BAAAA,uBACAC,6BAAAA,wBACAC,6BAAAA,wBACAC,6BAAAA;YALFP,UAAUQ,gBACRZ,IAAI,OAAO,CAAC,IAAI,EAChBQ,AAAAA,SAAAA,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,6BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,2BAA4B,KAAK,AAAD,KAAK5B,QAAQ,IAAI,CAAC,KAAK,EACvD6B,AAAAA,SAAAA,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,MAAM,AAAD,KAAK7B,QAAQ,IAAI,CAAC,MAAM,UACzD8B,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG;YAEjC7C,aAAa,WAAWsC;YAExB,MAAMS,aAAa;gBACjB,GAAGT,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YACA,IAAIU,UAAUC,iCAAiCnC,QAAQ,IAAI,EAAEiC;YAE7D,MAAMG,mBAAmBF,UACrBG,SAAS;gBAAE,GAAGH,QAAQ,MAAM,CAAC,EAAE;gBAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;YAAC,GAAGD,cACzD;YAEJ,IAAI,CAACC,WAAWE,mBAAmBE,mBACjCJ,UAAU5B,wBAAwB2B;YAGpC,IAAIC,SAAS;gBACXT,kBAAkB;oBAACS;iBAAQ;gBAC3BR,SAAS,EAAE;YACb;QACF;IACF,EAAE,OAAOa,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACb,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEc,IAAI,CAAC,CAAC;aAFtBd,SAAS;YAACc;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMhB;QACN,aAAa;YACX,UAAUC;YACVC;QACF;QACAJ;QACAjB;QACA,OAAOe,IAAI,KAAK;QAChB,kBACE,AAAuB,YAAvB,OAAOA,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCsB;IACR;AACF;AAEO,eAAeC,gBAAgB5C,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAE4C,kBAAkB,EAAE,GAAG7C;IACxC,MAAM,EAAEI,gBAAgB,EAAE,GAAGH;IAE7B,MAAMW,eAAekC,4BAA4BhC;IACjD,MAAMiC,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoB1D,wBAAwBuD;IAC9C;IACA,MAAMlD,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM2C;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAM3B,SAAS,MAAM1B,mBAAmB;YACtC,QAAQqD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACAlD,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAM+B,SAAS,MAAMC,SACnBvD,MACA2B,aAAa,YAAY;IAG3B,IAAI6B;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAapB,gBACjBmB,aACAnD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM;QAErBZ,aAAa,0BAA0BgE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9D5D,aAAa,wBAAwBiE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAS5B,MAAM,OAAO,CAAC4B,OAC/B,GAAG,CAAC,CAACA,OACGvB,gBAAgBuB,MAAMvD,QAAQ,IAAI,CAAC,KAAK,EAAEA,QAAQ,IAAI,CAAC,MAAM;QAExEZ,aAAa,qBAAqBkE;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DlE,aAAa,iBAAiBoE;QAG9BN,cAAcQ,iBAAiBF,YAAYxD,QAAQ,IAAI;QACvDZ,aAAa,2BAA2B8D;IAC1C;IAEA,IAAIS,cAAcxD;IAClB,IAAI+C,aACFS,cAAc,MAAMC,WAClBzD,kBACA+C,aACAW,qBAAqBC;IAIzB,OAAO;QACL,MAAMZ;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAazB,KAAK,SAAS,CAACyB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAee,qBAGpBhE,OAKD;QA0CKiE;IAzCJ,MAAM,EAAEC,SAAS,EAAEjE,OAAO,EAAEkE,aAAa,EAAE1E,gBAAgB,EAAE,GAAGO;IAChE,MAAMY,eAAewD;IAErB,MAAM,EAAEhE,gBAAgB,EAAE,GAAGH;IAC7B,MAAM,EAAEI,WAAW,EAAEC,WAAW,EAAE,GAAG,MAAME,iBAAiBP,SAAS;QACnE,oBAAoB;QACpB,sBAAsB;QACtB,aAAa;QACb,aAAakE,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,WAAW;IACzC;IAEA,MAAME,wBAAwB,MAAMC,uBAClCjE,aACA6D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKnE;YACL,QAAQ;QACV;IACF;IAGFmE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM1E,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS2D;QACX;KACD;IAED,IAAI,QAAAN,CAAAA,yBAAAA,QAAQ,aAAa,AAAD,IAApBA,KAAAA,IAAAA,uBAAuB,aAAa,EACtCtE,KAAK,IAAI,CAAC;QACR,MAAM;QACN,SAAS;IACX;IAGF,IAAIF,kBAAkB;QACpB,MAAMyB,SAAS,MAAM1B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAM+B,SAAS,MAAMC,SACnBvD,MACA2B,aAAa,YAAY;IAE3B,OAAO;QACL,aAAa2B,OAAO,OAAO;QAC3B3C;QACA,OAAO2C,OAAO,KAAK;IACrB;AACF;AAEO,eAAeuB,SAEpBxE,OAAoE;IACpE,MAAM,EAAEyE,SAAS,EAAExE,OAAO,EAAE,GAAGD;IAE/BS,OAAOgE,WAAW;IAElB,MAAM,EAAErE,gBAAgB,EAAE,GAAGH;IAE7B,MAAMW,eAAe8D,qBAAqB;QACxC,UAAUZ,qBAAqBa;IACjC;IAEA,MAAMC,gBAAgBtF,wBAAwBmF;IAE9C,MAAM9E,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM,CAAC;;;AAGjB,EAAEwE,cAAc;;EAEd,CAAC;gBACK;aACD;QACH;KACD;IAED,IAAI,AAAqB,YAArB,OAAOH,WAAwB;QACjC,MAAMvD,SAAS,MAAM1B,mBAAmB;YACtC,QAAQiF,UAAU,MAAM;YACxB,yBAAyBA,UAAU,uBAAuB;QAC5D;QACA9E,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAM,EAAE,SAAS2D,YAAY,EAAEC,KAAK,EAAE,GAAG,MAAM5B,SAC7CvD,MACA2B,aAAa,MAAM;IAErB,OAAO;QACL,SAASuD;QACTC;IACF;AACF"}

package/dist/es/ai-model/llm-planning.mjs

@@ -0,0 +1,85 @@
+import { vlLocateMode } from "@midscene/shared/env";
+import { paddingToMatchBlockByBase64 } from "@midscene/shared/img";
+import { assert } from "@midscene/shared/utils";
+import { AIActionType, buildYamlFlowFromPlans, callAiFn, fillBboxParam, markupImageForLLM, warnGPT4oSizeLimit } from "./common.mjs";
+import { automationUserPrompt, generateTaskBackgroundContext, systemPromptToTaskPlanning } from "./prompt/llm-planning.mjs";
+import { describeUserPage } from "./prompt/util.mjs";
+async function plan(userInstruction, opts) {
+    var _planFromAI_action;
+    const { callAI, context } = opts || {};
+    const { screenshotBase64, size } = context;
+    const { description: pageDescription, elementById } = await describeUserPage(context);
+    const systemPrompt = await systemPromptToTaskPlanning({
+        actionSpace: opts.actionSpace,
+        vlMode: vlLocateMode()
+    });
+    const taskBackgroundContextText = generateTaskBackgroundContext(userInstruction, opts.log, opts.actionContext);
+    const userInstructionPrompt = await automationUserPrompt(vlLocateMode()).format({
+        pageDescription,
+        taskBackgroundContext: taskBackgroundContextText
+    });
+    let imagePayload = screenshotBase64;
+    if ('qwen-vl' === vlLocateMode()) imagePayload = await paddingToMatchBlockByBase64(imagePayload);
+    else if (!vlLocateMode()) imagePayload = await markupImageForLLM(screenshotBase64, context.tree, context.size);
+    warnGPT4oSizeLimit(size);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: userInstructionPrompt
+                }
+            ]
+        }
+    ];
+    const call = callAI || callAiFn;
+    const { content, usage } = await call(msgs, AIActionType.PLAN);
+    const rawResponse = JSON.stringify(content, void 0, 2);
+    const planFromAI = content;
+    const actions = ((null == (_planFromAI_action = planFromAI.action) ? void 0 : _planFromAI_action.type) ? [
+        planFromAI.action
+    ] : planFromAI.actions) || [];
+    const returnValue = {
+        ...planFromAI,
+        actions,
+        rawResponse,
+        usage,
+        yamlFlow: buildYamlFlowFromPlans(actions, planFromAI.sleep)
+    };
+    assert(planFromAI, "can't get plans from AI");
+    if (vlLocateMode()) {
+        actions.forEach((action)=>{
+            if (action.locate) try {
+                action.locate = fillBboxParam(action.locate, size.width, size.height);
+            } catch (e) {
+                throw new Error(`Failed to fill locate param: ${planFromAI.error} (${e instanceof Error ? e.message : 'unknown error'})`, {
+                    cause: e
+                });
+            }
+        });
+        assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);
+    } else actions.forEach((action)=>{
+        var _action_locate;
+        if (null == (_action_locate = action.locate) ? void 0 : _action_locate.id) {
+            const element = elementById(action.locate.id);
+            if (element) action.locate.id = element.id;
+        }
+    });
+    if (0 === actions.length && returnValue.more_actions_needed_by_instruction && !returnValue.sleep) console.warn('No actions planned for the prompt, but model said more actions are needed:', userInstruction);
+    return returnValue;
+}
+export { plan };
+
+//# sourceMappingURL=llm-planning.mjs.map

package/dist/es/ai-model/llm-planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/llm-planning.mjs","sources":["webpack://@midscene/core/./src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n  DeviceAction,\n  PageType,\n  PlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport { vlLocateMode } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { assert } from '@midscene/shared/utils';\nimport {\n  AIActionType,\n  type AIArgs,\n  buildYamlFlowFromPlans,\n  callAiFn,\n  fillBboxParam,\n  markupImageForLLM,\n  warnGPT4oSizeLimit,\n} from './common';\nimport {\n  automationUserPrompt,\n  generateTaskBackgroundContext,\n  systemPromptToTaskPlanning,\n} from './prompt/llm-planning';\nimport { describeUserPage } from './prompt/util';\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    pageType: PageType;\n    actionSpace: DeviceAction[];\n    callAI?: typeof callAiFn<PlanningAIResponse>;\n    log?: string;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { callAI, context } = opts || {};\n  const { screenshotBase64, size } = context;\n  const { description: pageDescription, elementById } =\n    await describeUserPage(context);\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode: vlLocateMode(),\n  });\n  const taskBackgroundContextText = generateTaskBackgroundContext(\n    userInstruction,\n    opts.log,\n    opts.actionContext,\n  );\n  const userInstructionPrompt = await automationUserPrompt(\n    vlLocateMode(),\n  ).format({\n    pageDescription,\n    taskBackgroundContext: taskBackgroundContextText,\n  });\n\n  let imagePayload = screenshotBase64;\n  if (vlLocateMode() === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlLocateMode()) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  warnGPT4oSizeLimit(size);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  const call = callAI || callAiFn;\n  const { content, usage } = await call(msgs, AIActionType.PLAN);\n  const rawResponse = JSON.stringify(content, undefined, 2);\n  const planFromAI = content;\n\n  const actions =\n    (planFromAI.action?.type ? [planFromAI.action] : planFromAI.actions) || [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(actions, planFromAI.sleep),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  if (vlLocateMode()) {\n    actions.forEach((action) => {\n      if (action.locate) {\n        try {\n          action.locate = fillBboxParam(action.locate, size.width, size.height);\n        } catch (e) {\n          throw new Error(\n            `Failed to fill locate param: ${planFromAI.error} (${\n              e instanceof Error ? e.message : 'unknown error'\n            })`,\n            {\n              cause: e,\n            },\n          );\n        }\n      }\n    });\n    // in Qwen-VL, error means error. In GPT-4o, error may mean more actions are needed.\n    assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);\n  } else {\n    actions.forEach((action) => {\n      if (action.locate?.id) {\n        // The model may return indexId, need to perform a query correction to avoid exceptions\n        const element = elementById(action.locate.id);\n        if (element) {\n          action.locate.id = element.id;\n        }\n      }\n    });\n  }\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  return returnValue;\n}\n"],"names":["plan","userInstruction","opts","_planFromAI_action","callAI","context","screenshotBase64","size","pageDescription","elementById","describeUserPage","systemPrompt","systemPromptToTaskPlanning","vlLocateMode","taskBackgroundContextText","generateTaskBackgroundContext","userInstructionPrompt","automationUserPrompt","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","warnGPT4oSizeLimit","msgs","call","callAiFn","content","usage","AIActionType","rawResponse","JSON","undefined","planFromAI","actions","returnValue","buildYamlFlowFromPlans","assert","action","fillBboxParam","e","Error","_action_locate","element","console"],"mappings":";;;;;;AAyBO,eAAeA,KACpBC,eAAuB,EACvBC,IAOC;QA8DEC;IA5DH,MAAM,EAAEC,MAAM,EAAEC,OAAO,EAAE,GAAGH,QAAQ,CAAC;IACrC,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGF;IACnC,MAAM,EAAE,aAAaG,eAAe,EAAEC,WAAW,EAAE,GACjD,MAAMC,iBAAiBL;IAEzB,MAAMM,eAAe,MAAMC,2BAA2B;QACpD,aAAaV,KAAK,WAAW;QAC7B,QAAQW;IACV;IACA,MAAMC,4BAA4BC,8BAChCd,iBACAC,KAAK,GAAG,EACRA,KAAK,aAAa;IAEpB,MAAMc,wBAAwB,MAAMC,qBAClCJ,gBACA,MAAM,CAAC;QACPL;QACA,uBAAuBM;IACzB;IAEA,IAAII,eAAeZ;IACnB,IAAIO,AAAmB,cAAnBA,gBACFK,eAAe,MAAMC,4BAA4BD;SAC5C,IAAI,CAACL,gBACVK,eAAe,MAAME,kBACnBd,kBACAD,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhBgB,mBAAmBd;IAEnB,MAAMe,OAAe;QACnB;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKO;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMF;gBACR;aACD;QACH;KACD;IAED,MAAMO,OAAOnB,UAAUoB;IACvB,MAAM,EAAEC,OAAO,EAAEC,KAAK,EAAE,GAAG,MAAMH,KAAKD,MAAMK,aAAa,IAAI;IAC7D,MAAMC,cAAcC,KAAK,SAAS,CAACJ,SAASK,QAAW;IACvD,MAAMC,aAAaN;IAEnB,MAAMO,UACH7B,AAAAA,CAAAA,SAAAA,CAAAA,qBAAAA,WAAW,MAAM,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,IAAI;QAAC4B,WAAW,MAAM;KAAC,GAAGA,WAAW,OAAM,KAAM,EAAE;IAC5E,MAAME,cAAkC;QACtC,GAAGF,UAAU;QACbC;QACAJ;QACAF;QACA,UAAUQ,uBAAuBF,SAASD,WAAW,KAAK;IAC5D;IAEAI,OAAOJ,YAAY;IAEnB,IAAIlB,gBAAgB;QAClBmB,QAAQ,OAAO,CAAC,CAACI;YACf,IAAIA,OAAO,MAAM,EACf,IAAI;gBACFA,OAAO,MAAM,GAAGC,cAAcD,OAAO,MAAM,EAAE7B,KAAK,KAAK,EAAEA,KAAK,MAAM;YACtE,EAAE,OAAO+B,GAAG;gBACV,MAAM,IAAIC,MACR,CAAC,6BAA6B,EAAER,WAAW,KAAK,CAAC,EAAE,EACjDO,aAAaC,QAAQD,EAAE,OAAO,GAAG,gBAClC,CAAC,CAAC,EACH;oBACE,OAAOA;gBACT;YAEJ;QAEJ;QAEAH,OAAO,CAACJ,WAAW,KAAK,EAAE,CAAC,wBAAwB,EAAEA,WAAW,KAAK,EAAE;IACzE,OACEC,QAAQ,OAAO,CAAC,CAACI;YACXI;QAAJ,IAAI,QAAAA,CAAAA,iBAAAA,OAAO,MAAM,AAAD,IAAZA,KAAAA,IAAAA,eAAe,EAAE,EAAE;YAErB,MAAMC,UAAUhC,YAAY2B,OAAO,MAAM,CAAC,EAAE;YAC5C,IAAIK,SACFL,OAAO,MAAM,CAAC,EAAE,GAAGK,QAAQ,EAAE;QAEjC;IACF;IAGF,IACET,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBS,QAAQ,IAAI,CACV,8EACAzC;IAIJ,OAAOgC;AACT"}

package/dist/es/ai-model/prompt/assertion.mjs

@@ -0,0 +1,55 @@
+import { getPreferredLanguage } from "@midscene/shared/env";
+const defaultAssertionPrompt = 'You are a senior testing engineer. User will give an assertion and a screenshot of a page. By carefully viewing the screenshot, please tell whether the assertion is truthy.';
+const defaultAssertionResponseJsonFormat = `Return in the following JSON format:
+{
+  pass: boolean, // whether the assertion is truthy
+  thought: string | null, // string, if the result is falsy, give the reason why it is falsy. Otherwise, put null.
+}`;
+const getUiTarsAssertionResponseJsonFormat = ()=>`## Output Json String Format
+\`\`\`
+"{
+  "pass": <<is a boolean value from the enum [true, false], true means the assertion is truthy>>, 
+  "thought": "<<is a string, give the reason why the assertion is falsy or truthy. Otherwise.>>"
+}"
+\`\`\`
+
+## Rules **MUST** follow
+- Make sure to return **only** the JSON, with **no additional** text or explanations.
+- Use ${getPreferredLanguage()} in \`thought\` part.
+- You **MUST** strictly follow up the **Output Json String Format**.`;
+function systemPromptToAssert(model) {
+    return `${defaultAssertionPrompt}
+
+${model.isUITars ? getUiTarsAssertionResponseJsonFormat() : defaultAssertionResponseJsonFormat}`;
+}
+const assertSchema = {
+    type: 'json_schema',
+    json_schema: {
+        name: 'assert',
+        strict: true,
+        schema: {
+            type: 'object',
+            properties: {
+                pass: {
+                    type: 'boolean',
+                    description: 'Whether the assertion passed or failed'
+                },
+                thought: {
+                    type: [
+                        'string',
+                        'null'
+                    ],
+                    description: 'The thought process behind the assertion'
+                }
+            },
+            required: [
+                'pass',
+                'thought'
+            ],
+            additionalProperties: false
+        }
+    }
+};
+export { assertSchema, systemPromptToAssert };
+
+//# sourceMappingURL=assertion.mjs.map

package/dist/es/ai-model/prompt/assertion.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/assertion.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/assertion.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\n\nconst defaultAssertionPrompt =\n  'You are a senior testing engineer. User will give an assertion and a screenshot of a page. By carefully viewing the screenshot, please tell whether the assertion is truthy.';\n\nconst defaultAssertionResponseJsonFormat = `Return in the following JSON format:\n{\n  pass: boolean, // whether the assertion is truthy\n  thought: string | null, // string, if the result is falsy, give the reason why it is falsy. Otherwise, put null.\n}`;\n\nconst getUiTarsAssertionResponseJsonFormat = () => `## Output Json String Format\n\\`\\`\\`\n\"{\n  \"pass\": <<is a boolean value from the enum [true, false], true means the assertion is truthy>>, \n  \"thought\": \"<<is a string, give the reason why the assertion is falsy or truthy. Otherwise.>>\"\n}\"\n\\`\\`\\`\n\n## Rules **MUST** follow\n- Make sure to return **only** the JSON, with **no additional** text or explanations.\n- Use ${getPreferredLanguage()} in \\`thought\\` part.\n- You **MUST** strictly follow up the **Output Json String Format**.`;\n\nexport function systemPromptToAssert(model: { isUITars: boolean }) {\n  return `${defaultAssertionPrompt}\n\n${model.isUITars ? getUiTarsAssertionResponseJsonFormat() : defaultAssertionResponseJsonFormat}`;\n}\n\nexport const assertSchema: ResponseFormatJSONSchema = {\n  type: 'json_schema',\n  json_schema: {\n    name: 'assert',\n    strict: true,\n    schema: {\n      type: 'object',\n      properties: {\n        pass: {\n          type: 'boolean',\n          description: 'Whether the assertion passed or failed',\n        },\n        thought: {\n          type: ['string', 'null'],\n          description: 'The thought process behind the assertion',\n        },\n      },\n      required: ['pass', 'thought'],\n      additionalProperties: false,\n    },\n  },\n};\n"],"names":["defaultAssertionPrompt","defaultAssertionResponseJsonFormat","getUiTarsAssertionResponseJsonFormat","getPreferredLanguage","systemPromptToAssert","model","assertSchema"],"mappings":";AAGA,MAAMA,yBACJ;AAEF,MAAMC,qCAAqC,CAAC;;;;CAI3C,CAAC;AAEF,MAAMC,uCAAuC,IAAM,CAAC;;;;;;;;;;MAU9C,EAAEC,uBAAuB;oEACqC,CAAC;AAE9D,SAASC,qBAAqBC,KAA4B;IAC/D,OAAO,GAAGL,uBAAuB;;AAEnC,EAAEK,MAAM,QAAQ,GAAGH,yCAAyCD,oCAAoC;AAChG;AAEO,MAAMK,eAAyC;IACpD,MAAM;IACN,aAAa;QACX,MAAM;QACN,QAAQ;QACR,QAAQ;YACN,MAAM;YACN,YAAY;gBACV,MAAM;oBACJ,MAAM;oBACN,aAAa;gBACf;gBACA,SAAS;oBACP,MAAM;wBAAC;wBAAU;qBAAO;oBACxB,aAAa;gBACf;YACF;YACA,UAAU;gBAAC;gBAAQ;aAAU;YAC7B,sBAAsB;QACxB;IACF;AACF"}

package/dist/es/ai-model/prompt/common.mjs

@@ -0,0 +1,7 @@
+function bboxDescription(vlMode) {
+    if ('gemini' === vlMode) return '2d bounding box as [ymin, xmin, ymax, xmax]';
+    return '2d bounding box as [xmin, ymin, xmax, ymax]';
+}
+export { bboxDescription };
+
+//# sourceMappingURL=common.mjs.map

package/dist/es/ai-model/prompt/common.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/common.ts"],"sourcesContent":["import type { vlLocateMode } from '@midscene/shared/env';\nexport function bboxDescription(vlMode: ReturnType<typeof vlLocateMode>) {\n  if (vlMode === 'gemini') {\n    return '2d bounding box as [ymin, xmin, ymax, xmax]';\n  }\n  return '2d bounding box as [xmin, ymin, xmax, ymax]';\n}\n"],"names":["bboxDescription","vlMode"],"mappings":"AACO,SAASA,gBAAgBC,MAAuC;IACrE,IAAIA,AAAW,aAAXA,QACF,OAAO;IAET,OAAO;AACT"}

package/dist/es/ai-model/prompt/describe.mjs

@@ -0,0 +1,44 @@
+import { getPreferredLanguage } from "@midscene/shared/env";
+const elementDescriberInstruction = ()=>`
+Describe the element in the red rectangle for precise identification. Use ${getPreferredLanguage()}.
+
+CRITICAL REQUIREMENTS:
+1. UNIQUENESS: The description must uniquely identify this element on the current page
+2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts
+3. PRECISION: Be specific enough to distinguish from similar elements
+
+DESCRIPTION STRUCTURE:
+1. Element type (button, input, link, div, etc.)
+2. Primary identifier (in order of preference):
+   - Unique text content: "with text 'Login'"
+   - Unique attribute: "with aria-label 'Search'"
+   - Unique class/ID: "with class 'primary-button'"
+   - Unique position: "in header navigation"
+3. Secondary identifiers (if needed for uniqueness):
+   - Visual features: "blue background", "with icon"
+   - Relative position: "below search bar", "in sidebar"
+   - Parent context: "in login form", "in main menu"
+
+GUIDELINES:
+- Keep description under 25 words
+- Prioritize semantic identifiers over visual ones
+- Use consistent terminology across similar elements
+- Avoid page-specific or temporary content
+- Don't mention the red rectangle or selection box
+- Focus on stable, reusable characteristics
+
+EXAMPLES:
+- "Login button with text 'Sign In'"
+- "Search input with placeholder 'Enter keywords'"
+- "Navigation link with text 'Home' in header"
+- "Submit button in contact form"
+- "Menu icon with aria-label 'Open menu'"
+
+Return JSON:
+{
+  "description": "unique element identifier",
+  "error"?: "error message if any"
+}`;
+export { elementDescriberInstruction };
+
+//# sourceMappingURL=describe.mjs.map

package/dist/es/ai-model/prompt/describe.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/describe.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/describe.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport const elementDescriberInstruction = () => {\n  return `\nDescribe the element in the red rectangle for precise identification. Use ${getPreferredLanguage()}.\n\nCRITICAL REQUIREMENTS:\n1. UNIQUENESS: The description must uniquely identify this element on the current page\n2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts\n3. PRECISION: Be specific enough to distinguish from similar elements\n\nDESCRIPTION STRUCTURE:\n1. Element type (button, input, link, div, etc.)\n2. Primary identifier (in order of preference):\n   - Unique text content: \"with text 'Login'\"\n   - Unique attribute: \"with aria-label 'Search'\"\n   - Unique class/ID: \"with class 'primary-button'\"\n   - Unique position: \"in header navigation\"\n3. Secondary identifiers (if needed for uniqueness):\n   - Visual features: \"blue background\", \"with icon\"\n   - Relative position: \"below search bar\", \"in sidebar\"\n   - Parent context: \"in login form\", \"in main menu\"\n\nGUIDELINES:\n- Keep description under 25 words\n- Prioritize semantic identifiers over visual ones\n- Use consistent terminology across similar elements\n- Avoid page-specific or temporary content\n- Don't mention the red rectangle or selection box\n- Focus on stable, reusable characteristics\n\nEXAMPLES:\n- \"Login button with text 'Sign In'\"\n- \"Search input with placeholder 'Enter keywords'\"\n- \"Navigation link with text 'Home' in header\"\n- \"Submit button in contact form\"\n- \"Menu icon with aria-label 'Open menu'\"\n\nReturn JSON:\n{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n};\n"],"names":["elementDescriberInstruction","getPreferredLanguage"],"mappings":";AAEO,MAAMA,8BAA8B,IAClC,CAAC;0EACgE,EAAEC,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsClG,CAAC"}