@midscene/core 1.9.8 → 1.10.0

package/dist/lib/ai-model/workflows/inspect/planning-action-locate.js

@@ -0,0 +1,129 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    resolvePlanningTapLocator: ()=>resolvePlanningTapLocator
+});
+const logger_namespaceObject = require("@midscene/shared/logger");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const external_zod_namespaceObject = require("zod");
+const external_common_js_namespaceObject = require("../../../common.js");
+const external_screenshot_item_js_namespaceObject = require("../../../screenshot-item.js");
+const external_conversation_history_js_namespaceObject = require("../../conversation-history.js");
+const index_js_namespaceObject = require("../../service-caller/index.js");
+const custom_planning_js_namespaceObject = require("../planning/custom-planning.js");
+const debugInspect = (0, logger_namespaceObject.getDebug)('ai:inspect');
+const planningActionLocatorActionSpace = [
+    {
+        name: 'Tap',
+        description: 'Tap the element',
+        paramSchema: external_zod_namespaceObject.z.object({
+            locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)()
+        }),
+        call: async ()=>void 0
+    }
+];
+async function buildPlanningTapLocatorPlanOptions(locateRequest) {
+    const { options, locateImage } = locateRequest;
+    const { context } = options;
+    return {
+        ...options,
+        context: {
+            ...context,
+            screenshot: external_screenshot_item_js_namespaceObject.ScreenshotItem.create(locateImage.imageBase64, context.screenshot.capturedAt),
+            shotSize: {
+                width: locateImage.width,
+                height: locateImage.height
+            }
+        },
+        actionSpace: planningActionLocatorActionSpace,
+        conversationHistory: new external_conversation_history_js_namespaceObject.ConversationHistory(),
+        includeLocateInPlanning: true,
+        referenceImageMessages: locateRequest.referenceImageMessages
+    };
+}
+function resolvePlanningTapLocator(definition, planner) {
+    const locatorPlanner = {
+        ...planner,
+        messages: {
+            ...planner.messages,
+            buildSystemPrompt: definition.buildSystemPrompt,
+            buildUserInstruction: (instruction)=>`Tap: ${instruction}`
+        }
+    };
+    return async (elementDescription, options, locateRequest)=>{
+        (0, utils_namespaceObject.assert)(elementDescription, "cannot find the target element description");
+        let errors = [];
+        let reasoningContent = '';
+        let rawResponse = '';
+        let rawChoiceMessage;
+        let usage;
+        try {
+            const locatePlanOptions = await buildPlanningTapLocatorPlanOptions(locateRequest);
+            const planningResponse = await (0, custom_planning_js_namespaceObject.runCustomPlanning)(elementDescription, locatePlanOptions, locatorPlanner);
+            rawResponse = planningResponse.rawResponse ?? '';
+            rawChoiceMessage = planningResponse.rawChoiceMessage;
+            usage = planningResponse.usage;
+            reasoningContent = planningResponse.log;
+            debugInspect('planning-tap-locator rawResponse:', rawResponse);
+            const locatedPixelBbox = definition.getLocatedPixelBbox(planningResponse.actions ?? []);
+            if (!locatedPixelBbox) throw new Error('No locatedPixelBbox found in planner response');
+            return {
+                locatedPixelBbox,
+                rawResponse,
+                rawChoiceMessage,
+                usage,
+                reasoningContent
+            };
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            if (error instanceof index_js_namespaceObject.AIResponseParseError) {
+                rawResponse = error.rawResponse;
+                rawChoiceMessage = error.rawChoiceMessage;
+                usage = error.usage;
+            }
+            errors = [
+                errorMessage || 'Failed to parse planning tap locator response'
+            ];
+            debugInspect('planning-tap-locator parse error:', errors[0]);
+        }
+        return {
+            rawResponse,
+            rawChoiceMessage,
+            usage,
+            reasoningContent,
+            errors
+        };
+    };
+}
+exports.resolvePlanningTapLocator = __webpack_exports__.resolvePlanningTapLocator;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "resolvePlanningTapLocator"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=planning-action-locate.js.map

package/dist/lib/ai-model/workflows/inspect/planning-action-locate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/inspect/planning-action-locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/inspect/planning-action-locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/device';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { z } from 'zod';\nimport { type TUserPrompt, getMidsceneLocationSchema } from '../../../common';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport { ConversationHistory } from '../../conversation-history';\nimport type { ResolvedCustomPlanningDefinition } from '../../model-adapter/custom-planning-types';\nimport type { PlanningTapLocatorDefinition } from '../../model-adapter/types';\nimport { AIResponseParseError } from '../../service-caller/index';\nimport { runCustomPlanning } from '../planning/custom-planning';\nimport type { PlanOptions } from '../planning/types';\nimport type {\n  LocateFn,\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequestContext,\n} from './types';\n\nconst debugInspect = getDebug('ai:inspect');\n\nconst planningActionLocatorActionSpace: DeviceAction[] = [\n  {\n    name: 'Tap',\n    description: 'Tap the element',\n    paramSchema: z.object({\n      locate: getMidsceneLocationSchema(),\n    }),\n    call: async () => undefined,\n  },\n];\n\nasync function buildPlanningTapLocatorPlanOptions(\n  locateRequest: LocateRequestContext,\n): Promise<PlanOptions> {\n  const { options, locateImage } = locateRequest;\n  const { context } = options;\n\n  return {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        locateImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: {\n        width: locateImage.width,\n        height: locateImage.height,\n      },\n    },\n    actionSpace: planningActionLocatorActionSpace,\n    conversationHistory: new ConversationHistory(),\n    includeLocateInPlanning: true,\n    referenceImageMessages: locateRequest.referenceImageMessages,\n  };\n}\n\nexport function resolvePlanningTapLocator<TParsed>(\n  definition: PlanningTapLocatorDefinition,\n  planner: ResolvedCustomPlanningDefinition<TParsed>,\n): LocateFn {\n  const locatorPlanner: ResolvedCustomPlanningDefinition<TParsed> = {\n    ...planner,\n    messages: {\n      ...planner.messages,\n      buildSystemPrompt: definition.buildSystemPrompt,\n      buildUserInstruction: (instruction) => `Tap: ${instruction}`,\n    },\n  };\n\n  return async (\n    elementDescription: TUserPrompt,\n    options: LocateOptions,\n    locateRequest: LocateRequestContext,\n  ): Promise<LocateModelResponse> => {\n    assert(elementDescription, 'cannot find the target element description');\n\n    let errors: string[] = [];\n    let reasoningContent = '';\n    let rawResponse = '';\n    let rawChoiceMessage: unknown;\n    let usage: LocateModelResponse['usage'];\n\n    try {\n      const locatePlanOptions =\n        await buildPlanningTapLocatorPlanOptions(locateRequest);\n      const planningResponse = await runCustomPlanning(\n        elementDescription,\n        locatePlanOptions,\n        locatorPlanner,\n      );\n\n      rawResponse = planningResponse.rawResponse ?? '';\n      rawChoiceMessage = planningResponse.rawChoiceMessage;\n      usage = planningResponse.usage;\n      reasoningContent = planningResponse.log;\n\n      debugInspect('planning-tap-locator rawResponse:', rawResponse);\n\n      const locatedPixelBbox = definition.getLocatedPixelBbox(\n        planningResponse.actions ?? [],\n      );\n\n      if (!locatedPixelBbox) {\n        throw new Error('No locatedPixelBbox found in planner response');\n      }\n\n      return {\n        locatedPixelBbox,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoningContent,\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      if (error instanceof AIResponseParseError) {\n        rawResponse = error.rawResponse;\n        rawChoiceMessage = error.rawChoiceMessage;\n        usage = error.usage;\n      }\n      errors = [\n        errorMessage || 'Failed to parse planning tap locator response',\n      ];\n      debugInspect('planning-tap-locator parse error:', errors[0]);\n    }\n\n    return {\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoningContent,\n      errors,\n    };\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInspect","getDebug","planningActionLocatorActionSpace","z","getMidsceneLocationSchema","undefined","buildPlanningTapLocatorPlanOptions","locateRequest","options","locateImage","context","ScreenshotItem","ConversationHistory","resolvePlanningTapLocator","planner","locatorPlanner","instruction","elementDescription","assert","errors","reasoningContent","rawResponse","rawChoiceMessage","usage","locatePlanOptions","planningResponse","runCustomPlanning","locatedPixelBbox","Error","error","errorMessage","String","AIResponseParseError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACaA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAE9B,MAAMC,mCAAmD;IACvD;QACE,MAAM;QACN,aAAa;QACb,aAAaC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;YACpB,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA;QACV;QACA,MAAM,UAAYC;IACpB;CACD;AAED,eAAeC,mCACbC,aAAmC;IAEnC,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGF;IACjC,MAAM,EAAEG,OAAO,EAAE,GAAGF;IAEpB,OAAO;QACL,GAAGA,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYC,4CAAAA,cAAAA,CAAAA,MAAqB,CAC/BF,YAAY,WAAW,EACvBC,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAU;gBACR,OAAOD,YAAY,KAAK;gBACxB,QAAQA,YAAY,MAAM;YAC5B;QACF;QACA,aAAaP;QACb,qBAAqB,IAAIU,iDAAAA,mBAAmBA;QAC5C,yBAAyB;QACzB,wBAAwBL,cAAc,sBAAsB;IAC9D;AACF;AAEO,SAASM,0BACdnB,UAAwC,EACxCoB,OAAkD;IAElD,MAAMC,iBAA4D;QAChE,GAAGD,OAAO;QACV,UAAU;YACR,GAAGA,QAAQ,QAAQ;YACnB,mBAAmBpB,WAAW,iBAAiB;YAC/C,sBAAsB,CAACsB,cAAgB,CAAC,KAAK,EAAEA,aAAa;QAC9D;IACF;IAEA,OAAO,OACLC,oBACAT,SACAD;QAEAW,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,oBAAoB;QAE3B,IAAIE,SAAmB,EAAE;QACzB,IAAIC,mBAAmB;QACvB,IAAIC,cAAc;QAClB,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMC,oBACJ,MAAMlB,mCAAmCC;YAC3C,MAAMkB,mBAAmB,MAAMC,AAAAA,IAAAA,mCAAAA,iBAAAA,AAAAA,EAC7BT,oBACAO,mBACAT;YAGFM,cAAcI,iBAAiB,WAAW,IAAI;YAC9CH,mBAAmBG,iBAAiB,gBAAgB;YACpDF,QAAQE,iBAAiB,KAAK;YAC9BL,mBAAmBK,iBAAiB,GAAG;YAEvCzB,aAAa,qCAAqCqB;YAElD,MAAMM,mBAAmBjC,WAAW,mBAAmB,CACrD+B,iBAAiB,OAAO,IAAI,EAAE;YAGhC,IAAI,CAACE,kBACH,MAAM,IAAIC,MAAM;YAGlB,OAAO;gBACLD;gBACAN;gBACAC;gBACAC;gBACAH;YACF;QACF,EAAE,OAAOS,OAAO;YACd,MAAMC,eACJD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;YAClD,IAAIA,iBAAiBG,yBAAAA,oBAAoBA,EAAE;gBACzCX,cAAcQ,MAAM,WAAW;gBAC/BP,mBAAmBO,MAAM,gBAAgB;gBACzCN,QAAQM,MAAM,KAAK;YACrB;YACAV,SAAS;gBACPW,gBAAgB;aACjB;YACD9B,aAAa,qCAAqCmB,MAAM,CAAC,EAAE;QAC7D;QAEA,OAAO;YACLE;YACAC;YACAC;YACAH;YACAD;QACF;IACF;AACF"}

package/dist/lib/ai-model/workflows/planning/custom-planning.js

@@ -0,0 +1,163 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    runCustomPlanning: ()=>runCustomPlanning,
+    buildCustomPlanningMessages: ()=>buildCustomPlanningMessages
+});
+const external_common_js_namespaceObject = require("../../../common.js");
+const external_screenshot_item_js_namespaceObject = require("../../../screenshot-item.js");
+const index_js_namespaceObject = require("../../service-caller/index.js");
+const external_image_preprocess_js_namespaceObject = require("../image-preprocess.js");
+const external_locate_normalization_js_namespaceObject = require("./locate-normalization.js");
+function appendHighPriorityKnowledge(systemPrompt, actionContext) {
+    return systemPrompt + (actionContext ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\n` : '');
+}
+function buildCustomPlanningMessages(input, config) {
+    const { options, userInstructionText } = input;
+    const { conversationHistory, context, actionContext } = options;
+    const systemPrompt = appendHighPriorityKnowledge(config.buildSystemPrompt(), actionContext);
+    const userInstruction = config.buildUserInstruction ? config.buildUserInstruction(userInstructionText) : userInstructionText;
+    if (conversationHistory.pendingFeedbackMessage) {
+        conversationHistory.append({
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`
+                }
+            ]
+        });
+        conversationHistory.resetPendingFeedbackMessageIfExists();
+    }
+    conversationHistory.append({
+        role: 'user',
+        content: [
+            {
+                type: 'image_url',
+                image_url: {
+                    url: context.screenshot.base64
+                }
+            }
+        ]
+    });
+    if ('system-message' === config.systemPromptPlacement) return [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: userInstruction
+                }
+            ]
+        },
+        ...options.referenceImageMessages ?? [],
+        ...conversationHistory.snapshot(config.historyImageLimit)
+    ];
+    return [
+        {
+            role: 'user',
+            content: `${systemPrompt}${userInstruction}`
+        },
+        ...options.referenceImageMessages ?? [],
+        ...conversationHistory.snapshot(config.historyImageLimit)
+    ];
+}
+async function runCustomPlanning(userInstruction, options, config) {
+    const { context } = options;
+    const preparedImage = await (0, external_image_preprocess_js_namespaceObject.prepareModelImage)({
+        imageBase64: context.screenshot.base64,
+        width: context.shotSize.width,
+        height: context.shotSize.height,
+        policy: options.modelRuntime.adapter.imagePreprocess
+    });
+    const preparedOptions = {
+        ...options,
+        context: {
+            ...context,
+            screenshot: external_screenshot_item_js_namespaceObject.ScreenshotItem.create(preparedImage.imageBase64, context.screenshot.capturedAt),
+            shotSize: preparedImage.preparedSize
+        }
+    };
+    const input = {
+        userInstruction,
+        userInstructionText: (0, external_common_js_namespaceObject.userPromptToString)(userInstruction),
+        options: preparedOptions,
+        coordinateSystem: config.coordinateSystem
+    };
+    const messages = buildCustomPlanningMessages(input, config.messages);
+    const { content, usage, rawChoiceMessage } = await (0, index_js_namespaceObject.callAIWithStringResponse)(messages, preparedOptions.modelRuntime, {
+        abortSignal: preparedOptions.abortSignal,
+        requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning
+    });
+    let parsed;
+    let actions;
+    let shouldContinuePlanning;
+    try {
+        parsed = config.parseResponse(content, input);
+        actions = config.transformActions(parsed, input);
+        (0, external_locate_normalization_js_namespaceObject.normalizePlanningActionLocateFields)(actions, {
+            actionSpace: preparedOptions.actionSpace,
+            includeLocateInPlanning: preparedOptions.includeLocateInPlanning,
+            locateResultAdapter: config.coordinateNormalizer,
+            locateResultContext: {
+                preparedSize: preparedImage.preparedSize,
+                contentSize: preparedImage.contentSize
+            }
+        });
+        shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);
+    } catch (parseError) {
+        const errorMessage = `Parse error: ${parseError instanceof Error ? parseError.message : String(parseError)}`;
+        throw new index_js_namespaceObject.AIResponseParseError(errorMessage, JSON.stringify(content, void 0, 2), usage, rawChoiceMessage);
+    }
+    const assistantContent = config.messages.buildAssistantContent?.(parsed, content, input);
+    if (assistantContent) options.conversationHistory.append({
+        role: 'assistant',
+        content: assistantContent
+    });
+    return {
+        actions,
+        log: config.buildResponseLog(parsed, content),
+        usage,
+        shouldContinuePlanning,
+        rawResponse: JSON.stringify(content, void 0, 2),
+        rawChoiceMessage
+    };
+}
+exports.buildCustomPlanningMessages = __webpack_exports__.buildCustomPlanningMessages;
+exports.runCustomPlanning = __webpack_exports__.runCustomPlanning;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "buildCustomPlanningMessages",
+    "runCustomPlanning"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=custom-planning.js.map

package/dist/lib/ai-model/workflows/planning/custom-planning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/planning/custom-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type TUserPrompt, userPromptToString } from '@/common';\nimport type { PlanningAIResponse, PlanningAction } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport { prepareModelImage } from '../image-preprocess';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendHighPriorityKnowledge(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return (\n    systemPrompt +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\\n`\n      : '')\n  );\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ChatCompletionMessageParam[] {\n  const { options, userInstructionText } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendHighPriorityKnowledge(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.append({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: { url: context.screenshot.base64 },\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...(options.referenceImageMessages ?? []),\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...(options.referenceImageMessages ?? []),\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: TUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    userInstructionText: userPromptToString(userInstruction),\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultAdapter: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    });\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  if (assistantContent) {\n    options.conversationHistory.append({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawChoiceMessage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","appendHighPriorityKnowledge","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","userInstructionText","conversationHistory","context","userInstruction","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","userPromptToString","messages","content","usage","rawChoiceMessage","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACWA,SAASI,4BACPC,YAAoB,EACpBC,aAAsB;IAEtB,OACED,eACCC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,4BAA4B,CAAC,GACvE,EAAC;AAET;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGH;IACzC,MAAM,EAAEI,mBAAmB,EAAEC,OAAO,EAAEP,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,4BACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMQ,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACE,uBAC5BA;IAEJ,IAAIC,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBAAE,KAAKC,QAAQ,UAAU,CAAC,MAAM;gBAAC;YAC9C;SACD;IACH;IAEA,IAAIJ,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACIJ,QAAQ,sBAAsB,IAAI,EAAE;WACrCE,oBAAoB,QAAQ,CAACH,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACIJ,QAAQ,sBAAsB,IAAI,EAAE;WACrCE,oBAAoB,QAAQ,CAACH,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeM,kBACpBD,eAA4B,EAC5BJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEI,OAAO,EAAE,GAAGH;IACpB,MAAMM,gBAAgB,MAAMC,AAAAA,IAAAA,6CAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaJ,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQH,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMQ,kBAA+B;QACnC,GAAGR,OAAO;QACV,SAAS;YACP,GAAGG,OAAO;YACV,YAAYM,4CAAAA,cAAAA,CAAAA,MAAqB,CAC/BH,cAAc,WAAW,EACzBH,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUG,cAAc,YAAY;QACtC;IACF;IACA,MAAMR,QAA6B;QACjCM;QACA,qBAAqBM,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBN;QACxC,SAASI;QACT,kBAAkBT,OAAO,gBAAgB;IAC3C;IAEA,MAAMY,WAAWd,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEa,OAAO,EAAEC,KAAK,EAAEC,gBAAgB,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACjDJ,UACAH,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIQ;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASjB,OAAO,aAAa,CAACa,SAASd;QACvCmB,UAAUlB,OAAO,gBAAgB,CAACiB,QAAQlB;QAC1CqB,IAAAA,iDAAAA,mCAAAA,AAAAA,EAAoCF,SAAS;YAC3C,aAAaT,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,qBAAqBT,OAAO,oBAAoB;YAChD,qBAAqB;gBACnB,cAAcO,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;QACF;QACAY,yBAAyBnB,OAAO,sBAAsB,CAACiB,QAAQC;IACjE,EAAE,OAAOG,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,yBAAAA,oBAAoBA,CAC5BH,cACAI,KAAK,SAAS,CAACb,SAASc,QAAW,IACnCb,OACAC;IAEJ;IAEA,MAAMa,mBAAmB5B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DiB,QACAJ,SACAd;IAEF,IAAI6B,kBACF3B,QAAQ,mBAAmB,CAAC,MAAM,CAAC;QACjC,MAAM;QACN,SAAS2B;IACX;IAGF,OAAO;QACLV;QACA,KAAKlB,OAAO,gBAAgB,CAACiB,QAAQJ;QACrCC;QACAK;QACA,aAAaO,KAAK,SAAS,CAACb,SAASc,QAAW;QAChDZ;IACF;AACF"}

package/dist/lib/ai-model/workflows/planning/locate-normalization.js

@@ -0,0 +1,65 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    normalizePlanningActionLocateFields: ()=>normalizePlanningActionLocateFields
+});
+const external_common_js_namespaceObject = require("../../../common.js");
+const logger_namespaceObject = require("@midscene/shared/logger");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const debug = (0, logger_namespaceObject.getDebug)('planning');
+function normalizePlanningActionLocateFields(actions, { actionSpace, includeLocateInPlanning, locateResultAdapter, locateResultContext }) {
+    actions.forEach((action)=>{
+        const actionInActionSpace = actionSpace.find((actionInSpace)=>actionInSpace.name === action.type);
+        if (!actionInActionSpace) return void debug('skip locate normalization for action outside actionSpace', action);
+        debug('actionInActionSpace matched', actionInActionSpace);
+        const locateFields = actionInActionSpace ? (0, external_common_js_namespaceObject.findAllMidsceneLocatorField)(actionInActionSpace.paramSchema) : [];
+        debug('locateFields', locateFields);
+        locateFields.forEach((field)=>{
+            const locateResult = action.param?.[field];
+            if (!locateResult) return;
+            if (!includeLocateInPlanning) {
+                if ('object' == typeof locateResult) action.param[field] = {
+                    prompt: locateResult.prompt
+                };
+                return;
+            }
+            (0, utils_namespaceObject.assert)(locateResultAdapter, 'planning locate normalization requires a locate result adapter');
+            action.param[field] = {
+                ...locateResult,
+                locatedPixelBbox: locateResultAdapter.adaptPlanningParamToPixelBbox(locateResult, locateResultContext)
+            };
+        });
+    });
+}
+exports.normalizePlanningActionLocateFields = __webpack_exports__.normalizePlanningActionLocateFields;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "normalizePlanningActionLocateFields"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=locate-normalization.js.map

package/dist/lib/ai-model/workflows/planning/locate-normalization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/planning/locate-normalization.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/planning/locate-normalization.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  LocateResultAdapter,\n  LocateResultContext,\n} from '../../shared/model-locate-result/types';\n\nconst debug = getDebug('planning');\n\nexport function normalizePlanningActionLocateFields(\n  actions: PlanningAction[],\n  {\n    actionSpace,\n    includeLocateInPlanning,\n    locateResultAdapter,\n    locateResultContext,\n  }: {\n    actionSpace: DeviceAction[];\n    includeLocateInPlanning: boolean;\n    locateResultAdapter?: LocateResultAdapter;\n    locateResultContext: LocateResultContext;\n  },\n): void {\n  actions.forEach((action) => {\n    const actionInActionSpace = actionSpace.find(\n      (actionInSpace) => actionInSpace.name === action.type,\n    );\n    if (!actionInActionSpace) {\n      debug('skip locate normalization for action outside actionSpace', action);\n      return;\n    }\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param?.[field];\n      if (!locateResult) {\n        return;\n      }\n\n      if (!includeLocateInPlanning) {\n        if (typeof locateResult === 'object') {\n          // In prompt-only planning mode, ignore any accidental coordinates from the model.\n          action.param[field] = { prompt: locateResult.prompt };\n        }\n        return;\n      }\n\n      assert(\n        locateResultAdapter,\n        'planning locate normalization requires a locate result adapter',\n      );\n      action.param[field] = {\n        ...locateResult,\n        locatedPixelBbox: locateResultAdapter.adaptPlanningParamToPixelBbox(\n          locateResult,\n          locateResultContext,\n        ),\n      };\n    });\n  });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","normalizePlanningActionLocateFields","actions","actionSpace","includeLocateInPlanning","locateResultAdapter","locateResultContext","action","actionInActionSpace","actionInSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","assert"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACIA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,SAASC,oCACdC,OAAyB,EACzB,EACEC,WAAW,EACXC,uBAAuB,EACvBC,mBAAmB,EACnBC,mBAAmB,EAMpB;IAEDJ,QAAQ,OAAO,CAAC,CAACK;QACf,MAAMC,sBAAsBL,YAAY,IAAI,CAC1C,CAACM,gBAAkBA,cAAc,IAAI,KAAKF,OAAO,IAAI;QAEvD,IAAI,CAACC,qBAAqB,YACxBT,MAAM,4DAA4DQ;QAIpER,MAAM,+BAA+BS;QACrC,MAAME,eAAeF,sBACjBG,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BH,oBAAoB,WAAW,IAC3D,EAAE;QAENT,MAAM,gBAAgBW;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,EAAE,CAACK,MAAM;YAC1C,IAAI,CAACC,cACH;YAGF,IAAI,CAACT,yBAAyB;gBAC5B,IAAI,AAAwB,YAAxB,OAAOS,cAETN,OAAO,KAAK,CAACK,MAAM,GAAG;oBAAE,QAAQC,aAAa,MAAM;gBAAC;gBAEtD;YACF;YAEAC,IAAAA,sBAAAA,MAAAA,AAAAA,EACET,qBACA;YAEFE,OAAO,KAAK,CAACK,MAAM,GAAG;gBACpB,GAAGC,YAAY;gBACf,kBAAkBR,oBAAoB,6BAA6B,CACjEQ,cACAP;YAEJ;QACF;IACF;AACF"}

package/dist/lib/common.js

@@ -35,13 +35,14 @@ __webpack_require__.d(__webpack_exports__, {
     SizeSchema: ()=>SizeSchema,
     findAllMidsceneLocatorField: ()=>findAllMidsceneLocatorField,
     multimodalPromptToChatMessages: ()=>multimodalPromptToChatMessages,
+    findActionInActionSpaceOrThrow: ()=>findActionInActionSpaceOrThrow,
     getMidsceneLocationSchema: ()=>getMidsceneLocationSchema,
     finalizeActionName: ()=>finalizeActionName,
     TMultimodalPromptSchema: ()=>TMultimodalPromptSchema,
-    ifMidsceneLocatorField: ()=>ifMidsceneLocatorField,
     PointSchema: ()=>PointSchema,
     RectSchema: ()=>RectSchema,
     dumpActionParam: ()=>dumpActionParam,
+    ifMidsceneLocatorField: ()=>ifMidsceneLocatorField,
     userPromptToMultimodalPrompt: ()=>userPromptToMultimodalPrompt
 });
 const constants_namespaceObject = require("@midscene/shared/constants");
@@ -91,15 +92,19 @@ async function markupImageForLLM(screenshotBase64, tree, size) {
     });
     return imagePayload;
 }
+function findActionInActionSpaceOrThrow(planType, actionSpace) {
+    const action = actionSpace.find((item)=>item.name === planType);
+    if (!action) {
+        const available = actionSpace.map((item)=>item.name).join(', ');
+        throw new Error(`Action type '${planType}' is not in the current action space. Available actions: ${available || '(none)'}`);
+    }
+    return action;
+}
 function buildYamlFlowFromPlans(plans, actionSpace) {
     const flow = [];
     for (const plan of plans){
         const verb = plan.type;
-        const action = actionSpace.find((action)=>action.name === verb);
-        if (!action) {
-            console.warn(`Cannot convert action ${verb} to yaml flow. Will ignore it.`);
-            continue;
-        }
+        const action = findActionInActionSpaceOrThrow(verb, actionSpace);
         const flowKey = action.interfaceAlias || verb;
         const flowParam = action.paramSchema ? dumpActionParam(plan.param || {}, action.paramSchema) : {};
         const shortcutField = 'Launch' === action.name || 'launch' === action.interfaceAlias ? 'uri' : 'Terminate' === action.name || 'terminate' === action.interfaceAlias ? 'uri' : 'RunAdbShell' === action.name || 'runAdbShell' === action.interfaceAlias || 'RunHdcShell' === action.name || 'runHdcShell' === action.interfaceAlias ? 'command' : void 0;
@@ -315,6 +320,7 @@ exports.dumpActionParam = __webpack_exports__.dumpActionParam;
 exports.dumpMidsceneLocatorField = __webpack_exports__.dumpMidsceneLocatorField;
 exports.expandSearchArea = __webpack_exports__.expandSearchArea;
 exports.finalizeActionName = __webpack_exports__.finalizeActionName;
+exports.findActionInActionSpaceOrThrow = __webpack_exports__.findActionInActionSpaceOrThrow;
 exports.findAllMidsceneLocatorField = __webpack_exports__.findAllMidsceneLocatorField;
 exports.getMidsceneLocationSchema = __webpack_exports__.getMidsceneLocationSchema;
 exports.getReadableTimeString = __webpack_exports__.getReadableTimeString;
@@ -335,6 +341,7 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "dumpMidsceneLocatorField",
     "expandSearchArea",
     "finalizeActionName",
+    "findActionInActionSpaceOrThrow",
     "findAllMidsceneLocatorField",
     "getMidsceneLocationSchema",
     "getReadableTimeString",

package/dist/lib/common.js.map

@@ -1 +1 @@
-{"version":3,"file":"common.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/common.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport { treeToList } from '@midscene/shared/extractor';\nimport {\n  compositeElementInfoImg,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { assert, isPlainObject } from '@midscene/shared/utils';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\nimport { z } from 'zod';\n\n/**\n * Expand the search area to at least 400 x 400 pixels\n *\n * Step 1: Extend 100px on each side (top, right, bottom, left)\n * - If the element is near a boundary, expansion on that side will be limited\n * - No compensation is made for boundary limitations (this is intentional)\n *\n * Step 2: Ensure the area is at least 400x400 pixels\n * - Scale up proportionally from the center if needed\n * - Final result is clamped to screen boundaries\n */\nexport function expandSearchArea(rect: Rect, screenSize: Size): Rect {\n  const minArea = 400 * 400;\n  const expandSize = 100;\n\n  // Step 1: Extend each side by expandSize (100px), clamped to screen boundaries\n  // Note: If element is near boundary, actual expansion may be less than 100px on that side\n  const expandedLeft = Math.max(rect.left - expandSize, 0);\n  const expandedTop = Math.max(rect.top - expandSize, 0);\n\n  const expandRect = {\n    left: expandedLeft,\n    top: expandedTop,\n    width: Math.min(\n      rect.left - expandedLeft + rect.width + expandSize,\n      screenSize.width - expandedLeft,\n    ),\n    height: Math.min(\n      rect.top - expandedTop + rect.height + expandSize,\n      screenSize.height - expandedTop,\n    ),\n  };\n\n  // Step 2: Check if area is already >= 400x400\n  const currentArea = expandRect.width * expandRect.height;\n\n  if (currentArea >= minArea) {\n    return expandRect;\n  }\n\n  // Step 2: Scale up from center to reach minimum 400x400 area\n  const centerX = expandRect.left + expandRect.width / 2;\n  const centerY = expandRect.top + expandRect.height / 2;\n\n  // Calculate scale factor needed to reach minimum area\n  const scaleFactor = Math.sqrt(minArea / currentArea);\n  const newWidth = Math.round(expandRect.width * scaleFactor);\n  const newHeight = Math.round(expandRect.height * scaleFactor);\n\n  // Calculate new position based on center point\n  const newLeft = Math.round(centerX - newWidth / 2);\n  const newTop = Math.round(centerY - newHeight / 2);\n\n  // Clamp to screen boundaries\n  const left = Math.max(newLeft, 0);\n  const top = Math.max(newTop, 0);\n\n  return {\n    left,\n    top,\n    width: Math.min(newWidth, screenSize.width - left),\n    height: Math.min(newHeight, screenSize.height - top),\n  };\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    // For actions whose param is a single string field (e.g. Launch/Terminate's\n    // `uri`, RunAdbShell's `command`), inline the value on the flowKey. Writing\n    // `{ terminate: '', uri: '...' }` makes the YAML player treat the empty\n    // string as the param and drop the sibling `uri`, so cache replay would\n    // call the action with an empty argument.\n    const shortcutField =\n      action.name === 'Launch' || action.interfaceAlias === 'launch'\n        ? 'uri'\n        : action.name === 'Terminate' || action.interfaceAlias === 'terminate'\n          ? 'uri'\n          : action.name === 'RunAdbShell' ||\n              action.interfaceAlias === 'runAdbShell' ||\n              action.name === 'RunHdcShell' ||\n              action.interfaceAlias === 'runHdcShell'\n            ? 'command'\n            : undefined;\n    const shortcutKeys = shortcutField ? Object.keys(flowParam) : [];\n    const canInlineShortcut =\n      shortcutField &&\n      shortcutKeys.length === 1 &&\n      shortcutKeys[0] === shortcutField &&\n      typeof flowParam[shortcutField] === 'string';\n\n    const flowItem: MidsceneYamlFlowItem = canInlineShortcut\n      ? { [flowKey]: flowParam[shortcutField as string] }\n      : { [flowKey]: '', ...flowParam };\n\n    flow.push(flowItem);\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nexport const userPromptToString = (prompt: TUserPrompt): string => {\n  return typeof prompt === 'string' ? prompt : prompt.prompt;\n};\n\nexport const userPromptToMultimodalPrompt = (\n  prompt: TUserPrompt,\n): TMultimodalPrompt | undefined => {\n  if (typeof prompt === 'string' || !prompt.images) {\n    return undefined;\n  }\n  return {\n    images: prompt.images,\n    convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n  };\n};\n\nexport const multimodalPromptToChatMessages = async (\n  multimodalPrompt?: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images. These reference images are supporting context only, not the current screenshot being evaluated, unless the task explicitly asks for comparison or matching.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const imagePayload = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}'. It is a reference image, not the current screenshot:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepLocate: z.boolean().optional(),\n    deepThink: z\n      .boolean()\n      .optional()\n      .describe('@deprecated Use `deepLocate` instead.'),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Method 1: Check for the location field flag (for result schema)\n    if (locateFieldFlagName in shape) {\n      return true;\n    }\n\n    // Method 2: Check if it's the input schema by checking for 'prompt' field\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nconst formatPromptWithImages = (\n  promptObj: Exclude<TUserPrompt, string>,\n): string => {\n  let promptString = promptObj.prompt;\n  if (Array.isArray(promptObj.images) && promptObj.images.length > 0) {\n    const imageCount = promptObj.images.length;\n    promptString += ` (with ${imageCount} image${imageCount > 1 ? 's' : ''})`;\n  }\n  return promptString;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return formatPromptWithImages(field.prompt);\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Prevent spreading strings into {0: 'c', 1: 'o', ...}\n  if (!isPlainObject(jsonObject)) {\n    return {};\n  }\n\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = formatPromptWithImages(fieldValue.prompt);\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n *\n * When shrunkShotToLogicalRatio is provided and !== 1, coordinates in locate fields\n * are transformed from screenshot space to logical space.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any> | undefined,\n  zodSchema?: z.ZodType<any>,\n  options?: { shrunkShotToLogicalRatio?: number },\n): Record<string, any> | undefined => {\n  // If no schema is provided, return undefined (action takes no parameters)\n  if (!zodSchema) {\n    return undefined;\n  }\n\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement),\n  // and transform coordinates from screenshot space to logical space if needed\n  const ratio = options?.shrunkShotToLogicalRatio;\n  for (const fieldName in locateFieldValues) {\n    let value = locateFieldValues[fieldName];\n    if (\n      ratio !== undefined &&\n      ratio !== 1 &&\n      value &&\n      typeof value === 'object' &&\n      value.center &&\n      value.rect\n    ) {\n      value = {\n        ...value,\n        center: [\n          Math.round(value.center[0] / ratio),\n          Math.round(value.center[1] / ratio),\n        ],\n        rect: {\n          ...value.rect,\n          left: Math.round(value.rect.left / ratio),\n          top: Math.round(value.rect.top / ratio),\n          width: Math.round(value.rect.width / ratio),\n          height: Math.round(value.rect.height / ratio),\n        },\n      };\n    }\n    validated[fieldName] = value;\n  }\n\n  return validated;\n};\n\nexport const finalizeActionName = 'Finalize';\n\n/**\n * Get a readable time string for a given timestamp or the current time\n * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'\n * @param timestamp - Optional timestamp in milliseconds. If not provided, uses current system time.\n * @returns A formatted time string with format label\n */\nexport const getReadableTimeString = (\n  format = 'YYYY-MM-DD HH:mm:ss',\n  timestamp?: number,\n): string => {\n  const now = timestamp !== undefined ? new Date(timestamp) : new Date();\n  const year = now.getFullYear();\n  const month = String(now.getMonth() + 1).padStart(2, '0');\n  const day = String(now.getDate()).padStart(2, '0');\n  const hours = String(now.getHours()).padStart(2, '0');\n  const minutes = String(now.getMinutes()).padStart(2, '0');\n  const seconds = String(now.getSeconds()).padStart(2, '0');\n\n  const timeString = format\n    .replace('YYYY', String(year))\n    .replace('MM', month)\n    .replace('DD', day)\n    .replace('HH', hours)\n    .replace('mm', minutes)\n    .replace('ss', seconds);\n\n  return `${timeString} (${format})`;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","expandSearchArea","rect","screenSize","minArea","expandSize","expandedLeft","Math","expandedTop","expandRect","currentArea","centerX","centerY","scaleFactor","newWidth","newHeight","newLeft","newTop","left","top","markupImageForLLM","screenshotBase64","tree","size","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","flow","plan","verb","action","console","flowKey","flowParam","dumpActionParam","shortcutField","undefined","shortcutKeys","canInlineShortcut","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","userPromptToString","prompt","userPromptToMultimodalPrompt","multimodalPromptToChatMessages","multimodalPrompt","msgs","item","preProcessImageUrl","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","formatPromptWithImages","promptObj","promptString","Array","imageCount","dumpMidsceneLocatorField","assert","String","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","jsonObject","zodSchema","isPlainObject","locatorFields","result","fieldName","fieldValue","parseActionParam","rawParam","options","param","locateFields","locateFieldValues","paramsForValidation","validated","ratio","value","finalizeActionName","getReadableTimeString","format","timestamp","now","Date","year","month","day","hours","minutes","seconds","timeString"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACwBO,SAASI,iBAAiBC,IAAU,EAAEC,UAAgB;IAC3D,MAAMC,UAAU;IAChB,MAAMC,aAAa;IAInB,MAAMC,eAAeC,KAAK,GAAG,CAACL,KAAK,IAAI,GAAGG,YAAY;IACtD,MAAMG,cAAcD,KAAK,GAAG,CAACL,KAAK,GAAG,GAAGG,YAAY;IAEpD,MAAMI,aAAa;QACjB,MAAMH;QACN,KAAKE;QACL,OAAOD,KAAK,GAAG,CACbL,KAAK,IAAI,GAAGI,eAAeJ,KAAK,KAAK,GAAGG,YACxCF,WAAW,KAAK,GAAGG;QAErB,QAAQC,KAAK,GAAG,CACdL,KAAK,GAAG,GAAGM,cAAcN,KAAK,MAAM,GAAGG,YACvCF,WAAW,MAAM,GAAGK;IAExB;IAGA,MAAME,cAAcD,WAAW,KAAK,GAAGA,WAAW,MAAM;IAExD,IAAIC,eAAeN,SACjB,OAAOK;IAIT,MAAME,UAAUF,WAAW,IAAI,GAAGA,WAAW,KAAK,GAAG;IACrD,MAAMG,UAAUH,WAAW,GAAG,GAAGA,WAAW,MAAM,GAAG;IAGrD,MAAMI,cAAcN,KAAK,IAAI,CAACH,UAAUM;IACxC,MAAMI,WAAWP,KAAK,KAAK,CAACE,WAAW,KAAK,GAAGI;IAC/C,MAAME,YAAYR,KAAK,KAAK,CAACE,WAAW,MAAM,GAAGI;IAGjD,MAAMG,UAAUT,KAAK,KAAK,CAACI,UAAUG,WAAW;IAChD,MAAMG,SAASV,KAAK,KAAK,CAACK,UAAUG,YAAY;IAGhD,MAAMG,OAAOX,KAAK,GAAG,CAACS,SAAS;IAC/B,MAAMG,MAAMZ,KAAK,GAAG,CAACU,QAAQ;IAE7B,OAAO;QACLC;QACAC;QACA,OAAOZ,KAAK,GAAG,CAACO,UAAUX,WAAW,KAAK,GAAGe;QAC7C,QAAQX,KAAK,GAAG,CAACQ,WAAWZ,WAAW,MAAM,GAAGgB;IAClD;AACF;AAEO,eAAeC,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCC,IAAU;IAEV,MAAMC,eAAeC,AAAAA,IAAAA,0BAAAA,UAAAA,AAAAA,EAAWH;IAChC,MAAMI,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,0BAAAA,QAAAA,CAAAA,IAAa,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;QACjD,gBAAgBT;QAChB,sBAAsBK;QACtBH;IACF;IACA,OAAOM;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC;IAEhC,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQH,MAAO;QACxB,MAAMI,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASJ,YAAY,IAAI,CAAC,CAACI,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEF,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMG,UAAUF,OAAO,cAAc,IAAID;QACzC,MAAMI,YAAYH,OAAO,WAAW,GAChCI,gBAAgBN,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAOL,MAAMK,gBACJL,AAAgB,aAAhBA,OAAO,IAAI,IAAiBA,AAA0B,aAA1BA,OAAO,cAAc,GAC7C,QACAA,AAAgB,gBAAhBA,OAAO,IAAI,IAAoBA,AAA0B,gBAA1BA,OAAO,cAAc,GAClD,QACAA,AAAgB,kBAAhBA,OAAO,IAAI,IACTA,AAA0B,kBAA1BA,OAAO,cAAc,IACrBA,AAAgB,kBAAhBA,OAAO,IAAI,IACXA,AAA0B,kBAA1BA,OAAO,cAAc,GACrB,YACAM;QACV,MAAMC,eAAeF,gBAAgB7C,OAAO,IAAI,CAAC2C,aAAa,EAAE;QAChE,MAAMK,oBACJH,iBACAE,AAAwB,MAAxBA,aAAa,MAAM,IACnBA,YAAY,CAAC,EAAE,KAAKF,iBACpB,AAAoC,YAApC,OAAOF,SAAS,CAACE,cAAc;QAEjC,MAAMI,WAAiCD,oBACnC;YAAE,CAACN,QAAQ,EAAEC,SAAS,CAACE,cAAwB;QAAC,IAChD;YAAE,CAACH,QAAQ,EAAE;YAAI,GAAGC,SAAS;QAAC;QAElCN,KAAK,IAAI,CAACY;IACZ;IAEA,OAAOZ;AACT;AAGO,MAAMa,cAAcC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClC,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;IACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;AACf;AAEO,MAAMC,aAAaD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjC,OAAOA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;AAClB;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,QAAQA,6BAAAA,CAAAA,CAAAA,KACA,CACJA,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;QACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;QACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,IAED,QAAQ;IACX,yBAAyBA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;IACvCA,6BAAAA,CAAAA,CAAAA,MAAQ;IACRA,6BAAAA,CAAAA,CAAAA,MACS,CAAC;QACN,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMM,MAAME,qBAAqB,CAACC,SAC1B,AAAkB,YAAlB,OAAOA,SAAsBA,SAASA,OAAO,MAAM;AAGrD,MAAMC,+BAA+B,CAC1CD;IAEA,IAAI,AAAkB,YAAlB,OAAOA,UAAuB,CAACA,OAAO,MAAM,EAC9C;IAEF,OAAO;QACL,QAAQA,OAAO,MAAM;QACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;IAC3D;AACF;AAEO,MAAME,iCAAiC,OAC5CC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAM5B,eAAe,MAAM+B,AAAAA,IAAAA,oBAAAA,kBAAAA,AAAAA,EACzBD,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,uDAAuD,CAAC;oBAChH;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAK7B;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAO6B;AACT;AAEA,MAAMG,sBAAsB;AAG5B,MAAMC,wBAAwBd,6BAAAA,CAAAA,CAAAA,MACrB,CAAC;IACN,QAAQI;IACR,YAAYJ,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAChC,WAAWA,6BAAAA,CAAAA,CAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,OAAOA,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,OAAS;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAMP,MAAMe,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIL,uBAAuBM,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEA,MAAMC,yBAAyB,CAC7BC;IAEA,IAAIC,eAAeD,UAAU,MAAM;IACnC,IAAIE,MAAM,OAAO,CAACF,UAAU,MAAM,KAAKA,UAAU,MAAM,CAAC,MAAM,GAAG,GAAG;QAClE,MAAMG,aAAaH,UAAU,MAAM,CAAC,MAAM;QAC1CC,gBAAgB,CAAC,OAAO,EAAEE,WAAW,MAAM,EAAEA,aAAa,IAAI,MAAM,GAAG,CAAC,CAAC;IAC3E;IACA,OAAOF;AACT;AAEO,MAAMG,2BAA2B,CAACR;IACvCS,IAAAA,sBAAAA,MAAAA,AAAAA,EACEV,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOG,uBAAuBH,MAAM,MAAM;IAE9C;IAGA,OAAOU,OAAOV;AAChB;AAEO,MAAMW,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOnF,OAAO,IAAI,CAACkF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACpF;YAClB,MAAMqE,QAAQc,UAAU,KAAK,CAACnF,IAAI;YAClC,IAAI,CAACoE,uBAAuBC,QAC1B,OAAO;YAIT,IAAIa,cACF,OAAOb,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMxB,kBAAkB,CAC7BwC,YACAC;IAGA,IAAI,CAACC,AAAAA,IAAAA,sBAAAA,aAAAA,AAAAA,EAAcF,aACjB,OAAO,CAAC;IAGV,MAAMG,gBAAgBR,4BAA4BM;IAClD,MAAMG,SAAS;QAAE,GAAGJ,UAAU;IAAC;IAE/B,KAAK,MAAMK,aAAaF,cAAe;QACrC,MAAMG,aAAaF,MAAM,CAACC,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTF,MAAM,CAACC,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BF,MAAM,CAACC,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBF,MAAM,CAACC,UAAU,GAAGlB,uBAAuBmB,WAAW,MAAM;gBAC9D;YACF;QACF;IAEJ;IAEA,OAAOF;AACT;AAaO,MAAMG,mBAAmB,CAC9BC,UACAP,WACAQ;IAGA,IAAI,CAACR,WACH;IAIF,MAAMS,QAAQF,YAAY,CAAC;IAG3B,MAAMG,eAAehB,4BAA4BM;IAGjD,IAAIU,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOV,UAAU,KAAK,CAACS;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMlG,OAAO+F,MAChB,IAAIC,aAAa,QAAQ,CAAChG,MAExBkG,mBAAmB,CAAClG,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/CkG,mBAAmB,CAAClG,IAAI,GAAG+F,KAAK,CAAC/F,IAAI;IAKzC,MAAMmG,YAAYb,UAAU,KAAK,CAACY;IAIlC,MAAME,QAAQN,SAAS;IACvB,IAAK,MAAMJ,aAAaO,kBAAmB;QACzC,IAAII,QAAQJ,iBAAiB,CAACP,UAAU;QACxC,IACEU,AAAUrD,WAAVqD,SACAA,AAAU,MAAVA,SACAC,SACA,AAAiB,YAAjB,OAAOA,SACPA,MAAM,MAAM,IACZA,MAAM,IAAI,EAEVA,QAAQ;YACN,GAAGA,KAAK;YACR,QAAQ;gBACN1F,KAAK,KAAK,CAAC0F,MAAM,MAAM,CAAC,EAAE,GAAGD;gBAC7BzF,KAAK,KAAK,CAAC0F,MAAM,MAAM,CAAC,EAAE,GAAGD;aAC9B;YACD,MAAM;gBACJ,GAAGC,MAAM,IAAI;gBACb,MAAM1F,KAAK,KAAK,CAAC0F,MAAM,IAAI,CAAC,IAAI,GAAGD;gBACnC,KAAKzF,KAAK,KAAK,CAAC0F,MAAM,IAAI,CAAC,GAAG,GAAGD;gBACjC,OAAOzF,KAAK,KAAK,CAAC0F,MAAM,IAAI,CAAC,KAAK,GAAGD;gBACrC,QAAQzF,KAAK,KAAK,CAAC0F,MAAM,IAAI,CAAC,MAAM,GAAGD;YACzC;QACF;QAEFD,SAAS,CAACT,UAAU,GAAGW;IACzB;IAEA,OAAOF;AACT;AAEO,MAAMG,qBAAqB;AAQ3B,MAAMC,wBAAwB,CACnCC,SAAS,qBAAqB,EAC9BC;IAEA,MAAMC,MAAMD,AAAc1D,WAAd0D,YAA0B,IAAIE,KAAKF,aAAa,IAAIE;IAChE,MAAMC,OAAOF,IAAI,WAAW;IAC5B,MAAMG,QAAQ9B,OAAO2B,IAAI,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACrD,MAAMI,MAAM/B,OAAO2B,IAAI,OAAO,IAAI,QAAQ,CAAC,GAAG;IAC9C,MAAMK,QAAQhC,OAAO2B,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMM,UAAUjC,OAAO2B,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IACrD,MAAMO,UAAUlC,OAAO2B,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IAErD,MAAMQ,aAAaV,OAChB,OAAO,CAAC,QAAQzB,OAAO6B,OACvB,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,KACd,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,SACd,OAAO,CAAC,MAAMC;IAEjB,OAAO,GAAGC,WAAW,EAAE,EAAEV,OAAO,CAAC,CAAC;AACpC"}
+{"version":3,"file":"common.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/common.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport { treeToList } from '@midscene/shared/extractor';\nimport {\n  compositeElementInfoImg,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { assert, isPlainObject } from '@midscene/shared/utils';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\nimport { z } from 'zod';\n\n/**\n * Expand the search area to at least 400 x 400 pixels\n *\n * Step 1: Extend 100px on each side (top, right, bottom, left)\n * - If the element is near a boundary, expansion on that side will be limited\n * - No compensation is made for boundary limitations (this is intentional)\n *\n * Step 2: Ensure the area is at least 400x400 pixels\n * - Scale up proportionally from the center if needed\n * - Final result is clamped to screen boundaries\n */\nexport function expandSearchArea(rect: Rect, screenSize: Size): Rect {\n  const minArea = 400 * 400;\n  const expandSize = 100;\n\n  // Step 1: Extend each side by expandSize (100px), clamped to screen boundaries\n  // Note: If element is near boundary, actual expansion may be less than 100px on that side\n  const expandedLeft = Math.max(rect.left - expandSize, 0);\n  const expandedTop = Math.max(rect.top - expandSize, 0);\n\n  const expandRect = {\n    left: expandedLeft,\n    top: expandedTop,\n    width: Math.min(\n      rect.left - expandedLeft + rect.width + expandSize,\n      screenSize.width - expandedLeft,\n    ),\n    height: Math.min(\n      rect.top - expandedTop + rect.height + expandSize,\n      screenSize.height - expandedTop,\n    ),\n  };\n\n  // Step 2: Check if area is already >= 400x400\n  const currentArea = expandRect.width * expandRect.height;\n\n  if (currentArea >= minArea) {\n    return expandRect;\n  }\n\n  // Step 2: Scale up from center to reach minimum 400x400 area\n  const centerX = expandRect.left + expandRect.width / 2;\n  const centerY = expandRect.top + expandRect.height / 2;\n\n  // Calculate scale factor needed to reach minimum area\n  const scaleFactor = Math.sqrt(minArea / currentArea);\n  const newWidth = Math.round(expandRect.width * scaleFactor);\n  const newHeight = Math.round(expandRect.height * scaleFactor);\n\n  // Calculate new position based on center point\n  const newLeft = Math.round(centerX - newWidth / 2);\n  const newTop = Math.round(centerY - newHeight / 2);\n\n  // Clamp to screen boundaries\n  const left = Math.max(newLeft, 0);\n  const top = Math.max(newTop, 0);\n\n  return {\n    left,\n    top,\n    width: Math.min(newWidth, screenSize.width - left),\n    height: Math.min(newHeight, screenSize.height - top),\n  };\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function findActionInActionSpaceOrThrow(\n  planType: string,\n  actionSpace: DeviceAction<any>[],\n): DeviceAction<any> {\n  const action = actionSpace.find((item) => item.name === planType);\n  if (!action) {\n    const available = actionSpace.map((item) => item.name).join(', ');\n    throw new Error(\n      `Action type '${planType}' is not in the current action space. Available actions: ${available || '(none)'}`,\n    );\n  }\n  return action;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = findActionInActionSpaceOrThrow(verb, actionSpace);\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    // For actions whose param is a single string field (e.g. Launch/Terminate's\n    // `uri`, RunAdbShell's `command`), inline the value on the flowKey. Writing\n    // `{ terminate: '', uri: '...' }` makes the YAML player treat the empty\n    // string as the param and drop the sibling `uri`, so cache replay would\n    // call the action with an empty argument.\n    const shortcutField =\n      action.name === 'Launch' || action.interfaceAlias === 'launch'\n        ? 'uri'\n        : action.name === 'Terminate' || action.interfaceAlias === 'terminate'\n          ? 'uri'\n          : action.name === 'RunAdbShell' ||\n              action.interfaceAlias === 'runAdbShell' ||\n              action.name === 'RunHdcShell' ||\n              action.interfaceAlias === 'runHdcShell'\n            ? 'command'\n            : undefined;\n    const shortcutKeys = shortcutField ? Object.keys(flowParam) : [];\n    const canInlineShortcut =\n      shortcutField &&\n      shortcutKeys.length === 1 &&\n      shortcutKeys[0] === shortcutField &&\n      typeof flowParam[shortcutField] === 'string';\n\n    const flowItem: MidsceneYamlFlowItem = canInlineShortcut\n      ? { [flowKey]: flowParam[shortcutField as string] }\n      : { [flowKey]: '', ...flowParam };\n\n    flow.push(flowItem);\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nexport const userPromptToString = (prompt: TUserPrompt): string => {\n  return typeof prompt === 'string' ? prompt : prompt.prompt;\n};\n\nexport const userPromptToMultimodalPrompt = (\n  prompt: TUserPrompt,\n): TMultimodalPrompt | undefined => {\n  if (typeof prompt === 'string' || !prompt.images) {\n    return undefined;\n  }\n  return {\n    images: prompt.images,\n    convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n  };\n};\n\nexport const multimodalPromptToChatMessages = async (\n  multimodalPrompt?: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images. These reference images are supporting context only, not the current screenshot being evaluated, unless the task explicitly asks for comparison or matching.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const imagePayload = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}'. It is a reference image, not the current screenshot:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepLocate: z.boolean().optional(),\n    deepThink: z\n      .boolean()\n      .optional()\n      .describe('@deprecated Use `deepLocate` instead.'),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Method 1: Check for the location field flag (for result schema)\n    if (locateFieldFlagName in shape) {\n      return true;\n    }\n\n    // Method 2: Check if it's the input schema by checking for 'prompt' field\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nconst formatPromptWithImages = (\n  promptObj: Exclude<TUserPrompt, string>,\n): string => {\n  let promptString = promptObj.prompt;\n  if (Array.isArray(promptObj.images) && promptObj.images.length > 0) {\n    const imageCount = promptObj.images.length;\n    promptString += ` (with ${imageCount} image${imageCount > 1 ? 's' : ''})`;\n  }\n  return promptString;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return formatPromptWithImages(field.prompt);\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Prevent spreading strings into {0: 'c', 1: 'o', ...}\n  if (!isPlainObject(jsonObject)) {\n    return {};\n  }\n\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = formatPromptWithImages(fieldValue.prompt);\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n *\n * When shrunkShotToLogicalRatio is provided and !== 1, coordinates in locate fields\n * are transformed from screenshot space to logical space.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any> | undefined,\n  zodSchema?: z.ZodType<any>,\n  options?: { shrunkShotToLogicalRatio?: number },\n): Record<string, any> | undefined => {\n  // If no schema is provided, return undefined (action takes no parameters)\n  if (!zodSchema) {\n    return undefined;\n  }\n\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement),\n  // and transform coordinates from screenshot space to logical space if needed\n  const ratio = options?.shrunkShotToLogicalRatio;\n  for (const fieldName in locateFieldValues) {\n    let value = locateFieldValues[fieldName];\n    if (\n      ratio !== undefined &&\n      ratio !== 1 &&\n      value &&\n      typeof value === 'object' &&\n      value.center &&\n      value.rect\n    ) {\n      value = {\n        ...value,\n        center: [\n          Math.round(value.center[0] / ratio),\n          Math.round(value.center[1] / ratio),\n        ],\n        rect: {\n          ...value.rect,\n          left: Math.round(value.rect.left / ratio),\n          top: Math.round(value.rect.top / ratio),\n          width: Math.round(value.rect.width / ratio),\n          height: Math.round(value.rect.height / ratio),\n        },\n      };\n    }\n    validated[fieldName] = value;\n  }\n\n  return validated;\n};\n\nexport const finalizeActionName = 'Finalize';\n\n/**\n * Get a readable time string for a given timestamp or the current time\n * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'\n * @param timestamp - Optional timestamp in milliseconds. If not provided, uses current system time.\n * @returns A formatted time string with format label\n */\nexport const getReadableTimeString = (\n  format = 'YYYY-MM-DD HH:mm:ss',\n  timestamp?: number,\n): string => {\n  const now = timestamp !== undefined ? new Date(timestamp) : new Date();\n  const year = now.getFullYear();\n  const month = String(now.getMonth() + 1).padStart(2, '0');\n  const day = String(now.getDate()).padStart(2, '0');\n  const hours = String(now.getHours()).padStart(2, '0');\n  const minutes = String(now.getMinutes()).padStart(2, '0');\n  const seconds = String(now.getSeconds()).padStart(2, '0');\n\n  const timeString = format\n    .replace('YYYY', String(year))\n    .replace('MM', month)\n    .replace('DD', day)\n    .replace('HH', hours)\n    .replace('mm', minutes)\n    .replace('ss', seconds);\n\n  return `${timeString} (${format})`;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","expandSearchArea","rect","screenSize","minArea","expandSize","expandedLeft","Math","expandedTop","expandRect","currentArea","centerX","centerY","scaleFactor","newWidth","newHeight","newLeft","newTop","left","top","markupImageForLLM","screenshotBase64","tree","size","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","findActionInActionSpaceOrThrow","planType","actionSpace","action","item","available","Error","buildYamlFlowFromPlans","plans","flow","plan","verb","flowKey","flowParam","dumpActionParam","shortcutField","undefined","shortcutKeys","canInlineShortcut","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","userPromptToString","prompt","userPromptToMultimodalPrompt","multimodalPromptToChatMessages","multimodalPrompt","msgs","preProcessImageUrl","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","formatPromptWithImages","promptObj","promptString","Array","imageCount","dumpMidsceneLocatorField","assert","String","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","jsonObject","zodSchema","isPlainObject","locatorFields","result","fieldName","fieldValue","parseActionParam","rawParam","options","param","locateFields","locateFieldValues","paramsForValidation","validated","ratio","value","finalizeActionName","getReadableTimeString","format","timestamp","now","Date","year","month","day","hours","minutes","seconds","timeString"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACwBO,SAASI,iBAAiBC,IAAU,EAAEC,UAAgB;IAC3D,MAAMC,UAAU;IAChB,MAAMC,aAAa;IAInB,MAAMC,eAAeC,KAAK,GAAG,CAACL,KAAK,IAAI,GAAGG,YAAY;IACtD,MAAMG,cAAcD,KAAK,GAAG,CAACL,KAAK,GAAG,GAAGG,YAAY;IAEpD,MAAMI,aAAa;QACjB,MAAMH;QACN,KAAKE;QACL,OAAOD,KAAK,GAAG,CACbL,KAAK,IAAI,GAAGI,eAAeJ,KAAK,KAAK,GAAGG,YACxCF,WAAW,KAAK,GAAGG;QAErB,QAAQC,KAAK,GAAG,CACdL,KAAK,GAAG,GAAGM,cAAcN,KAAK,MAAM,GAAGG,YACvCF,WAAW,MAAM,GAAGK;IAExB;IAGA,MAAME,cAAcD,WAAW,KAAK,GAAGA,WAAW,MAAM;IAExD,IAAIC,eAAeN,SACjB,OAAOK;IAIT,MAAME,UAAUF,WAAW,IAAI,GAAGA,WAAW,KAAK,GAAG;IACrD,MAAMG,UAAUH,WAAW,GAAG,GAAGA,WAAW,MAAM,GAAG;IAGrD,MAAMI,cAAcN,KAAK,IAAI,CAACH,UAAUM;IACxC,MAAMI,WAAWP,KAAK,KAAK,CAACE,WAAW,KAAK,GAAGI;IAC/C,MAAME,YAAYR,KAAK,KAAK,CAACE,WAAW,MAAM,GAAGI;IAGjD,MAAMG,UAAUT,KAAK,KAAK,CAACI,UAAUG,WAAW;IAChD,MAAMG,SAASV,KAAK,KAAK,CAACK,UAAUG,YAAY;IAGhD,MAAMG,OAAOX,KAAK,GAAG,CAACS,SAAS;IAC/B,MAAMG,MAAMZ,KAAK,GAAG,CAACU,QAAQ;IAE7B,OAAO;QACLC;QACAC;QACA,OAAOZ,KAAK,GAAG,CAACO,UAAUX,WAAW,KAAK,GAAGe;QAC7C,QAAQX,KAAK,GAAG,CAACQ,WAAWZ,WAAW,MAAM,GAAGgB;IAClD;AACF;AAEO,eAAeC,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCC,IAAU;IAEV,MAAMC,eAAeC,AAAAA,IAAAA,0BAAAA,UAAAA,AAAAA,EAAWH;IAChC,MAAMI,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,0BAAAA,QAAAA,CAAAA,IAAa,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;QACjD,gBAAgBT;QAChB,sBAAsBK;QACtBH;IACF;IACA,OAAOM;AACT;AAEO,SAASE,+BACdC,QAAgB,EAChBC,WAAgC;IAEhC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;IACxD,IAAI,CAACE,QAAQ;QACX,MAAME,YAAYH,YAAY,GAAG,CAAC,CAACE,OAASA,KAAK,IAAI,EAAE,IAAI,CAAC;QAC5D,MAAM,IAAIE,MACR,CAAC,aAAa,EAAEL,SAAS,yDAAyD,EAAEI,aAAa,UAAU;IAE/G;IACA,OAAOF;AACT;AAEO,SAASI,uBACdC,KAAuB,EACvBN,WAAgC;IAEhC,MAAMO,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQF,MAAO;QACxB,MAAMG,OAAOD,KAAK,IAAI;QAEtB,MAAMP,SAASH,+BAA+BW,MAAMT;QAEpD,MAAMU,UAAUT,OAAO,cAAc,IAAIQ;QACzC,MAAME,YAAYV,OAAO,WAAW,GAChCW,gBAAgBJ,KAAK,KAAK,IAAI,CAAC,GAAGP,OAAO,WAAW,IACpD,CAAC;QAOL,MAAMY,gBACJZ,AAAgB,aAAhBA,OAAO,IAAI,IAAiBA,AAA0B,aAA1BA,OAAO,cAAc,GAC7C,QACAA,AAAgB,gBAAhBA,OAAO,IAAI,IAAoBA,AAA0B,gBAA1BA,OAAO,cAAc,GAClD,QACAA,AAAgB,kBAAhBA,OAAO,IAAI,IACTA,AAA0B,kBAA1BA,OAAO,cAAc,IACrBA,AAAgB,kBAAhBA,OAAO,IAAI,IACXA,AAA0B,kBAA1BA,OAAO,cAAc,GACrB,YACAa;QACV,MAAMC,eAAeF,gBAAgBjD,OAAO,IAAI,CAAC+C,aAAa,EAAE;QAChE,MAAMK,oBACJH,iBACAE,AAAwB,MAAxBA,aAAa,MAAM,IACnBA,YAAY,CAAC,EAAE,KAAKF,iBACpB,AAAoC,YAApC,OAAOF,SAAS,CAACE,cAAc;QAEjC,MAAMI,WAAiCD,oBACnC;YAAE,CAACN,QAAQ,EAAEC,SAAS,CAACE,cAAwB;QAAC,IAChD;YAAE,CAACH,QAAQ,EAAE;YAAI,GAAGC,SAAS;QAAC;QAElCJ,KAAK,IAAI,CAACU;IACZ;IAEA,OAAOV;AACT;AAGO,MAAMW,cAAcC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClC,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;IACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;AACf;AAEO,MAAMC,aAAaD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjC,OAAOA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;AAClB;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,QAAQA,6BAAAA,CAAAA,CAAAA,KACA,CACJA,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;QACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;QACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,IAED,QAAQ;IACX,yBAAyBA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;IACvCA,6BAAAA,CAAAA,CAAAA,MAAQ;IACRA,6BAAAA,CAAAA,CAAAA,MACS,CAAC;QACN,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMM,MAAME,qBAAqB,CAACC,SAC1B,AAAkB,YAAlB,OAAOA,SAAsBA,SAASA,OAAO,MAAM;AAGrD,MAAMC,+BAA+B,CAC1CD;IAEA,IAAI,AAAkB,YAAlB,OAAOA,UAAuB,CAACA,OAAO,MAAM,EAC9C;IAEF,OAAO;QACL,QAAQA,OAAO,MAAM;QACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;IAC3D;AACF;AAEO,MAAME,iCAAiC,OAC5CC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAM3B,QAAQ0B,iBAAiB,MAAM,CAAE;YAC1C,MAAMhC,eAAe,MAAMkC,AAAAA,IAAAA,oBAAAA,kBAAAA,AAAAA,EACzB5B,KAAK,GAAG,EACR,CAAC,CAAC0B,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAE3B,KAAK,IAAI,CAAC,uDAAuD,CAAC;oBAChH;iBACD;YACH;YAEA2B,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKjC;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOiC;AACT;AAEA,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBb,6BAAAA,CAAAA,CAAAA,MACrB,CAAC;IACN,QAAQI;IACR,YAAYJ,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAChC,WAAWA,6BAAAA,CAAAA,CAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,OAAOA,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,OAAS;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAMP,MAAMc,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIL,uBAAuBM,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEA,MAAMC,yBAAyB,CAC7BC;IAEA,IAAIC,eAAeD,UAAU,MAAM;IACnC,IAAIE,MAAM,OAAO,CAACF,UAAU,MAAM,KAAKA,UAAU,MAAM,CAAC,MAAM,GAAG,GAAG;QAClE,MAAMG,aAAaH,UAAU,MAAM,CAAC,MAAM;QAC1CC,gBAAgB,CAAC,OAAO,EAAEE,WAAW,MAAM,EAAEA,aAAa,IAAI,MAAM,GAAG,CAAC,CAAC;IAC3E;IACA,OAAOF;AACT;AAEO,MAAMG,2BAA2B,CAACR;IACvCS,IAAAA,sBAAAA,MAAAA,AAAAA,EACEV,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOG,uBAAuBH,MAAM,MAAM;IAE9C;IAGA,OAAOU,OAAOV;AAChB;AAEO,MAAMW,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOtF,OAAO,IAAI,CAACqF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACvF;YAClB,MAAMwE,QAAQc,UAAU,KAAK,CAACtF,IAAI;YAClC,IAAI,CAACuE,uBAAuBC,QAC1B,OAAO;YAIT,IAAIa,cACF,OAAOb,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMvB,kBAAkB,CAC7BuC,YACAC;IAGA,IAAI,CAACC,AAAAA,IAAAA,sBAAAA,aAAAA,AAAAA,EAAcF,aACjB,OAAO,CAAC;IAGV,MAAMG,gBAAgBR,4BAA4BM;IAClD,MAAMG,SAAS;QAAE,GAAGJ,UAAU;IAAC;IAE/B,KAAK,MAAMK,aAAaF,cAAe;QACrC,MAAMG,aAAaF,MAAM,CAACC,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTF,MAAM,CAACC,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BF,MAAM,CAACC,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBF,MAAM,CAACC,UAAU,GAAGlB,uBAAuBmB,WAAW,MAAM;gBAC9D;YACF;QACF;IAEJ;IAEA,OAAOF;AACT;AAaO,MAAMG,mBAAmB,CAC9BC,UACAP,WACAQ;IAGA,IAAI,CAACR,WACH;IAIF,MAAMS,QAAQF,YAAY,CAAC;IAG3B,MAAMG,eAAehB,4BAA4BM;IAGjD,IAAIU,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOV,UAAU,KAAK,CAACS;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMrG,OAAOkG,MAChB,IAAIC,aAAa,QAAQ,CAACnG,MAExBqG,mBAAmB,CAACrG,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/CqG,mBAAmB,CAACrG,IAAI,GAAGkG,KAAK,CAAClG,IAAI;IAKzC,MAAMsG,YAAYb,UAAU,KAAK,CAACY;IAIlC,MAAME,QAAQN,SAAS;IACvB,IAAK,MAAMJ,aAAaO,kBAAmB;QACzC,IAAII,QAAQJ,iBAAiB,CAACP,UAAU;QACxC,IACEU,AAAUpD,WAAVoD,SACAA,AAAU,MAAVA,SACAC,SACA,AAAiB,YAAjB,OAAOA,SACPA,MAAM,MAAM,IACZA,MAAM,IAAI,EAEVA,QAAQ;YACN,GAAGA,KAAK;YACR,QAAQ;gBACN7F,KAAK,KAAK,CAAC6F,MAAM,MAAM,CAAC,EAAE,GAAGD;gBAC7B5F,KAAK,KAAK,CAAC6F,MAAM,MAAM,CAAC,EAAE,GAAGD;aAC9B;YACD,MAAM;gBACJ,GAAGC,MAAM,IAAI;gBACb,MAAM7F,KAAK,KAAK,CAAC6F,MAAM,IAAI,CAAC,IAAI,GAAGD;gBACnC,KAAK5F,KAAK,KAAK,CAAC6F,MAAM,IAAI,CAAC,GAAG,GAAGD;gBACjC,OAAO5F,KAAK,KAAK,CAAC6F,MAAM,IAAI,CAAC,KAAK,GAAGD;gBACrC,QAAQ5F,KAAK,KAAK,CAAC6F,MAAM,IAAI,CAAC,MAAM,GAAGD;YACzC;QACF;QAEFD,SAAS,CAACT,UAAU,GAAGW;IACzB;IAEA,OAAOF;AACT;AAEO,MAAMG,qBAAqB;AAQ3B,MAAMC,wBAAwB,CACnCC,SAAS,qBAAqB,EAC9BC;IAEA,MAAMC,MAAMD,AAAczD,WAAdyD,YAA0B,IAAIE,KAAKF,aAAa,IAAIE;IAChE,MAAMC,OAAOF,IAAI,WAAW;IAC5B,MAAMG,QAAQ9B,OAAO2B,IAAI,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACrD,MAAMI,MAAM/B,OAAO2B,IAAI,OAAO,IAAI,QAAQ,CAAC,GAAG;IAC9C,MAAMK,QAAQhC,OAAO2B,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMM,UAAUjC,OAAO2B,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IACrD,MAAMO,UAAUlC,OAAO2B,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IAErD,MAAMQ,aAAaV,OAChB,OAAO,CAAC,QAAQzB,OAAO6B,OACvB,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,KACd,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,SACd,OAAO,CAAC,MAAMC;IAEjB,OAAO,GAAGC,WAAW,EAAE,EAAEV,OAAO,CAAC,CAAC;AACpC"}