@midscene/core 1.16.2 → 1.16.6-beta-20261023073257.0

package/dist/es/service/index.mjs

@@ -191,7 +191,7 @@ class Service {
             }
         };
     }
-    async extract(dataDemand, modelRuntime, opt, pageDescription, multimodalPrompt, context, executionOptions) {
+    async extract(dataDemand, modelRuntime, taskType, opt, pageDescription, multimodalPrompt, context, executionOptions) {
         assert(context, 'context is required for extract');
         assert('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
         const startTime = Date.now();
@@ -204,6 +204,7 @@ class Service {
             const result = await AiExtractElementInfo({
                 context,
                 dataQuery: dataDemand,
+                taskType,
                 multimodalPrompt,
                 extractOption: opt,
                 modelRuntime,

package/dist/es/service/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"service/index.mjs","sources":["../../../src/service/index.ts"],"sourcesContent":["import type { ModelRuntime } from '@/ai-model/models';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport {\n  AIResponseParseError,\n  callAIWithObjectResponse,\n} from '@/ai-model/service-caller';\nimport type { ModelCallMessages } from '@/ai-model/service-caller/types';\nimport type { MessageContent } from '@/ai-model/service-caller/types';\nimport { modelFamilyRequiredForLocateMessage } from '@/ai-model/shared/model-locate-result/errors';\nimport {\n  AiLocateElement,\n  AiLocateSection,\n  buildSearchAreaConfig,\n} from '@/ai-model/workflows/grounding';\nimport { mergeSearchAreaResults } from '@/ai-model/workflows/grounding/search-area';\nimport type { SearchAreaConfig } from '@/ai-model/workflows/grounding/types';\nimport { AiExtractElementInfo } from '@/ai-model/workflows/insight';\nimport type { RawAssistantOutput } from '@/types';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport {\n  compositeElementInfoImg,\n  compositePointMarkerImg,\n  cropByRect,\n  resizeBase64ImageToJpeg,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  createServiceDump,\n  getDescribeDeepContextAreas,\n  getDescribeDeepLocateResizeSize,\n  getDescribeMarkerBorderThickness,\n  getDescribeMarkerRect,\n  getRectInCrop,\n  recoverDescribeResponseFromParseError,\n} from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\ninterface LocateSearchAreaResult {\n  config?: SearchAreaConfig;\n  error?: string;\n  trace: {\n    sourceRect?: Rect;\n    rawResponse?: string;\n    rawAssistantOutput?: RawAssistantOutput;\n    usage?: AIUsageInfo;\n  };\n}\n\nconst debug = getDebug('ai:service');\n\nexport default class Service {\n  contextRetrieverFn: (signal?: AbortSignal) => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | UIContext\n      | ((signal?: AbortSignal) => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelRuntime: ModelRuntime,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const { config: modelConfig } = modelRuntime;\n    const queryPrompt = query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    if (!modelConfig.modelFamily) {\n      throw new Error(modelFamilyRequiredForLocateMessage(modelConfig.slot));\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    const searchAreaStartTime = Date.now();\n    const searchArea = await this.resolveLocateSearchArea({\n      query,\n      queryPrompt,\n      opt,\n      context,\n      modelRuntime,\n      abortSignal,\n    });\n\n    if (!searchArea.config && searchArea.error) {\n      const errorMessage = `cannot find search area for \"${queryPrompt}\": ${searchArea.error}`;\n      const taskInfo: ServiceTaskInfo = {\n        ...(this.taskInfo ? this.taskInfo : {}),\n        durationMs: Date.now() - searchAreaStartTime,\n        searchAreaRawResponse: searchArea.trace.rawResponse,\n        searchAreaRawChoiceMessage: searchArea.trace.rawAssistantOutput,\n        searchAreaUsage: searchArea.trace.usage,\n      };\n      const dump = createServiceDump({\n        type: 'locate',\n        userQuery: { element: queryPrompt },\n        matchedElement: [],\n        data: null,\n        taskInfo,\n        deepLocate: true,\n        error: errorMessage,\n      });\n      throw new ServiceError(errorMessage, dump);\n    }\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rawResponse,\n      rawAssistantOutput,\n      usage,\n      reasoning_content,\n    } = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchArea.config,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawAssistantOutput,\n      formatResponse: parseResult,\n      usage,\n      searchArea: searchArea.trace.sourceRect,\n      searchAreaRawResponse: searchArea.trace.rawResponse,\n      searchAreaRawChoiceMessage: searchArea.trace.rawAssistantOutput,\n      searchAreaUsage: searchArea.trace.usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea.trace.sourceRect,\n      error: errorLog,\n    };\n\n    const element = parseResult.element;\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: element ? [element] : [],\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (element) {\n      return {\n        element: {\n          center: element.center,\n          description: element.description,\n          ...(element.rect ? { rect: element.rect } : {}),\n        },\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      dump,\n    };\n  }\n\n  private async resolveLocateSearchArea(options: {\n    query: DetailedLocateParam;\n    queryPrompt: TUserPrompt;\n    opt: LocateOpts;\n    context: UIContext;\n    modelRuntime: ModelRuntime;\n    abortSignal?: AbortSignal;\n  }): Promise<LocateSearchAreaResult> {\n    const { query, queryPrompt, opt, context, modelRuntime, abortSignal } =\n      options;\n    const { adapter } = modelRuntime;\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.center;\n\n    if (!query.deepLocate) {\n      return { trace: {} };\n    }\n\n    // TODO: Make the plan, section-locate, and element-locate fallback paths\n    // resolve only the source rect, then build the search-area config once in\n    // this orchestration layer instead of building it inside AiLocateSection.\n    if (hasPlanLocatedElement) {\n      const config = await buildSearchAreaConfig({\n        context,\n        baseRect: mergeSearchAreaResults(opt.planLocatedElement!),\n      });\n\n      return {\n        config,\n        trace: {\n          sourceRect: config.sourceRect,\n          rawResponse: JSON.stringify({\n            source: 'plan-located-element',\n            center: opt.planLocatedElement!.center,\n            rect: opt.planLocatedElement!.rect,\n          }),\n        },\n      };\n    }\n\n    if (adapter.locate.kind === 'standard' && adapter.locate.searchArea) {\n      const searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: queryPrompt,\n        modelRuntime,\n        abortSignal,\n      });\n      const { searchAreaConfig } = searchAreaResponse;\n      if (!searchAreaConfig) {\n        return {\n          error: searchAreaResponse.error || 'unknown search area error',\n          trace: {\n            rawResponse: searchAreaResponse.rawResponse,\n            rawAssistantOutput: searchAreaResponse.rawAssistantOutput,\n            usage: searchAreaResponse.usage,\n          },\n        };\n      }\n\n      return {\n        config: searchAreaConfig,\n        trace: {\n          sourceRect: searchAreaConfig.sourceRect,\n          rawResponse: searchAreaResponse.rawResponse,\n          rawAssistantOutput: searchAreaResponse.rawAssistantOutput,\n          usage: searchAreaResponse.usage,\n        },\n      };\n    }\n\n    const firstPassLocateResult = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      modelRuntime,\n      abortSignal,\n    });\n    assert(\n      firstPassLocateResult.parseResult.element?.center,\n      `cannot find search area for \"${queryPrompt}\"${\n        firstPassLocateResult.parseResult.errors?.length\n          ? `: ${firstPassLocateResult.parseResult.errors.join('\\n')}`\n          : ''\n      }`,\n    );\n\n    const config = await buildSearchAreaConfig({\n      context,\n      baseRect: mergeSearchAreaResults(\n        firstPassLocateResult.parseResult.element!,\n      ),\n    });\n\n    return {\n      config,\n      trace: {\n        sourceRect: config.sourceRect,\n        rawResponse: JSON.stringify({\n          source: 'deep-locate-first-pass',\n          center: firstPassLocateResult.parseResult.element?.center,\n          rect: firstPassLocateResult.parseResult.element?.rect,\n          rawResponse: firstPassLocateResult.rawResponse,\n        }),\n        rawAssistantOutput: firstPassLocateResult.rawAssistantOutput,\n        usage: firstPassLocateResult.usage,\n      },\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let rawAssistantOutput: RawAssistantOutput | undefined;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelRuntime,\n        pageDescription,\n        abortSignal: executionOptions?.abortSignal,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      rawAssistantOutput = result.rawAssistantOutput;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          rawAssistantOutput: error.rawAssistantOutput,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawAssistantOutput,\n      formatResponse: parseResult,\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelRuntime: ModelRuntime,\n    opt?: {\n      deepDescribe?: boolean;\n      context?: UIContext;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = opt?.context || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetFromPoint = Array.isArray(target);\n    const targetRect: Rect = targetFromPoint\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n    const targetPoint = targetFromPoint\n      ? {\n          x: target[0],\n          y: target[1],\n        }\n      : undefined;\n\n    const usePointMarker = targetFromPoint;\n    const imagePayload = usePointMarker\n      ? await compositePointMarkerImg({\n          inputImgBase64: screenshotBase64,\n          size: shotSize,\n          point: targetPoint!,\n        })\n      : await compositeElementInfoImg({\n          inputImgBase64: screenshotBase64,\n          size: shotSize,\n          elementsPositionInfo: [\n            {\n              rect: getDescribeMarkerRect(targetRect),\n            },\n          ],\n          borderThickness: getDescribeMarkerBorderThickness(targetRect),\n          centerPoint: true,\n        });\n\n    const shouldDeepDescribe = opt?.deepDescribe;\n    let imageContent: MessageContent[];\n    if (shouldDeepDescribe) {\n      const contextAreas = getDescribeDeepContextAreas(targetRect, shotSize);\n      const contextImages = await Promise.all(\n        contextAreas.map(async (area) => {\n          debug('describe: cropping deep context area', area);\n          const croppedResult = await cropByRect(screenshotBase64, area.rect);\n          const cropSize = {\n            width: croppedResult.width,\n            height: croppedResult.height,\n          };\n          const targetInCrop = getRectInCrop(targetRect, area.rect, cropSize);\n          const markedCropPayload = targetFromPoint\n            ? await compositePointMarkerImg({\n                inputImgBase64: croppedResult.imageBase64,\n                size: cropSize,\n                point: {\n                  x: targetPoint!.x - area.rect.left,\n                  y: targetPoint!.y - area.rect.top,\n                },\n              })\n            : await compositeElementInfoImg({\n                inputImgBase64: croppedResult.imageBase64,\n                size: cropSize,\n                elementsPositionInfo: [\n                  {\n                    rect: getDescribeMarkerRect(targetInCrop),\n                  },\n                ],\n                borderThickness: getDescribeMarkerBorderThickness(targetInCrop),\n                centerPoint: true,\n              });\n          const resizeSize = getDescribeDeepLocateResizeSize(croppedResult);\n          return {\n            kind: area.kind,\n            imageBase64: resizeSize\n              ? await resizeBase64ImageToJpeg(markedCropPayload, {\n                  sourceSize: cropSize,\n                  targetSize: resizeSize,\n                })\n              : markedCropPayload,\n          };\n        }),\n      );\n      const contextImageContent = contextImages.flatMap<MessageContent>(\n        (item, index) => [\n          {\n            type: 'text',\n            text: `Image ${index + 2}: focused detail crop around the target, for reading text, icon shape, and exact local boundaries.`,\n          },\n          {\n            type: 'image',\n            url: item.imageBase64,\n          },\n        ],\n      );\n\n      imageContent = [\n        {\n          type: 'text' as const,\n          text: 'Use these images together to describe the real UI target marked by the temporary callout. Do not describe the marker itself.',\n        },\n        {\n          type: 'text' as const,\n          text: 'Image 1: full screenshot overview with the target marker, for page position and ownership context.',\n        },\n        {\n          type: 'image' as const,\n          url: imagePayload,\n        },\n        ...contextImageContent,\n      ];\n    } else {\n      imageContent = [\n        {\n          type: 'text' as const,\n          text: 'Full screenshot with a temporary callout marking the target:',\n        },\n        {\n          type: 'image' as const,\n          url: imagePayload,\n        },\n      ];\n    }\n\n    const msgs: ModelCallMessages = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: imageContent,\n      },\n    ];\n\n    let res: Awaited<\n      ReturnType<typeof callAIWithObjectResponse<AIDescribeElementResponse>>\n    >;\n    try {\n      res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n        msgs,\n        modelRuntime,\n      );\n    } catch (error) {\n      const recoveredResponse = recoverDescribeResponseFromParseError(error);\n      if (!recoveredResponse) {\n        throw error;\n      }\n      debug('describe: recovered malformed description JSON response');\n      return recoveredResponse;\n    }\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Service","query","opt","modelRuntime","abortSignal","modelConfig","queryPrompt","assert","Error","modelFamilyRequiredForLocateMessage","context","searchAreaStartTime","Date","searchArea","errorMessage","taskInfo","dump","createServiceDump","ServiceError","startTime","parseResult","rawResponse","rawAssistantOutput","usage","reasoning_content","AiLocateElement","timeCost","errorLog","dumpData","element","options","adapter","hasPlanLocatedElement","config","buildSearchAreaConfig","mergeSearchAreaResults","JSON","searchAreaResponse","AiLocateSection","searchAreaConfig","firstPassLocateResult","dataDemand","pageDescription","multimodalPrompt","executionOptions","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetFromPoint","Array","targetRect","Math","targetPoint","undefined","usePointMarker","imagePayload","compositePointMarkerImg","compositeElementInfoImg","getDescribeMarkerRect","getDescribeMarkerBorderThickness","shouldDeepDescribe","imageContent","contextAreas","getDescribeDeepContextAreas","contextImages","Promise","area","croppedResult","cropByRect","cropSize","targetInCrop","getRectInCrop","markedCropPayload","resizeSize","getDescribeDeepLocateResizeSize","resizeBase64ImageToJpeg","contextImageContent","item","index","msgs","res","callAIWithObjectResponse","recoveredResponse","recoverDescribeResponseFromParseError","content"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA4EA,MAAMA,QAAQC,SAAS;AAER,MAAMC;IAuBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,YAA0B,EAC1BC,WAAyB,EACM;QAC/B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGF;QAChC,MAAMG,cAAcL,MAAM,MAAM;QAChCM,OAAOD,aAAa;QAEpB,IAAI,CAACD,YAAY,WAAW,EAC1B,MAAM,IAAIG,MAAMC,oCAAoCJ,YAAY,IAAI;QAGtE,MAAMK,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,MAAMS,sBAAsBC,KAAK,GAAG;QACpC,MAAMC,aAAa,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACpDZ;YACAK;YACAJ;YACAQ;YACAP;YACAC;QACF;QAEA,IAAI,CAACS,WAAW,MAAM,IAAIA,WAAW,KAAK,EAAE;YAC1C,MAAMC,eAAe,CAAC,6BAA6B,EAAER,YAAY,GAAG,EAAEO,WAAW,KAAK,EAAE;YACxF,MAAME,WAA4B;gBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACtC,YAAYH,KAAK,GAAG,KAAKD;gBACzB,uBAAuBE,WAAW,KAAK,CAAC,WAAW;gBACnD,4BAA4BA,WAAW,KAAK,CAAC,kBAAkB;gBAC/D,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACzC;YACA,MAAMG,OAAOC,kBAAkB;gBAC7B,MAAM;gBACN,WAAW;oBAAE,SAASX;gBAAY;gBAClC,gBAAgB,EAAE;gBAClB,MAAM;gBACNS;gBACA,YAAY;gBACZ,OAAOD;YACT;YACA,MAAM,IAAII,aAAaJ,cAAcE;QACvC;QAEA,MAAMG,YAAYP,KAAK,GAAG;QAC1B,MAAM,EACJQ,WAAW,EACXC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,gBAAgB;YACxBf;YACA,0BAA0BJ;YAC1B,cAAcO,WAAW,MAAM;YAC/BV;YACAC;QACF;QAEA,MAAMsB,WAAWd,KAAK,GAAG,KAAKO;QAC9B,MAAMJ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYW;YACZL;YACAC;YACA,gBAAgBF;YAChBG;YACA,YAAYV,WAAW,KAAK,CAAC,UAAU;YACvC,uBAAuBA,WAAW,KAAK,CAAC,WAAW;YACnD,4BAA4BA,WAAW,KAAK,CAAC,kBAAkB;YAC/D,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACvCW;QACF;QAEA,IAAIG;QACJ,IAAIP,YAAY,MAAM,EAAE,QACtBO,WAAW,CAAC,4BAA4B,EAAEP,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMQ,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAStB;YACX;YACA,MAAM;YACNS;YACA,YAAY,CAAC,CAACF,WAAW,KAAK,CAAC,UAAU;YACzC,OAAOc;QACT;QAEA,MAAME,UAAUT,YAAY,OAAO;QAEnC,MAAMJ,OAAOC,kBAAkB;YAC7B,GAAGW,QAAQ;YACX,gBAAgBC,UAAU;gBAACA;aAAQ,GAAG,EAAE;QAC1C;QAEA,IAAIF,UACF,MAAM,IAAIT,aAAaS,UAAUX;QAGnC,IAAIa,SACF,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,MAAM;gBACtB,aAAaA,QAAQ,WAAW;gBAChC,GAAIA,QAAQ,IAAI,GAAG;oBAAE,MAAMA,QAAQ,IAAI;gBAAC,IAAI,CAAC,CAAC;YAChD;YACAb;QACF;QAGF,OAAO;YACL,SAAS;YACTA;QACF;IACF;IAEA,MAAc,wBAAwBc,OAOrC,EAAmC;QAClC,MAAM,EAAE7B,KAAK,EAAEK,WAAW,EAAEJ,GAAG,EAAEQ,OAAO,EAAEP,YAAY,EAAEC,WAAW,EAAE,GACnE0B;QACF,MAAM,EAAEC,OAAO,EAAE,GAAG5B;QACpB,MAAM6B,wBAAwB,CAAC,CAAC9B,KAAK,oBAAoB;QAEzD,IAAI,CAACD,MAAM,UAAU,EACnB,OAAO;YAAE,OAAO,CAAC;QAAE;QAMrB,IAAI+B,uBAAuB;YACzB,MAAMC,SAAS,MAAMC,sBAAsB;gBACzCxB;gBACA,UAAUyB,uBAAuBjC,IAAI,kBAAkB;YACzD;YAEA,OAAO;gBACL+B;gBACA,OAAO;oBACL,YAAYA,OAAO,UAAU;oBAC7B,aAAaG,KAAK,SAAS,CAAC;wBAC1B,QAAQ;wBACR,QAAQlC,IAAI,kBAAkB,CAAE,MAAM;wBACtC,MAAMA,IAAI,kBAAkB,CAAE,IAAI;oBACpC;gBACF;YACF;QACF;QAEA,IAAI6B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,IAAmBA,QAAQ,MAAM,CAAC,UAAU,EAAE;YACnE,MAAMM,qBAAqB,MAAMC,gBAAgB;gBAC/C5B;gBACA,oBAAoBJ;gBACpBH;gBACAC;YACF;YACA,MAAM,EAAEmC,gBAAgB,EAAE,GAAGF;YAC7B,IAAI,CAACE,kBACH,OAAO;gBACL,OAAOF,mBAAmB,KAAK,IAAI;gBACnC,OAAO;oBACL,aAAaA,mBAAmB,WAAW;oBAC3C,oBAAoBA,mBAAmB,kBAAkB;oBACzD,OAAOA,mBAAmB,KAAK;gBACjC;YACF;YAGF,OAAO;gBACL,QAAQE;gBACR,OAAO;oBACL,YAAYA,iBAAiB,UAAU;oBACvC,aAAaF,mBAAmB,WAAW;oBAC3C,oBAAoBA,mBAAmB,kBAAkB;oBACzD,OAAOA,mBAAmB,KAAK;gBACjC;YACF;QACF;QAEA,MAAMG,wBAAwB,MAAMf,gBAAgB;YAClDf;YACA,0BAA0BJ;YAC1BH;YACAC;QACF;QACAG,OACEiC,sBAAsB,WAAW,CAAC,OAAO,EAAE,QAC3C,CAAC,6BAA6B,EAAElC,YAAY,CAAC,EAC3CkC,sBAAsB,WAAW,CAAC,MAAM,EAAE,SACtC,CAAC,EAAE,EAAEA,sBAAsB,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,GAC1D,IACJ;QAGJ,MAAMP,SAAS,MAAMC,sBAAsB;YACzCxB;YACA,UAAUyB,uBACRK,sBAAsB,WAAW,CAAC,OAAO;QAE7C;QAEA,OAAO;YACLP;YACA,OAAO;gBACL,YAAYA,OAAO,UAAU;gBAC7B,aAAaG,KAAK,SAAS,CAAC;oBAC1B,QAAQ;oBACR,QAAQI,sBAAsB,WAAW,CAAC,OAAO,EAAE;oBACnD,MAAMA,sBAAsB,WAAW,CAAC,OAAO,EAAE;oBACjD,aAAaA,sBAAsB,WAAW;gBAChD;gBACA,oBAAoBA,sBAAsB,kBAAkB;gBAC5D,OAAOA,sBAAsB,KAAK;YACpC;QACF;IACF;IAEA,MAAM,QACJC,UAA+B,EAC/BtC,YAA0B,EAC1BD,GAA0B,EAC1BwC,eAAwB,EACxBC,gBAAoC,EACpCjC,OAAmB,EACnBkC,gBAEC,EACiC;QAClCrC,OAAOG,SAAS;QAChBH,OACE,AAAsB,YAAtB,OAAOkC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMtB,YAAYP,KAAK,GAAG;QAE1B,IAAIQ;QAGJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMqB,SAAS,MAAMC,qBAAwB;gBAC3CpC;gBACA,WAAW+B;gBACXE;gBACA,eAAezC;gBACfC;gBACAuC;gBACA,aAAaE,kBAAkB;YACjC;YACAxB,cAAcyB,OAAO,WAAW;YAChCxB,cAAcwB,OAAO,WAAW;YAChCvB,qBAAqBuB,OAAO,kBAAkB;YAC9CtB,QAAQsB,OAAO,KAAK;YACpBrB,oBAAoBqB,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,sBAAsB;gBAEzC,MAAMtB,WAAWd,KAAK,GAAG,KAAKO;gBAC9B,MAAMJ,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYW;oBACZ,aAAaqB,MAAM,WAAW;oBAC9B,oBAAoBA,MAAM,kBAAkB;oBAC5C,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAM/B,OAAOC,kBAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEwB;oBAAW;oBACxB,MAAM;oBACN1B;oBACA,OAAOgC,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAI7B,aAAa6B,MAAM,OAAO,EAAE/B;YACxC;YACA,MAAM+B;QACR;QAEA,MAAMrB,WAAWd,KAAK,GAAG,KAAKO;QAC9B,MAAMJ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYW;YACZL;YACAC;YACA,gBAAgBF;YAChBG;YACAC;QACF;QAEA,IAAIG;QACJ,IAAIP,YAAY,MAAM,EAAE,QACtBO,WAAW,CAAC,qBAAqB,EAAEP,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMQ,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTa;YACF;YACA,MAAM;YACN1B;YACA,OAAOY;QACT;QAEA,MAAM,EAAEsB,IAAI,EAAEC,OAAO,EAAE,GAAG9B,eAAe,CAAC;QAE1C,MAAMJ,OAAOC,kBAAkB;YAC7B,GAAGW,QAAQ;YACXqB;QACF;QAEA,IAAItB,YAAY,CAACsB,MACf,MAAM,IAAI/B,aAAaS,UAAUX;QAGnC,OAAO;YACLiC;YACAC;YACA3B;YACAC;YACAR;QACF;IACF;IAEA,MAAM,SACJmC,MAA+B,EAC/BhD,YAA0B,EAC1BD,GAGC,EACwD;QACzDK,OAAO4C,QAAQ;QACf,MAAMzC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAC9D,MAAM,EAAEkD,QAAQ,EAAE,GAAG1C;QACrB,MAAM2C,mBAAmB3C,QAAQ,UAAU,CAAC,MAAM;QAClDH,OAAO8C,kBAAkB;QACzB,MAAMC,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,kBAAkBC,MAAM,OAAO,CAACP;QACtC,MAAMQ,aAAmBF,kBACrB;YACE,MAAMG,KAAK,KAAK,CAACT,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKI,KAAK,KAAK,CAACT,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QACJ,MAAMU,cAAcJ,kBAChB;YACE,GAAGN,MAAM,CAAC,EAAE;YACZ,GAAGA,MAAM,CAAC,EAAE;QACd,IACAW;QAEJ,MAAMC,iBAAiBN;QACvB,MAAMO,eAAeD,iBACjB,MAAME,wBAAwB;YAC5B,gBAAgBZ;YAChB,MAAMD;YACN,OAAOS;QACT,KACA,MAAMK,wBAAwB;YAC5B,gBAAgBb;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMe,sBAAsBR;gBAC9B;aACD;YACD,iBAAiBS,iCAAiCT;YAClD,aAAa;QACf;QAEJ,MAAMU,qBAAqBnE,KAAK;QAChC,IAAIoE;QACJ,IAAID,oBAAoB;YACtB,MAAME,eAAeC,4BAA4Bb,YAAYP;YAC7D,MAAMqB,gBAAgB,MAAMC,QAAQ,GAAG,CACrCH,aAAa,GAAG,CAAC,OAAOI;gBACtB7E,MAAM,wCAAwC6E;gBAC9C,MAAMC,gBAAgB,MAAMC,WAAWxB,kBAAkBsB,KAAK,IAAI;gBAClE,MAAMG,WAAW;oBACf,OAAOF,cAAc,KAAK;oBAC1B,QAAQA,cAAc,MAAM;gBAC9B;gBACA,MAAMG,eAAeC,cAAcrB,YAAYgB,KAAK,IAAI,EAAEG;gBAC1D,MAAMG,oBAAoBxB,kBACtB,MAAMQ,wBAAwB;oBAC5B,gBAAgBW,cAAc,WAAW;oBACzC,MAAME;oBACN,OAAO;wBACL,GAAGjB,YAAa,CAAC,GAAGc,KAAK,IAAI,CAAC,IAAI;wBAClC,GAAGd,YAAa,CAAC,GAAGc,KAAK,IAAI,CAAC,GAAG;oBACnC;gBACF,KACA,MAAMT,wBAAwB;oBAC5B,gBAAgBU,cAAc,WAAW;oBACzC,MAAME;oBACN,sBAAsB;wBACpB;4BACE,MAAMX,sBAAsBY;wBAC9B;qBACD;oBACD,iBAAiBX,iCAAiCW;oBAClD,aAAa;gBACf;gBACJ,MAAMG,aAAaC,gCAAgCP;gBACnD,OAAO;oBACL,MAAMD,KAAK,IAAI;oBACf,aAAaO,aACT,MAAME,wBAAwBH,mBAAmB;wBAC/C,YAAYH;wBACZ,YAAYI;oBACd,KACAD;gBACN;YACF;YAEF,MAAMI,sBAAsBZ,cAAc,OAAO,CAC/C,CAACa,MAAMC,QAAU;oBACf;wBACE,MAAM;wBACN,MAAM,CAAC,MAAM,EAAEA,QAAQ,EAAE,kGAAkG,CAAC;oBAC9H;oBACA;wBACE,MAAM;wBACN,KAAKD,KAAK,WAAW;oBACvB;iBACD;YAGHhB,eAAe;gBACb;oBACE,MAAM;oBACN,MAAM;gBACR;gBACA;oBACE,MAAM;oBACN,MAAM;gBACR;gBACA;oBACE,MAAM;oBACN,KAAKN;gBACP;mBACGqB;aACJ;QACH,OACEf,eAAe;YACb;gBACE,MAAM;gBACN,MAAM;YACR;YACA;gBACE,MAAM;gBACN,KAAKN;YACP;SACD;QAGH,MAAMwB,OAA0B;YAC9B;gBAAE,MAAM;gBAAU,SAASlC;YAAa;YACxC;gBACE,MAAM;gBACN,SAASgB;YACX;SACD;QAED,IAAImB;QAGJ,IAAI;YACFA,MAAM,MAAMC,yBACVF,MACArF;QAEJ,EAAE,OAAO4C,OAAO;YACd,MAAM4C,oBAAoBC,sCAAsC7C;YAChE,IAAI,CAAC4C,mBACH,MAAM5C;YAERjD,MAAM;YACN,OAAO6F;QACT;QAEA,MAAM,EAAEE,OAAO,EAAE,GAAGJ;QACpBlF,OAAO,CAACsF,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DtF,OAAOsF,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IA5gBA,YACEnF,OAE8D,EAC9DR,GAAoB,CACpB;QATF;QAEA;QAQEK,OAAOG,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMgE,QAAQ,OAAO,CAAChE;QAGlD,IAAI,AAAyB,WAAlBR,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AA6fF"}
+{"version":3,"file":"service/index.mjs","sources":["../../../src/service/index.ts"],"sourcesContent":["import type { ModelRuntime } from '@/ai-model/models';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport {\n  AIResponseParseError,\n  callAIWithObjectResponse,\n} from '@/ai-model/service-caller';\nimport type { ModelCallMessages } from '@/ai-model/service-caller/types';\nimport type { MessageContent } from '@/ai-model/service-caller/types';\nimport { modelFamilyRequiredForLocateMessage } from '@/ai-model/shared/model-locate-result/errors';\nimport {\n  AiLocateElement,\n  AiLocateSection,\n  buildSearchAreaConfig,\n} from '@/ai-model/workflows/grounding';\nimport { mergeSearchAreaResults } from '@/ai-model/workflows/grounding/search-area';\nimport type { SearchAreaConfig } from '@/ai-model/workflows/grounding/types';\nimport { AiExtractElementInfo } from '@/ai-model/workflows/insight';\nimport type { RawAssistantOutput } from '@/types';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport {\n  compositeElementInfoImg,\n  compositePointMarkerImg,\n  cropByRect,\n  resizeBase64ImageToJpeg,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  createServiceDump,\n  getDescribeDeepContextAreas,\n  getDescribeDeepLocateResizeSize,\n  getDescribeMarkerBorderThickness,\n  getDescribeMarkerRect,\n  getRectInCrop,\n  recoverDescribeResponseFromParseError,\n} from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\ninterface LocateSearchAreaResult {\n  config?: SearchAreaConfig;\n  error?: string;\n  trace: {\n    sourceRect?: Rect;\n    rawResponse?: string;\n    rawAssistantOutput?: RawAssistantOutput;\n    usage?: AIUsageInfo;\n  };\n}\n\nconst debug = getDebug('ai:service');\n\nexport default class Service {\n  contextRetrieverFn: (signal?: AbortSignal) => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | UIContext\n      | ((signal?: AbortSignal) => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelRuntime: ModelRuntime,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const { config: modelConfig } = modelRuntime;\n    const queryPrompt = query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    if (!modelConfig.modelFamily) {\n      throw new Error(modelFamilyRequiredForLocateMessage(modelConfig.slot));\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    const searchAreaStartTime = Date.now();\n    const searchArea = await this.resolveLocateSearchArea({\n      query,\n      queryPrompt,\n      opt,\n      context,\n      modelRuntime,\n      abortSignal,\n    });\n\n    if (!searchArea.config && searchArea.error) {\n      const errorMessage = `cannot find search area for \"${queryPrompt}\": ${searchArea.error}`;\n      const taskInfo: ServiceTaskInfo = {\n        ...(this.taskInfo ? this.taskInfo : {}),\n        durationMs: Date.now() - searchAreaStartTime,\n        searchAreaRawResponse: searchArea.trace.rawResponse,\n        searchAreaRawChoiceMessage: searchArea.trace.rawAssistantOutput,\n        searchAreaUsage: searchArea.trace.usage,\n      };\n      const dump = createServiceDump({\n        type: 'locate',\n        userQuery: { element: queryPrompt },\n        matchedElement: [],\n        data: null,\n        taskInfo,\n        deepLocate: true,\n        error: errorMessage,\n      });\n      throw new ServiceError(errorMessage, dump);\n    }\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rawResponse,\n      rawAssistantOutput,\n      usage,\n      reasoning_content,\n    } = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchArea.config,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawAssistantOutput,\n      formatResponse: parseResult,\n      usage,\n      searchArea: searchArea.trace.sourceRect,\n      searchAreaRawResponse: searchArea.trace.rawResponse,\n      searchAreaRawChoiceMessage: searchArea.trace.rawAssistantOutput,\n      searchAreaUsage: searchArea.trace.usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea.trace.sourceRect,\n      error: errorLog,\n    };\n\n    const element = parseResult.element;\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: element ? [element] : [],\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (element) {\n      return {\n        element: {\n          center: element.center,\n          description: element.description,\n          ...(element.rect ? { rect: element.rect } : {}),\n        },\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      dump,\n    };\n  }\n\n  private async resolveLocateSearchArea(options: {\n    query: DetailedLocateParam;\n    queryPrompt: TUserPrompt;\n    opt: LocateOpts;\n    context: UIContext;\n    modelRuntime: ModelRuntime;\n    abortSignal?: AbortSignal;\n  }): Promise<LocateSearchAreaResult> {\n    const { query, queryPrompt, opt, context, modelRuntime, abortSignal } =\n      options;\n    const { adapter } = modelRuntime;\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.center;\n\n    if (!query.deepLocate) {\n      return { trace: {} };\n    }\n\n    // TODO: Make the plan, section-locate, and element-locate fallback paths\n    // resolve only the source rect, then build the search-area config once in\n    // this orchestration layer instead of building it inside AiLocateSection.\n    if (hasPlanLocatedElement) {\n      const config = await buildSearchAreaConfig({\n        context,\n        baseRect: mergeSearchAreaResults(opt.planLocatedElement!),\n      });\n\n      return {\n        config,\n        trace: {\n          sourceRect: config.sourceRect,\n          rawResponse: JSON.stringify({\n            source: 'plan-located-element',\n            center: opt.planLocatedElement!.center,\n            rect: opt.planLocatedElement!.rect,\n          }),\n        },\n      };\n    }\n\n    if (adapter.locate.kind === 'standard' && adapter.locate.searchArea) {\n      const searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: queryPrompt,\n        modelRuntime,\n        abortSignal,\n      });\n      const { searchAreaConfig } = searchAreaResponse;\n      if (!searchAreaConfig) {\n        return {\n          error: searchAreaResponse.error || 'unknown search area error',\n          trace: {\n            rawResponse: searchAreaResponse.rawResponse,\n            rawAssistantOutput: searchAreaResponse.rawAssistantOutput,\n            usage: searchAreaResponse.usage,\n          },\n        };\n      }\n\n      return {\n        config: searchAreaConfig,\n        trace: {\n          sourceRect: searchAreaConfig.sourceRect,\n          rawResponse: searchAreaResponse.rawResponse,\n          rawAssistantOutput: searchAreaResponse.rawAssistantOutput,\n          usage: searchAreaResponse.usage,\n        },\n      };\n    }\n\n    const firstPassLocateResult = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      modelRuntime,\n      abortSignal,\n    });\n    assert(\n      firstPassLocateResult.parseResult.element?.center,\n      `cannot find search area for \"${queryPrompt}\"${\n        firstPassLocateResult.parseResult.errors?.length\n          ? `: ${firstPassLocateResult.parseResult.errors.join('\\n')}`\n          : ''\n      }`,\n    );\n\n    const config = await buildSearchAreaConfig({\n      context,\n      baseRect: mergeSearchAreaResults(\n        firstPassLocateResult.parseResult.element!,\n      ),\n    });\n\n    return {\n      config,\n      trace: {\n        sourceRect: config.sourceRect,\n        rawResponse: JSON.stringify({\n          source: 'deep-locate-first-pass',\n          center: firstPassLocateResult.parseResult.element?.center,\n          rect: firstPassLocateResult.parseResult.element?.rect,\n          rawResponse: firstPassLocateResult.rawResponse,\n        }),\n        rawAssistantOutput: firstPassLocateResult.rawAssistantOutput,\n        usage: firstPassLocateResult.usage,\n      },\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    taskType: 'boolean' | 'extract',\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let rawAssistantOutput: RawAssistantOutput | undefined;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        taskType,\n        multimodalPrompt,\n        extractOption: opt,\n        modelRuntime,\n        pageDescription,\n        abortSignal: executionOptions?.abortSignal,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      rawAssistantOutput = result.rawAssistantOutput;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          rawAssistantOutput: error.rawAssistantOutput,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawAssistantOutput,\n      formatResponse: parseResult,\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelRuntime: ModelRuntime,\n    opt?: {\n      deepDescribe?: boolean;\n      context?: UIContext;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = opt?.context || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetFromPoint = Array.isArray(target);\n    const targetRect: Rect = targetFromPoint\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n    const targetPoint = targetFromPoint\n      ? {\n          x: target[0],\n          y: target[1],\n        }\n      : undefined;\n\n    const usePointMarker = targetFromPoint;\n    const imagePayload = usePointMarker\n      ? await compositePointMarkerImg({\n          inputImgBase64: screenshotBase64,\n          size: shotSize,\n          point: targetPoint!,\n        })\n      : await compositeElementInfoImg({\n          inputImgBase64: screenshotBase64,\n          size: shotSize,\n          elementsPositionInfo: [\n            {\n              rect: getDescribeMarkerRect(targetRect),\n            },\n          ],\n          borderThickness: getDescribeMarkerBorderThickness(targetRect),\n          centerPoint: true,\n        });\n\n    const shouldDeepDescribe = opt?.deepDescribe;\n    let imageContent: MessageContent[];\n    if (shouldDeepDescribe) {\n      const contextAreas = getDescribeDeepContextAreas(targetRect, shotSize);\n      const contextImages = await Promise.all(\n        contextAreas.map(async (area) => {\n          debug('describe: cropping deep context area', area);\n          const croppedResult = await cropByRect(screenshotBase64, area.rect);\n          const cropSize = {\n            width: croppedResult.width,\n            height: croppedResult.height,\n          };\n          const targetInCrop = getRectInCrop(targetRect, area.rect, cropSize);\n          const markedCropPayload = targetFromPoint\n            ? await compositePointMarkerImg({\n                inputImgBase64: croppedResult.imageBase64,\n                size: cropSize,\n                point: {\n                  x: targetPoint!.x - area.rect.left,\n                  y: targetPoint!.y - area.rect.top,\n                },\n              })\n            : await compositeElementInfoImg({\n                inputImgBase64: croppedResult.imageBase64,\n                size: cropSize,\n                elementsPositionInfo: [\n                  {\n                    rect: getDescribeMarkerRect(targetInCrop),\n                  },\n                ],\n                borderThickness: getDescribeMarkerBorderThickness(targetInCrop),\n                centerPoint: true,\n              });\n          const resizeSize = getDescribeDeepLocateResizeSize(croppedResult);\n          return {\n            kind: area.kind,\n            imageBase64: resizeSize\n              ? await resizeBase64ImageToJpeg(markedCropPayload, {\n                  sourceSize: cropSize,\n                  targetSize: resizeSize,\n                })\n              : markedCropPayload,\n          };\n        }),\n      );\n      const contextImageContent = contextImages.flatMap<MessageContent>(\n        (item, index) => [\n          {\n            type: 'text',\n            text: `Image ${index + 2}: focused detail crop around the target, for reading text, icon shape, and exact local boundaries.`,\n          },\n          {\n            type: 'image',\n            url: item.imageBase64,\n          },\n        ],\n      );\n\n      imageContent = [\n        {\n          type: 'text' as const,\n          text: 'Use these images together to describe the real UI target marked by the temporary callout. Do not describe the marker itself.',\n        },\n        {\n          type: 'text' as const,\n          text: 'Image 1: full screenshot overview with the target marker, for page position and ownership context.',\n        },\n        {\n          type: 'image' as const,\n          url: imagePayload,\n        },\n        ...contextImageContent,\n      ];\n    } else {\n      imageContent = [\n        {\n          type: 'text' as const,\n          text: 'Full screenshot with a temporary callout marking the target:',\n        },\n        {\n          type: 'image' as const,\n          url: imagePayload,\n        },\n      ];\n    }\n\n    const msgs: ModelCallMessages = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: imageContent,\n      },\n    ];\n\n    let res: Awaited<\n      ReturnType<typeof callAIWithObjectResponse<AIDescribeElementResponse>>\n    >;\n    try {\n      res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n        msgs,\n        modelRuntime,\n      );\n    } catch (error) {\n      const recoveredResponse = recoverDescribeResponseFromParseError(error);\n      if (!recoveredResponse) {\n        throw error;\n      }\n      debug('describe: recovered malformed description JSON response');\n      return recoveredResponse;\n    }\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Service","query","opt","modelRuntime","abortSignal","modelConfig","queryPrompt","assert","Error","modelFamilyRequiredForLocateMessage","context","searchAreaStartTime","Date","searchArea","errorMessage","taskInfo","dump","createServiceDump","ServiceError","startTime","parseResult","rawResponse","rawAssistantOutput","usage","reasoning_content","AiLocateElement","timeCost","errorLog","dumpData","element","options","adapter","hasPlanLocatedElement","config","buildSearchAreaConfig","mergeSearchAreaResults","JSON","searchAreaResponse","AiLocateSection","searchAreaConfig","firstPassLocateResult","dataDemand","taskType","pageDescription","multimodalPrompt","executionOptions","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetFromPoint","Array","targetRect","Math","targetPoint","undefined","usePointMarker","imagePayload","compositePointMarkerImg","compositeElementInfoImg","getDescribeMarkerRect","getDescribeMarkerBorderThickness","shouldDeepDescribe","imageContent","contextAreas","getDescribeDeepContextAreas","contextImages","Promise","area","croppedResult","cropByRect","cropSize","targetInCrop","getRectInCrop","markedCropPayload","resizeSize","getDescribeDeepLocateResizeSize","resizeBase64ImageToJpeg","contextImageContent","item","index","msgs","res","callAIWithObjectResponse","recoveredResponse","recoverDescribeResponseFromParseError","content"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA4EA,MAAMA,QAAQC,SAAS;AAER,MAAMC;IAuBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,YAA0B,EAC1BC,WAAyB,EACM;QAC/B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGF;QAChC,MAAMG,cAAcL,MAAM,MAAM;QAChCM,OAAOD,aAAa;QAEpB,IAAI,CAACD,YAAY,WAAW,EAC1B,MAAM,IAAIG,MAAMC,oCAAoCJ,YAAY,IAAI;QAGtE,MAAMK,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,MAAMS,sBAAsBC,KAAK,GAAG;QACpC,MAAMC,aAAa,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACpDZ;YACAK;YACAJ;YACAQ;YACAP;YACAC;QACF;QAEA,IAAI,CAACS,WAAW,MAAM,IAAIA,WAAW,KAAK,EAAE;YAC1C,MAAMC,eAAe,CAAC,6BAA6B,EAAER,YAAY,GAAG,EAAEO,WAAW,KAAK,EAAE;YACxF,MAAME,WAA4B;gBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACtC,YAAYH,KAAK,GAAG,KAAKD;gBACzB,uBAAuBE,WAAW,KAAK,CAAC,WAAW;gBACnD,4BAA4BA,WAAW,KAAK,CAAC,kBAAkB;gBAC/D,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACzC;YACA,MAAMG,OAAOC,kBAAkB;gBAC7B,MAAM;gBACN,WAAW;oBAAE,SAASX;gBAAY;gBAClC,gBAAgB,EAAE;gBAClB,MAAM;gBACNS;gBACA,YAAY;gBACZ,OAAOD;YACT;YACA,MAAM,IAAII,aAAaJ,cAAcE;QACvC;QAEA,MAAMG,YAAYP,KAAK,GAAG;QAC1B,MAAM,EACJQ,WAAW,EACXC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,gBAAgB;YACxBf;YACA,0BAA0BJ;YAC1B,cAAcO,WAAW,MAAM;YAC/BV;YACAC;QACF;QAEA,MAAMsB,WAAWd,KAAK,GAAG,KAAKO;QAC9B,MAAMJ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYW;YACZL;YACAC;YACA,gBAAgBF;YAChBG;YACA,YAAYV,WAAW,KAAK,CAAC,UAAU;YACvC,uBAAuBA,WAAW,KAAK,CAAC,WAAW;YACnD,4BAA4BA,WAAW,KAAK,CAAC,kBAAkB;YAC/D,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACvCW;QACF;QAEA,IAAIG;QACJ,IAAIP,YAAY,MAAM,EAAE,QACtBO,WAAW,CAAC,4BAA4B,EAAEP,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMQ,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAStB;YACX;YACA,MAAM;YACNS;YACA,YAAY,CAAC,CAACF,WAAW,KAAK,CAAC,UAAU;YACzC,OAAOc;QACT;QAEA,MAAME,UAAUT,YAAY,OAAO;QAEnC,MAAMJ,OAAOC,kBAAkB;YAC7B,GAAGW,QAAQ;YACX,gBAAgBC,UAAU;gBAACA;aAAQ,GAAG,EAAE;QAC1C;QAEA,IAAIF,UACF,MAAM,IAAIT,aAAaS,UAAUX;QAGnC,IAAIa,SACF,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,MAAM;gBACtB,aAAaA,QAAQ,WAAW;gBAChC,GAAIA,QAAQ,IAAI,GAAG;oBAAE,MAAMA,QAAQ,IAAI;gBAAC,IAAI,CAAC,CAAC;YAChD;YACAb;QACF;QAGF,OAAO;YACL,SAAS;YACTA;QACF;IACF;IAEA,MAAc,wBAAwBc,OAOrC,EAAmC;QAClC,MAAM,EAAE7B,KAAK,EAAEK,WAAW,EAAEJ,GAAG,EAAEQ,OAAO,EAAEP,YAAY,EAAEC,WAAW,EAAE,GACnE0B;QACF,MAAM,EAAEC,OAAO,EAAE,GAAG5B;QACpB,MAAM6B,wBAAwB,CAAC,CAAC9B,KAAK,oBAAoB;QAEzD,IAAI,CAACD,MAAM,UAAU,EACnB,OAAO;YAAE,OAAO,CAAC;QAAE;QAMrB,IAAI+B,uBAAuB;YACzB,MAAMC,SAAS,MAAMC,sBAAsB;gBACzCxB;gBACA,UAAUyB,uBAAuBjC,IAAI,kBAAkB;YACzD;YAEA,OAAO;gBACL+B;gBACA,OAAO;oBACL,YAAYA,OAAO,UAAU;oBAC7B,aAAaG,KAAK,SAAS,CAAC;wBAC1B,QAAQ;wBACR,QAAQlC,IAAI,kBAAkB,CAAE,MAAM;wBACtC,MAAMA,IAAI,kBAAkB,CAAE,IAAI;oBACpC;gBACF;YACF;QACF;QAEA,IAAI6B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,IAAmBA,QAAQ,MAAM,CAAC,UAAU,EAAE;YACnE,MAAMM,qBAAqB,MAAMC,gBAAgB;gBAC/C5B;gBACA,oBAAoBJ;gBACpBH;gBACAC;YACF;YACA,MAAM,EAAEmC,gBAAgB,EAAE,GAAGF;YAC7B,IAAI,CAACE,kBACH,OAAO;gBACL,OAAOF,mBAAmB,KAAK,IAAI;gBACnC,OAAO;oBACL,aAAaA,mBAAmB,WAAW;oBAC3C,oBAAoBA,mBAAmB,kBAAkB;oBACzD,OAAOA,mBAAmB,KAAK;gBACjC;YACF;YAGF,OAAO;gBACL,QAAQE;gBACR,OAAO;oBACL,YAAYA,iBAAiB,UAAU;oBACvC,aAAaF,mBAAmB,WAAW;oBAC3C,oBAAoBA,mBAAmB,kBAAkB;oBACzD,OAAOA,mBAAmB,KAAK;gBACjC;YACF;QACF;QAEA,MAAMG,wBAAwB,MAAMf,gBAAgB;YAClDf;YACA,0BAA0BJ;YAC1BH;YACAC;QACF;QACAG,OACEiC,sBAAsB,WAAW,CAAC,OAAO,EAAE,QAC3C,CAAC,6BAA6B,EAAElC,YAAY,CAAC,EAC3CkC,sBAAsB,WAAW,CAAC,MAAM,EAAE,SACtC,CAAC,EAAE,EAAEA,sBAAsB,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,GAC1D,IACJ;QAGJ,MAAMP,SAAS,MAAMC,sBAAsB;YACzCxB;YACA,UAAUyB,uBACRK,sBAAsB,WAAW,CAAC,OAAO;QAE7C;QAEA,OAAO;YACLP;YACA,OAAO;gBACL,YAAYA,OAAO,UAAU;gBAC7B,aAAaG,KAAK,SAAS,CAAC;oBAC1B,QAAQ;oBACR,QAAQI,sBAAsB,WAAW,CAAC,OAAO,EAAE;oBACnD,MAAMA,sBAAsB,WAAW,CAAC,OAAO,EAAE;oBACjD,aAAaA,sBAAsB,WAAW;gBAChD;gBACA,oBAAoBA,sBAAsB,kBAAkB;gBAC5D,OAAOA,sBAAsB,KAAK;YACpC;QACF;IACF;IAEA,MAAM,QACJC,UAA+B,EAC/BtC,YAA0B,EAC1BuC,QAA+B,EAC/BxC,GAA0B,EAC1ByC,eAAwB,EACxBC,gBAAoC,EACpClC,OAAmB,EACnBmC,gBAEC,EACiC;QAClCtC,OAAOG,SAAS;QAChBH,OACE,AAAsB,YAAtB,OAAOkC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMtB,YAAYP,KAAK,GAAG;QAE1B,IAAIQ;QAGJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMsB,SAAS,MAAMC,qBAAwB;gBAC3CrC;gBACA,WAAW+B;gBACXC;gBACAE;gBACA,eAAe1C;gBACfC;gBACAwC;gBACA,aAAaE,kBAAkB;YACjC;YACAzB,cAAc0B,OAAO,WAAW;YAChCzB,cAAcyB,OAAO,WAAW;YAChCxB,qBAAqBwB,OAAO,kBAAkB;YAC9CvB,QAAQuB,OAAO,KAAK;YACpBtB,oBAAoBsB,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,sBAAsB;gBAEzC,MAAMvB,WAAWd,KAAK,GAAG,KAAKO;gBAC9B,MAAMJ,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYW;oBACZ,aAAasB,MAAM,WAAW;oBAC9B,oBAAoBA,MAAM,kBAAkB;oBAC5C,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMhC,OAAOC,kBAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEwB;oBAAW;oBACxB,MAAM;oBACN1B;oBACA,OAAOiC,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAI9B,aAAa8B,MAAM,OAAO,EAAEhC;YACxC;YACA,MAAMgC;QACR;QAEA,MAAMtB,WAAWd,KAAK,GAAG,KAAKO;QAC9B,MAAMJ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYW;YACZL;YACAC;YACA,gBAAgBF;YAChBG;YACAC;QACF;QAEA,IAAIG;QACJ,IAAIP,YAAY,MAAM,EAAE,QACtBO,WAAW,CAAC,qBAAqB,EAAEP,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMQ,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTa;YACF;YACA,MAAM;YACN1B;YACA,OAAOY;QACT;QAEA,MAAM,EAAEuB,IAAI,EAAEC,OAAO,EAAE,GAAG/B,eAAe,CAAC;QAE1C,MAAMJ,OAAOC,kBAAkB;YAC7B,GAAGW,QAAQ;YACXsB;QACF;QAEA,IAAIvB,YAAY,CAACuB,MACf,MAAM,IAAIhC,aAAaS,UAAUX;QAGnC,OAAO;YACLkC;YACAC;YACA5B;YACAC;YACAR;QACF;IACF;IAEA,MAAM,SACJoC,MAA+B,EAC/BjD,YAA0B,EAC1BD,GAGC,EACwD;QACzDK,OAAO6C,QAAQ;QACf,MAAM1C,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAC9D,MAAM,EAAEmD,QAAQ,EAAE,GAAG3C;QACrB,MAAM4C,mBAAmB5C,QAAQ,UAAU,CAAC,MAAM;QAClDH,OAAO+C,kBAAkB;QACzB,MAAMC,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,kBAAkBC,MAAM,OAAO,CAACP;QACtC,MAAMQ,aAAmBF,kBACrB;YACE,MAAMG,KAAK,KAAK,CAACT,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKI,KAAK,KAAK,CAACT,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QACJ,MAAMU,cAAcJ,kBAChB;YACE,GAAGN,MAAM,CAAC,EAAE;YACZ,GAAGA,MAAM,CAAC,EAAE;QACd,IACAW;QAEJ,MAAMC,iBAAiBN;QACvB,MAAMO,eAAeD,iBACjB,MAAME,wBAAwB;YAC5B,gBAAgBZ;YAChB,MAAMD;YACN,OAAOS;QACT,KACA,MAAMK,wBAAwB;YAC5B,gBAAgBb;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMe,sBAAsBR;gBAC9B;aACD;YACD,iBAAiBS,iCAAiCT;YAClD,aAAa;QACf;QAEJ,MAAMU,qBAAqBpE,KAAK;QAChC,IAAIqE;QACJ,IAAID,oBAAoB;YACtB,MAAME,eAAeC,4BAA4Bb,YAAYP;YAC7D,MAAMqB,gBAAgB,MAAMC,QAAQ,GAAG,CACrCH,aAAa,GAAG,CAAC,OAAOI;gBACtB9E,MAAM,wCAAwC8E;gBAC9C,MAAMC,gBAAgB,MAAMC,WAAWxB,kBAAkBsB,KAAK,IAAI;gBAClE,MAAMG,WAAW;oBACf,OAAOF,cAAc,KAAK;oBAC1B,QAAQA,cAAc,MAAM;gBAC9B;gBACA,MAAMG,eAAeC,cAAcrB,YAAYgB,KAAK,IAAI,EAAEG;gBAC1D,MAAMG,oBAAoBxB,kBACtB,MAAMQ,wBAAwB;oBAC5B,gBAAgBW,cAAc,WAAW;oBACzC,MAAME;oBACN,OAAO;wBACL,GAAGjB,YAAa,CAAC,GAAGc,KAAK,IAAI,CAAC,IAAI;wBAClC,GAAGd,YAAa,CAAC,GAAGc,KAAK,IAAI,CAAC,GAAG;oBACnC;gBACF,KACA,MAAMT,wBAAwB;oBAC5B,gBAAgBU,cAAc,WAAW;oBACzC,MAAME;oBACN,sBAAsB;wBACpB;4BACE,MAAMX,sBAAsBY;wBAC9B;qBACD;oBACD,iBAAiBX,iCAAiCW;oBAClD,aAAa;gBACf;gBACJ,MAAMG,aAAaC,gCAAgCP;gBACnD,OAAO;oBACL,MAAMD,KAAK,IAAI;oBACf,aAAaO,aACT,MAAME,wBAAwBH,mBAAmB;wBAC/C,YAAYH;wBACZ,YAAYI;oBACd,KACAD;gBACN;YACF;YAEF,MAAMI,sBAAsBZ,cAAc,OAAO,CAC/C,CAACa,MAAMC,QAAU;oBACf;wBACE,MAAM;wBACN,MAAM,CAAC,MAAM,EAAEA,QAAQ,EAAE,kGAAkG,CAAC;oBAC9H;oBACA;wBACE,MAAM;wBACN,KAAKD,KAAK,WAAW;oBACvB;iBACD;YAGHhB,eAAe;gBACb;oBACE,MAAM;oBACN,MAAM;gBACR;gBACA;oBACE,MAAM;oBACN,MAAM;gBACR;gBACA;oBACE,MAAM;oBACN,KAAKN;gBACP;mBACGqB;aACJ;QACH,OACEf,eAAe;YACb;gBACE,MAAM;gBACN,MAAM;YACR;YACA;gBACE,MAAM;gBACN,KAAKN;YACP;SACD;QAGH,MAAMwB,OAA0B;YAC9B;gBAAE,MAAM;gBAAU,SAASlC;YAAa;YACxC;gBACE,MAAM;gBACN,SAASgB;YACX;SACD;QAED,IAAImB;QAGJ,IAAI;YACFA,MAAM,MAAMC,yBACVF,MACAtF;QAEJ,EAAE,OAAO6C,OAAO;YACd,MAAM4C,oBAAoBC,sCAAsC7C;YAChE,IAAI,CAAC4C,mBACH,MAAM5C;YAERlD,MAAM;YACN,OAAO8F;QACT;QAEA,MAAM,EAAEE,OAAO,EAAE,GAAGJ;QACpBnF,OAAO,CAACuF,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DvF,OAAOuF,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IA9gBA,YACEpF,OAE8D,EAC9DR,GAAoB,CACpB;QATF;QAEA;QAQEK,OAAOG,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMiE,QAAQ,OAAO,CAACjE;QAGlD,IAAI,AAAyB,WAAlBR,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AA+fF"}

package/dist/es/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { AgentAIContexts } from '@midscene/shared/agent-tools/agent-context';\nimport type { SerializedError } from '@midscene/shared/agent-tools/error-formatter';\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type OpenAI from 'openai';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport type { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport type { SerializedError } from '@midscene/shared/agent-tools/error-formatter';\nexport type {\n  AgentAIContextKey,\n  AgentAIContexts,\n  AiApiName,\n} from '@midscene/shared/agent-tools/agent-context';\nexport * from './yaml';\n\nexport { ServiceError } from './errors';\nexport {\n  ExecutionDump,\n  ReportActionDump,\n  GroupedActionDump,\n} from './dump/report-action-dump';\n\nexport type RawAssistantOutput =\n  | {\n      type: 'chat-completion';\n      rawValue: OpenAI.Chat.Completions.ChatCompletionMessage;\n    }\n  | {\n      type: 'responses';\n      rawValue: OpenAI.Responses.ResponseOutputItem[];\n    };\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  /** Duration of the final successful request attempt, in milliseconds. */\n  time_cost: number | undefined;\n  /** Request duration including all attempts and retry waits, excluding preparation, in milliseconds. */\n  total_time_cost?: number;\n  /** Request retries performed within this call; excludes semantic parse retries. */\n  retry_count?: number;\n  /** Actual protocol used for the model call, resolved by Midscene. */\n  api_protocol?: 'chat-completion' | 'responses' | 'codex' | 'system-one';\n  /** Why a task uses the default model instead of its configured specialist. */\n  modelRouting?: {\n    requestedModel: string;\n    requestedSlot: string;\n    task: string;\n    reason: string;\n  };\n  /** Whether the model call used streaming, after resolving call options. */\n  stream?: boolean;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  /**\n   * Raw top-level `.model` value returned by the model service response.\n   */\n  response_model_name: string | undefined;\n  /**\n   * Semantic intent of the model call, such as default, planning, or insight.\n   */\n  intent: string | undefined;\n  /**\n   * Config slot where the model config was resolved from. For example, a\n   * planning call may use the default slot when no planning model is configured.\n   */\n  slot: string | undefined;\n  request_id: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type LocateResultPoint = [number, number];\nexport type Bbox = [number, number, number, number];\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\n\nexport interface AIElementLocateResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  error?: string;\n}\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  references_bbox?: LocateResultBbox[];\n  references_point?: LocateResultPoint[];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect?: Rect;\n  center: [number, number];\n  centerDistance?: number;\n  /** Whether the expected point is inside the located rect; absent for point-only results. */\n  includedInRect?: boolean;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepLocate: boolean;\n  deepDescribe: boolean;\n  verifyResult?: LocateValidatorResult;\n  success: boolean;\n  error?: string;\n  failureStage?: 'describe' | 'verify';\n}\n\n/**\n * context\n */\n\nexport interface UiNode {\n  type: string;\n  attrs: Record<string, string | undefined>;\n  bounds: Rect;\n  children: UiNode[];\n}\n\nexport interface UITreeSnapshot {\n  platform: 'android';\n  capturedAt: number;\n  root: UiNode;\n}\n\nexport abstract class UIContext {\n  /**\n   * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n   */\n  abstract screenshot: ScreenshotItem;\n\n  /**\n   * Optional sequence of screenshots captured over a short time window, in\n   * temporal order (earliest first, latest last). When present with more than\n   * one frame, extract/assert flows submit all frames to the model so it can\n   * observe transient UI (toasts, carousels, auto-hiding controls). The last\n   * frame is the same state as {@link screenshot}.\n   */\n  abstract screenshotSequence?: ScreenshotItem[];\n\n  /**\n   * screenshot size after shrinking\n   */\n  abstract shotSize: Size;\n\n  /**\n   * The ratio for converting shrunk screenshot coordinates to logical coordinates.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - User-defined screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n   * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n   */\n  abstract shrunkShotToLogicalRatio: number;\n\n  abstract _isFrozen?: boolean;\n\n  // @deprecated - backward compatibility for aiLocate\n  abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type AiActEffort = 'fast' | 'balance' | 'deepThink';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: unknown;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: unknown;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  /**\n   * Adapter-extracted content from the search-area model call. This is not the\n   * full provider response or choices[0].message.\n   */\n  searchAreaRawResponse?: string;\n  searchAreaRawChoiceMessage?: unknown;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport type ReportAttributes = Record<\n  string,\n  string | number | boolean | null | undefined\n>;\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: ReportAttributes;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement?: LocateResultElement[];\n  deepLocate?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n  /**\n   * Additional facts, decision rules, constraints, or output requirements for\n   * this assertion. It overrides `aiContexts.aiAssert` and\n   * `aiContexts.default`;\n   * `''` disables inherited user context for this call.\n   */\n  context?: string;\n  abortSignal?: AbortSignal;\n}\n\nexport interface AgentAssertResult {\n  pass: boolean;\n  thought?: string;\n  message?: string;\n}\n\nexport type QueryOptions = ServiceExtractOption;\nexport type AssertOptions = AgentAssertOpt & ServiceExtractOption;\n\n/** Options for fixed observations, which never read the live DOM. */\nexport type ObservationQueryOptions = Omit<QueryOptions, 'domIncluded'> & {\n  domIncluded?: never;\n};\n\n/** Assertion options for fixed observations, which never read the live DOM. */\nexport type ObservationAssertOptions = Omit<AssertOptions, 'domIncluded'> & {\n  domIncluded?: never;\n};\n\n/** Read-only AI operations supported by both live Agents and UI observations. */\nexport interface InsightAPI<\n  QueryOpt extends QueryOptions = QueryOptions,\n  AssertOpt extends AssertOptions = AssertOptions,\n> {\n  aiQuery<ReturnType = any>(\n    demand: ServiceExtractParam,\n    options?: QueryOpt,\n  ): Promise<ReturnType>;\n  aiBoolean(prompt: TUserPrompt, options?: QueryOpt): Promise<boolean>;\n  aiNumber(prompt: TUserPrompt, options?: QueryOpt): Promise<number>;\n  aiString(prompt: TUserPrompt, options?: QueryOpt): Promise<string>;\n  aiAsk(prompt: TUserPrompt, options?: QueryOpt): Promise<string>;\n  aiAssert(\n    assertion: TUserPrompt,\n    message?: string,\n    options?: AssertOpt,\n  ): Promise<AgentAssertResult | undefined>;\n}\n\n/**\n * planning\n *\n */\n\ntype PlanningActionBase = {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n};\n\nexport type PlanningAction<ParamType = undefined> = PlanningActionBase &\n  ([ParamType] extends [undefined] ? { param?: any } : { param: ParamType });\n\nexport type SubGoalStatus = 'pending' | 'running' | 'finished';\n\nexport interface SubGoal {\n  index: number;\n  status: SubGoalStatus;\n  description: string;\n  logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction | null;\n  thought?: string;\n  log: string;\n  memory?: string;\n  error?: string;\n  finalizeMessage?: string;\n  finalizeSuccess?: boolean;\n  updateSubGoals?: SubGoal[];\n  markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n  output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (\n    task: ExecutionTask,\n    signal?: AbortSignal,\n  ) => Promise<void> | void;\n}\n\n/**\n * Generic agent progress bus.\n *\n * Progress notifications are a single, generic stream the agent broadcasts as\n * it works. Each event is a thin envelope: a `scope` naming the producer, a\n * `phase` within that producer's lifecycle, a monotonic `sequence`, and a\n * structured, presentation-free `data` payload. The bus knows nothing about any\n * particular producer's payload - consumers narrow by `scope`. `aiAct` is the\n * first producer (\"pilot\") on this bus; others (queries, waits, ...) can be\n * added without touching the bus, the listener API, or the renderer core.\n */\nexport interface AgentProgressEvent<\n  TScope extends string = string,\n  TData = unknown,\n  TPhase extends string = string,\n> {\n  scope: TScope;\n  phase: TPhase;\n  sequence: number;\n  data: TData;\n}\n\nexport type AgentProgressListener<\n  TScope extends string = string,\n  TData = unknown,\n  TPhase extends string = string,\n> = (event: AgentProgressEvent<TScope, TData, TPhase>) => Promise<void> | void;\n\n// --- aiAct: the first producer on the generic progress bus ---\n\nexport type AiActProgressPhase =\n  | 'start'\n  | 'plan_thinking'\n  | 'plan_planned'\n  | 'plan_action'\n  | 'plan_failed'\n  | 'action_running'\n  | 'action_done'\n  | 'action_failed'\n  | 'complete'\n  | 'failed';\n\nexport interface AiActProgressAction {\n  name: string;\n  target?: string;\n  point?: [number, number];\n  bbox?: [number, number, number, number];\n  /**\n   * Structured, compacted summary of the action params for actions that are\n   * not described by a point/bbox (e.g. `Sleep` -> `{ timeMs }`). This is data,\n   * never a pre-formatted display string; consumers decide how to render it.\n   */\n  param?: unknown;\n}\n\n/**\n * Structured payload carried by aiAct progress events. The producer only\n * reports *what happened* as data: the action involved, the raw text the model\n * produced, timings and errors. It never assembles human-readable log lines or\n * truncates strings for display - that belongs to whichever layer consumes the\n * stream (e.g. the CLI verbose renderer).\n */\nexport interface AiActProgressData {\n  /** Original user instruction, present on the `start` phase. */\n  prompt?: string;\n  planIndex?: number;\n  planLimit?: number;\n  /** Latest screenshot, present on `plan_thinking`. */\n  screenshot?: ScreenshotItem;\n  /** Structured action descriptor, present on the `*action*` phases. */\n  action?: AiActProgressAction;\n  /**\n   * Raw, untruncated semantic text produced by the model. Consumers choose\n   * which field to surface and how to format/truncate it.\n   */\n  thought?: string;\n  log?: string;\n  output?: string;\n  /** Wall-clock cost of an action, present on `action_done`/`action_failed`. */\n  durationMs?: number;\n  error?: string;\n}\n\nexport const aiActProgressScope = 'aiAct';\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  description?: string;\n  timing?: string;\n}\n\nexport interface RecordToReportScreenshot {\n  /**\n   * PNG/JPEG data URI, or raw PNG base64 body.\n   */\n  base64: string;\n  description?: string;\n}\n\nexport interface RecordToReportOptions {\n  content?: string;\n  /**\n   * @deprecated Use `screenshots: [{ base64 }]` instead.\n   */\n  screenshotBase64?: string;\n  /**\n   * Custom screenshots to display under a single report entry.\n   */\n  screenshots?: RecordToReportScreenshot[];\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  /** Cancellation of this execution; adapters may forward it to cancellable I/O. */\n  signal?: AbortSignal;\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n  /** Custom readiness owns optional post-action settling, not input/gesture timing. */\n  skipDefaultWait?: boolean;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  /**\n   * Maximum planning feedback length inherited from the action definition.\n   * Set to 'unlimited' to disable truncation for this task.\n   */\n  planningFeedbackMaxLength?: number | 'unlimited';\n  executor: (\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    taskId: string;\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    /** Readiness strategy selected for this atomic action execution. */\n    actionReadiness?: 'default' | 'custom';\n    /**\n     * Optional feedback produced by a task for the next planning round.\n     * This is execution metadata, not part of the action return value.\n     */\n    planningFeedback?: string;\n    /**\n     * A bounded diagnostic DTO created when the task executor throws. Arbitrary\n     * upstream payloads are intentionally omitted; use this field for structured\n     * diagnostics or errorMessage/errorStack for the common display path.\n     */\n    error?: SerializedError;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      getUiContextStart?: number;\n      getUiContextEnd?: number;\n      callAiStart?: number;\n      callAiEnd?: number;\n      beforeInvokeActionHookStart?: number;\n      beforeInvokeActionHookEnd?: number;\n      callActionStart?: number;\n      callActionEnd?: number;\n      afterInvokeActionHookStart?: number;\n      afterInvokeActionHookEnd?: number;\n      waitForActionReadyStart?: number;\n      waitForActionReadyEnd?: number;\n      captureAfterCallingSnapshotStart?: number;\n      captureAfterCallingSnapshotEnd?: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    /**\n     * Pixel rect of the deepLocate first-stage search area in screenshot\n     * coordinates. Used by reports to explain the crop/zoom area that the\n     * final locate ran against.\n     */\n    searchArea?: Rect;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  /** Stable unique identifier for this execution run */\n  id?: string;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/*\ntask - service-locate\n*/\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  DetailedLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n  domIncluded?: boolean | 'visible-only';\n  context?: string;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport interface ExecutionTaskPlanningParam {\n  userInstruction: TUserPrompt;\n  userInstructionDisplay?: string;\n  replanningCycleLimit?: number;\n  aiActContext?: string;\n  imagesIncludeCount?: number;\n  effort?: AiActEffort;\n  subGoalStatus?: string;\n  memoriesStatus?: string;\n}\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningParam,\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  DetailedLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nHow a report file stores screenshots:\n- `inline`: base64 image script tags embedded in the single HTML file\n- `directory`: external PNG files under a sibling `screenshots/` dir\n*/\nexport type ScreenshotMode = 'inline' | 'directory';\n\n/*\nReport metadata - extracted from ReportActionDump for per-execution writes\n*/\nexport interface ReportMeta {\n  groupName: string;\n  groupDescription?: string;\n  sdkVersion: string;\n  modelBriefs: ModelBrief[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupMeta = ReportMeta;\n\n/*\nReport dump\n*/\nexport interface IReportActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: ModelBrief[];\n  executions: IExecutionDump[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type IGroupedActionDump = IReportActionDump;\n\nexport interface ModelBrief {\n  /**\n   * The intent/category of the model call, for example \"planning\" or \"insight\".\n   */\n  intent?: string;\n\n  /**\n   * The model name returned by usage metadata, for example \"gpt-4o\".\n   */\n  name?: string;\n\n  /**\n   * Optional human-readable model description, for example \"qwen2.5-vl mode\".\n   */\n  modelDescription?: string;\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Provider token usage, without guaranteed Midscene metadata. See the final response for AIUsageInfo. */\n  usage?: OpenAI.CompletionUsage;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  /**\n   * Maximum number of characters from planningFeedback sent to the next\n   * planning round. Omit to use the core default, or set to 'unlimited' to\n   * disable truncation for this action.\n   */\n  planningFeedbackMaxLength?: number | 'unlimited';\n  call: (\n    param: TParam,\n    context?: ExecutorContext,\n  ) => Promise<TReturn> | TReturn;\n  delayBeforeRunner?: number;\n  delayAfterRunner?: number;\n  /**\n   * An example param object for this action.\n   * Locate fields with { prompt } may be resolved to internal pixel bboxes when needed.\n   */\n  sample?: { [K in keyof TParam]?: any };\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n  /**\n   * Optional cache directory path.\n   * When set, cache files are written to this directory instead of\n   * `<MIDSCENE_RUN_DIR>/cache`.\n   */\n  cacheDir?: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface ActionReadyContext {\n  /** Cancels this execution. Use it to stop polling, requests, or listeners. */\n  readonly signal?: AbortSignal;\n  /** A new identity for each executed atomic action, including cache replay. */\n  action: {\n    readonly id: string;\n    readonly name: string;\n    /** Resolved action parameters. Treat them as read-only. */\n    readonly param: unknown;\n  };\n}\n\n/** Runs after a successful atomic action. Slow callbacks warn after 5 seconds. */\nexport type WaitForActionReady = (context: ActionReadyContext) => Promise<void>;\n\nexport interface AgentOpt {\n  // @deprecated Use `reportFileName` and `cache.id` instead.\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if persist per-execution dump files next to the report, default false */\n  persistExecutionDump?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n\n  /**\n   * Use directory-based report format with separate image files.\n   *\n   * When enabled:\n   * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n   * - Report is generated as `index.html` with relative image paths\n   * - Reduces memory usage and report file size\n   *\n   * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n   * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n   * work due to browser CORS restrictions.\n   *\n   * @default 'single-html'\n   */\n  outputFormat?: 'single-html' | 'html-and-external-assets';\n\n  onTaskStartTip?: OnTaskStartTip;\n  /**\n   * Agent-level AI guidance containing business facts, rules, constraints, or\n   * output requirements. `aiContexts.default` is a shared fallback used only\n   * when neither the call nor the matching API key provides a context.\n   *\n   * A per-call `options.context` overrides the matching API context, which\n   * overrides `aiContexts.default`. An empty string explicitly clears\n   * inherited user context for that scope. These layers are not automatically\n   * merged.\n   */\n  aiContexts?: AgentAIContexts;\n  /**\n   * Compatibility alias for `aiContexts.aiAct`. `aiContexts.aiAct` takes\n   * precedence when both are provided.\n   * @deprecated Use `aiContexts.aiAct` instead.\n   */\n  aiActContext?: string;\n  /**\n   * Older compatibility alias for `aiContexts.aiAct`.\n   * @deprecated Use `aiContexts.aiAct` instead.\n   */\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  reportAttributes?: ReportAttributes;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults are resolved by the active model adapter: 20 for standard planning,\n   * 40 for UI-TARS, and 100 for Auto-GLM.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Wait time in milliseconds after each action execution.\n   * This allows the UI to settle and stabilize before the next action.\n   * Defaults to 300ms when not provided.\n   */\n  waitAfterAction?: number;\n\n  /**\n   * Await application readiness after each successful atomic action, including\n   * aiAct substeps and cache replay. Replaces default action waits.\n   * After 5 seconds, logs one warning and continues waiting without a timeout.\n   */\n  waitForActionReady?: WaitForActionReady;\n\n  /**\n   * When set to true, Midscene will use the target device's formatted local\n   * time instead of the runtime system time. The target interface must implement\n   * getDeviceLocalTimeString to provide device-local wall-clock time.\n   * Default: false\n   */\n  useDeviceTime?: boolean;\n\n  /**\n   * Custom screenshot shrink factor to reduce AI token usage.\n   * When set, the screenshot will be scaled down by this factor from the physical resolution.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - AI analyzes the 1500px screenshot\n   * - Coordinates are transformed back to logical (500px) before actions execute\n   *\n   * Benefits:\n   * - Reduces token usage for high-resolution screenshots\n   * - Maintains accuracy by scaling coordinates appropriately\n   *\n   * Must be >= 1 (shrinking only, enlarging is not supported).\n   *\n   * @default 1 (no shrinking, uses original physical screenshot)\n   */\n  screenshotShrinkFactor?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n\n  /**\n   * Called once per LLM call as soon as its usage is available, with the raw\n   * {@link AIUsageInfo} (token counts, model name, intent, request id, etc.).\n   *\n   * Use this for real-time, per-spec cost observability — e.g. push each call\n   * to Langfuse without waiting for the run to finish. For aggregated totals,\n   * read `agent.metrics` instead.\n   */\n  onLLMUsage?: (usage: AIUsageInfo) => void;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileAttributes {\n  testDuration: number;\n  testStatus: TestStatus;\n  testTitle: string;\n  testId: string;\n  testDescription: string;\n}\n\ntype SkippedReportFileAttributes = Omit<ReportFileAttributes, 'testStatus'> & {\n  testStatus: 'skipped';\n};\n\nexport type ReportFileWithAttributes =\n  | {\n      reportFilePath: string;\n      reportAttributes: ReportFileAttributes;\n    }\n  | {\n      reportFilePath?: undefined;\n      reportAttributes: SkippedReportFileAttributes;\n    };\n"],"names":["UIContext","aiActProgressScope"],"mappings":";;;AAqLO,MAAeA;AAqCtB;AA8VO,MAAMC,qBAAqB"}
+{"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { AgentAIContexts } from '@midscene/shared/agent-tools/agent-context';\nimport type { SerializedError } from '@midscene/shared/agent-tools/error-formatter';\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type OpenAI from 'openai';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport type { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport type { SerializedError } from '@midscene/shared/agent-tools/error-formatter';\nexport type {\n  AgentAIContextKey,\n  AgentAIContexts,\n  AiApiName,\n} from '@midscene/shared/agent-tools/agent-context';\nexport * from './yaml';\n\nexport { ServiceError } from './errors';\nexport {\n  ExecutionDump,\n  ReportActionDump,\n  GroupedActionDump,\n} from './dump/report-action-dump';\n\nexport type RawAssistantOutput =\n  | {\n      type: 'chat-completion';\n      rawValue: OpenAI.Chat.Completions.ChatCompletionMessage;\n    }\n  | {\n      type: 'responses';\n      rawValue: OpenAI.Responses.ResponseOutputItem[];\n    };\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  /** Duration of the final successful request attempt, in milliseconds. */\n  time_cost: number | undefined;\n  /** Request duration including all attempts and retry waits, excluding preparation, in milliseconds. */\n  total_time_cost?: number;\n  /** Request retries performed within this call; excludes semantic parse retries. */\n  retry_count?: number;\n  /** Actual protocol used for the model call, resolved by Midscene. */\n  request_kind?: 'chat-completion' | 'responses' | 'codex' | 'system-one';\n  /** Why a task uses the default model instead of its configured specialist. */\n  modelFallback?: {\n    requestedModel: string;\n    requestedSlot: string;\n    taskType: string;\n    reason: string;\n  };\n  /** Whether the model call used streaming, after resolving call options. */\n  stream?: boolean;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  /**\n   * Raw top-level `.model` value returned by the model service response.\n   */\n  response_model_name: string | undefined;\n  /**\n   * Semantic intent of the model call, such as default, planning, or insight.\n   */\n  intent: string | undefined;\n  /**\n   * Config slot where the model config was resolved from. For example, a\n   * planning call may use the default slot when no planning model is configured.\n   */\n  slot: string | undefined;\n  request_id: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type LocateResultPoint = [number, number];\nexport type Bbox = [number, number, number, number];\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\n\nexport interface AIElementLocateResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  error?: string;\n}\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  references_bbox?: LocateResultBbox[];\n  references_point?: LocateResultPoint[];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect?: Rect;\n  center: [number, number];\n  centerDistance?: number;\n  /** Whether the expected point is inside the located rect; absent for point-only results. */\n  includedInRect?: boolean;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepLocate: boolean;\n  deepDescribe: boolean;\n  verifyResult?: LocateValidatorResult;\n  success: boolean;\n  error?: string;\n  failureStage?: 'describe' | 'verify';\n}\n\n/**\n * context\n */\n\nexport interface UiNode {\n  type: string;\n  attrs: Record<string, string | undefined>;\n  bounds: Rect;\n  children: UiNode[];\n}\n\nexport interface UITreeSnapshot {\n  platform: 'android';\n  capturedAt: number;\n  root: UiNode;\n}\n\nexport abstract class UIContext {\n  /**\n   * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n   */\n  abstract screenshot: ScreenshotItem;\n\n  /**\n   * Optional sequence of screenshots captured over a short time window, in\n   * temporal order (earliest first, latest last). When present with more than\n   * one frame, extract/assert flows submit all frames to the model so it can\n   * observe transient UI (toasts, carousels, auto-hiding controls). The last\n   * frame is the same state as {@link screenshot}.\n   */\n  abstract screenshotSequence?: ScreenshotItem[];\n\n  /**\n   * screenshot size after shrinking\n   */\n  abstract shotSize: Size;\n\n  /**\n   * The ratio for converting shrunk screenshot coordinates to logical coordinates.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - User-defined screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n   * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n   */\n  abstract shrunkShotToLogicalRatio: number;\n\n  abstract _isFrozen?: boolean;\n\n  // @deprecated - backward compatibility for aiLocate\n  abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type AiActEffort = 'fast' | 'balance' | 'deepThink';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: unknown;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: unknown;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  /**\n   * Adapter-extracted content from the search-area model call. This is not the\n   * full provider response or choices[0].message.\n   */\n  searchAreaRawResponse?: string;\n  searchAreaRawChoiceMessage?: unknown;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport type ReportAttributes = Record<\n  string,\n  string | number | boolean | null | undefined\n>;\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: ReportAttributes;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement?: LocateResultElement[];\n  deepLocate?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n  /**\n   * Additional facts, decision rules, constraints, or output requirements for\n   * this assertion. It overrides `aiContexts.aiAssert` and\n   * `aiContexts.default`;\n   * `''` disables inherited user context for this call.\n   */\n  context?: string;\n  abortSignal?: AbortSignal;\n}\n\nexport interface AgentAssertResult {\n  pass: boolean;\n  thought?: string;\n  message?: string;\n}\n\nexport type QueryOptions = ServiceExtractOption;\nexport type AssertOptions = AgentAssertOpt & ServiceExtractOption;\n\n/** Options for fixed observations, which never read the live DOM. */\nexport type ObservationQueryOptions = Omit<QueryOptions, 'domIncluded'> & {\n  domIncluded?: never;\n};\n\n/** Assertion options for fixed observations, which never read the live DOM. */\nexport type ObservationAssertOptions = Omit<AssertOptions, 'domIncluded'> & {\n  domIncluded?: never;\n};\n\n/** Read-only AI operations supported by both live Agents and UI observations. */\nexport interface InsightAPI<\n  QueryOpt extends QueryOptions = QueryOptions,\n  AssertOpt extends AssertOptions = AssertOptions,\n> {\n  aiQuery<ReturnType = any>(\n    demand: ServiceExtractParam,\n    options?: QueryOpt,\n  ): Promise<ReturnType>;\n  aiBoolean(prompt: TUserPrompt, options?: QueryOpt): Promise<boolean>;\n  aiNumber(prompt: TUserPrompt, options?: QueryOpt): Promise<number>;\n  aiString(prompt: TUserPrompt, options?: QueryOpt): Promise<string>;\n  aiAsk(prompt: TUserPrompt, options?: QueryOpt): Promise<string>;\n  aiAssert(\n    assertion: TUserPrompt,\n    message?: string,\n    options?: AssertOpt,\n  ): Promise<AgentAssertResult | undefined>;\n}\n\n/**\n * planning\n *\n */\n\ntype PlanningActionBase = {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n};\n\nexport type PlanningAction<ParamType = undefined> = PlanningActionBase &\n  ([ParamType] extends [undefined] ? { param?: any } : { param: ParamType });\n\nexport type SubGoalStatus = 'pending' | 'running' | 'finished';\n\nexport interface SubGoal {\n  index: number;\n  status: SubGoalStatus;\n  description: string;\n  logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction | null;\n  thought?: string;\n  log: string;\n  memory?: string;\n  error?: string;\n  finalizeMessage?: string;\n  finalizeSuccess?: boolean;\n  updateSubGoals?: SubGoal[];\n  markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  /** Request another planning cycle using the default model, without executing actions. */\n  delegation?: {\n    reason: string;\n    suggestedAction?: string;\n  };\n  usage?: AIUsageInfo;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n  output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (\n    task: ExecutionTask,\n    signal?: AbortSignal,\n  ) => Promise<void> | void;\n}\n\n/**\n * Generic agent progress bus.\n *\n * Progress notifications are a single, generic stream the agent broadcasts as\n * it works. Each event is a thin envelope: a `scope` naming the producer, a\n * `phase` within that producer's lifecycle, a monotonic `sequence`, and a\n * structured, presentation-free `data` payload. The bus knows nothing about any\n * particular producer's payload - consumers narrow by `scope`. `aiAct` is the\n * first producer (\"pilot\") on this bus; others (queries, waits, ...) can be\n * added without touching the bus, the listener API, or the renderer core.\n */\nexport interface AgentProgressEvent<\n  TScope extends string = string,\n  TData = unknown,\n  TPhase extends string = string,\n> {\n  scope: TScope;\n  phase: TPhase;\n  sequence: number;\n  data: TData;\n}\n\nexport type AgentProgressListener<\n  TScope extends string = string,\n  TData = unknown,\n  TPhase extends string = string,\n> = (event: AgentProgressEvent<TScope, TData, TPhase>) => Promise<void> | void;\n\n// --- aiAct: the first producer on the generic progress bus ---\n\nexport type AiActProgressPhase =\n  | 'start'\n  | 'plan_thinking'\n  | 'plan_planned'\n  | 'plan_action'\n  | 'plan_failed'\n  | 'action_running'\n  | 'action_done'\n  | 'action_failed'\n  | 'complete'\n  | 'failed';\n\nexport interface AiActProgressAction {\n  name: string;\n  target?: string;\n  point?: [number, number];\n  bbox?: [number, number, number, number];\n  /**\n   * Structured, compacted summary of the action params for actions that are\n   * not described by a point/bbox (e.g. `Sleep` -> `{ timeMs }`). This is data,\n   * never a pre-formatted display string; consumers decide how to render it.\n   */\n  param?: unknown;\n}\n\n/**\n * Structured payload carried by aiAct progress events. The producer only\n * reports *what happened* as data: the action involved, the raw text the model\n * produced, timings and errors. It never assembles human-readable log lines or\n * truncates strings for display - that belongs to whichever layer consumes the\n * stream (e.g. the CLI verbose renderer).\n */\nexport interface AiActProgressData {\n  /** Original user instruction, present on the `start` phase. */\n  prompt?: string;\n  planIndex?: number;\n  planLimit?: number;\n  /** Latest screenshot, present on `plan_thinking`. */\n  screenshot?: ScreenshotItem;\n  /** Structured action descriptor, present on the `*action*` phases. */\n  action?: AiActProgressAction;\n  /**\n   * Raw, untruncated semantic text produced by the model. Consumers choose\n   * which field to surface and how to format/truncate it.\n   */\n  thought?: string;\n  log?: string;\n  output?: string;\n  /** Wall-clock cost of an action, present on `action_done`/`action_failed`. */\n  durationMs?: number;\n  error?: string;\n}\n\nexport const aiActProgressScope = 'aiAct';\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  description?: string;\n  timing?: string;\n}\n\nexport interface RecordToReportScreenshot {\n  /**\n   * PNG/JPEG data URI, or raw PNG base64 body.\n   */\n  base64: string;\n  description?: string;\n}\n\nexport interface RecordToReportOptions {\n  content?: string;\n  /**\n   * @deprecated Use `screenshots: [{ base64 }]` instead.\n   */\n  screenshotBase64?: string;\n  /**\n   * Custom screenshots to display under a single report entry.\n   */\n  screenshots?: RecordToReportScreenshot[];\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  /** Cancellation of this execution; adapters may forward it to cancellable I/O. */\n  signal?: AbortSignal;\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n  /** Custom readiness owns optional post-action settling, not input/gesture timing. */\n  skipDefaultWait?: boolean;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  /**\n   * Maximum planning feedback length inherited from the action definition.\n   * Set to 'unlimited' to disable truncation for this task.\n   */\n  planningFeedbackMaxLength?: number | 'unlimited';\n  executor: (\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    taskId: string;\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    /** Readiness strategy selected for this atomic action execution. */\n    actionReadiness?: 'default' | 'custom';\n    /**\n     * Optional feedback produced by a task for the next planning round.\n     * This is execution metadata, not part of the action return value.\n     */\n    planningFeedback?: string;\n    /**\n     * A bounded diagnostic DTO created when the task executor throws. Arbitrary\n     * upstream payloads are intentionally omitted; use this field for structured\n     * diagnostics or errorMessage/errorStack for the common display path.\n     */\n    error?: SerializedError;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      getUiContextStart?: number;\n      getUiContextEnd?: number;\n      callAiStart?: number;\n      callAiEnd?: number;\n      beforeInvokeActionHookStart?: number;\n      beforeInvokeActionHookEnd?: number;\n      callActionStart?: number;\n      callActionEnd?: number;\n      afterInvokeActionHookStart?: number;\n      afterInvokeActionHookEnd?: number;\n      waitForActionReadyStart?: number;\n      waitForActionReadyEnd?: number;\n      captureAfterCallingSnapshotStart?: number;\n      captureAfterCallingSnapshotEnd?: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    /**\n     * Pixel rect of the deepLocate first-stage search area in screenshot\n     * coordinates. Used by reports to explain the crop/zoom area that the\n     * final locate ran against.\n     */\n    searchArea?: Rect;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  /** Stable unique identifier for this execution run */\n  id?: string;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/*\ntask - service-locate\n*/\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  DetailedLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n  domIncluded?: boolean | 'visible-only';\n  context?: string;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport interface ExecutionTaskPlanningParam {\n  userInstruction: TUserPrompt;\n  userInstructionDisplay?: string;\n  replanningCycleLimit?: number;\n  aiActContext?: string;\n  imagesIncludeCount?: number;\n  effort?: AiActEffort;\n  subGoalStatus?: string;\n  memoriesStatus?: string;\n}\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningParam,\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  DetailedLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nHow a report file stores screenshots:\n- `inline`: base64 image script tags embedded in the single HTML file\n- `directory`: external PNG files under a sibling `screenshots/` dir\n*/\nexport type ScreenshotMode = 'inline' | 'directory';\n\n/*\nReport metadata - extracted from ReportActionDump for per-execution writes\n*/\nexport interface ReportMeta {\n  groupName: string;\n  groupDescription?: string;\n  sdkVersion: string;\n  modelBriefs: ModelBrief[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupMeta = ReportMeta;\n\n/*\nReport dump\n*/\nexport interface IReportActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: ModelBrief[];\n  executions: IExecutionDump[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type IGroupedActionDump = IReportActionDump;\n\nexport interface ModelBrief {\n  /**\n   * The intent/category of the model call, for example \"planning\" or \"insight\".\n   */\n  intent?: string;\n\n  /**\n   * The model name returned by usage metadata, for example \"gpt-4o\".\n   */\n  name?: string;\n\n  /**\n   * Optional human-readable model description, for example \"qwen2.5-vl mode\".\n   */\n  modelDescription?: string;\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Provider token usage, without guaranteed Midscene metadata. See the final response for AIUsageInfo. */\n  usage?: OpenAI.CompletionUsage;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  /**\n   * Allow System One planners to construct this action directly.\n   * Only enable for actions with no parameters or locator-only parameters.\n   * Omitted/false actions are offered as delegation choices to the default model.\n   */\n  supportedBySystemOne?: boolean;\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  /**\n   * Maximum number of characters from planningFeedback sent to the next\n   * planning round. Omit to use the core default, or set to 'unlimited' to\n   * disable truncation for this action.\n   */\n  planningFeedbackMaxLength?: number | 'unlimited';\n  call: (\n    param: TParam,\n    context?: ExecutorContext,\n  ) => Promise<TReturn> | TReturn;\n  delayBeforeRunner?: number;\n  delayAfterRunner?: number;\n  /**\n   * An example param object for this action.\n   * Locate fields with { prompt } may be resolved to internal pixel bboxes when needed.\n   */\n  sample?: { [K in keyof TParam]?: any };\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n  /**\n   * Optional cache directory path.\n   * When set, cache files are written to this directory instead of\n   * `<MIDSCENE_RUN_DIR>/cache`.\n   */\n  cacheDir?: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface ActionReadyContext {\n  /** Cancels this execution. Use it to stop polling, requests, or listeners. */\n  readonly signal?: AbortSignal;\n  /** A new identity for each executed atomic action, including cache replay. */\n  action: {\n    readonly id: string;\n    readonly name: string;\n    /** Resolved action parameters. Treat them as read-only. */\n    readonly param: unknown;\n  };\n}\n\n/** Runs after a successful atomic action. Slow callbacks warn after 5 seconds. */\nexport type WaitForActionReady = (context: ActionReadyContext) => Promise<void>;\n\nexport interface AgentOpt {\n  // @deprecated Use `reportFileName` and `cache.id` instead.\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if persist per-execution dump files next to the report, default false */\n  persistExecutionDump?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n\n  /**\n   * Use directory-based report format with separate image files.\n   *\n   * When enabled:\n   * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n   * - Report is generated as `index.html` with relative image paths\n   * - Reduces memory usage and report file size\n   *\n   * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n   * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n   * work due to browser CORS restrictions.\n   *\n   * @default 'single-html'\n   */\n  outputFormat?: 'single-html' | 'html-and-external-assets';\n\n  onTaskStartTip?: OnTaskStartTip;\n  /**\n   * Agent-level AI guidance containing business facts, rules, constraints, or\n   * output requirements. `aiContexts.default` is a shared fallback used only\n   * when neither the call nor the matching API key provides a context.\n   *\n   * A per-call `options.context` overrides the matching API context, which\n   * overrides `aiContexts.default`. An empty string explicitly clears\n   * inherited user context for that scope. These layers are not automatically\n   * merged.\n   */\n  aiContexts?: AgentAIContexts;\n  /**\n   * Compatibility alias for `aiContexts.aiAct`. `aiContexts.aiAct` takes\n   * precedence when both are provided.\n   * @deprecated Use `aiContexts.aiAct` instead.\n   */\n  aiActContext?: string;\n  /**\n   * Older compatibility alias for `aiContexts.aiAct`.\n   * @deprecated Use `aiContexts.aiAct` instead.\n   */\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  reportAttributes?: ReportAttributes;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults are resolved by the active model adapter: 20 for standard planning,\n   * 40 for UI-TARS, and 100 for Auto-GLM.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Wait time in milliseconds after each action execution.\n   * This allows the UI to settle and stabilize before the next action.\n   * Defaults to 300ms when not provided.\n   */\n  waitAfterAction?: number;\n\n  /**\n   * Await application readiness after each successful atomic action, including\n   * aiAct substeps and cache replay. Replaces default action waits.\n   * After 5 seconds, logs one warning and continues waiting without a timeout.\n   */\n  waitForActionReady?: WaitForActionReady;\n\n  /**\n   * When set to true, Midscene will use the target device's formatted local\n   * time instead of the runtime system time. The target interface must implement\n   * getDeviceLocalTimeString to provide device-local wall-clock time.\n   * Default: false\n   */\n  useDeviceTime?: boolean;\n\n  /**\n   * Custom screenshot shrink factor to reduce AI token usage.\n   * When set, the screenshot will be scaled down by this factor from the physical resolution.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - AI analyzes the 1500px screenshot\n   * - Coordinates are transformed back to logical (500px) before actions execute\n   *\n   * Benefits:\n   * - Reduces token usage for high-resolution screenshots\n   * - Maintains accuracy by scaling coordinates appropriately\n   *\n   * Must be >= 1 (shrinking only, enlarging is not supported).\n   *\n   * @default 1 (no shrinking, uses original physical screenshot)\n   */\n  screenshotShrinkFactor?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n\n  /**\n   * Called once per LLM call as soon as its usage is available, with the raw\n   * {@link AIUsageInfo} (token counts, model name, intent, request id, etc.).\n   *\n   * Use this for real-time, per-spec cost observability — e.g. push each call\n   * to Langfuse without waiting for the run to finish. For aggregated totals,\n   * read `agent.metrics` instead.\n   */\n  onLLMUsage?: (usage: AIUsageInfo) => void;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileAttributes {\n  testDuration: number;\n  testStatus: TestStatus;\n  testTitle: string;\n  testId: string;\n  testDescription: string;\n}\n\ntype SkippedReportFileAttributes = Omit<ReportFileAttributes, 'testStatus'> & {\n  testStatus: 'skipped';\n};\n\nexport type ReportFileWithAttributes =\n  | {\n      reportFilePath: string;\n      reportAttributes: ReportFileAttributes;\n    }\n  | {\n      reportFilePath?: undefined;\n      reportAttributes: SkippedReportFileAttributes;\n    };\n"],"names":["UIContext","aiActProgressScope"],"mappings":";;;AAqLO,MAAeA;AAqCtB;AAmWO,MAAMC,qBAAqB"}

package/dist/es/utils.mjs

@@ -184,7 +184,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "1.16.2";
+    return "1.16.6-beta-20261023073257.0";
 }
 function debugLog(...message) {
     const debugMode = process.env[MIDSCENE_DEBUG_MODE];

package/dist/lib/agent/agent.js

@@ -36,7 +36,7 @@ __webpack_require__.d(__webpack_exports__, {
     Agent: ()=>Agent,
     createAgent: ()=>createAgent
 });
-const task_routing_js_namespaceObject = require("../ai-model/models/task-routing.js");
+const model_fallback_js_namespaceObject = require("../ai-model/models/model-fallback.js");
 const index_js_namespaceObject = require("../ai-model/service-caller/index.js");
 const external_js_yaml_namespaceObject = require("js-yaml");
 var external_js_yaml_default = /*#__PURE__*/ __webpack_require__.n(external_js_yaml_namespaceObject);
@@ -164,8 +164,8 @@ class Agent {
     resolveReplanningCycleLimit(planningModel) {
         return this.opts.replanningCycleLimit ?? env_namespaceObject.globalConfigManager.getEnvConfigValueAsNumber(env_namespaceObject.MIDSCENE_REPLANNING_CYCLE_LIMIT) ?? planningModel.adapter.planning.defaultReplanningCycleLimit;
     }
-    resolveModelRuntime(intent, task) {
-        const runtime = (0, task_routing_js_namespaceObject.resolveModelForTask)(this.modelConfigManager, intent, task ?? ('default' === intent ? 'text' : 'insight' === intent ? 'extract' : intent));
+    resolveModelRuntime(intent, taskType) {
+        const runtime = (0, model_fallback_js_namespaceObject.resolveModelForTaskType)(this.modelConfigManager, intent, taskType);
         return {
             ...runtime,
             onUsage: (usage)=>{
@@ -181,7 +181,7 @@ class Agent {
     createInsight(getUIContext) {
         return new external_insight_js_namespaceObject.Insight({
             createTypeQueryExecution: (...args)=>this.withReportOperation(()=>this.taskExecutor.createTypeQueryExecution(...args))
-        }, (task)=>this.resolveModelRuntime('insight', task), (apiName, callContext)=>this.resolveUserContext(apiName, callContext), getUIContext);
+        }, (taskType)=>this.resolveModelRuntime('insight', taskType), (apiName, callContext)=>this.resolveUserContext(apiName, callContext), getUIContext);
     }
     async getActionSpace() {
         return this.fullActionSpace;
@@ -337,8 +337,8 @@ class Agent {
             actionPlan
         ].filter(Boolean);
         const title = (0, external_ui_utils_js_namespaceObject.taskTitleStr)(type, (0, external_ui_utils_js_namespaceObject.locateParamStr)(opt?.locate || {}));
-        const locateModel = this.resolveModelRuntime('locate');
-        const planningModel = this.resolveModelRuntime('planning');
+        const locateModel = this.resolveModelRuntime('locate', 'locate');
+        const planningModel = this.resolveModelRuntime('planning', 'planning');
         const { output } = await this.taskExecutor.runPlans(title, plans, planningModel, locateModel, {
             abortSignal: abortSignal ?? (0, execution_session_js_namespaceObject.currentYamlSignal)(this)
         });
@@ -494,8 +494,8 @@ class Agent {
         const abortSignal = opt?.abortSignal;
         if (abortSignal?.aborted) throw new Error(`aiAct aborted: ${abortSignal.reason || 'signal already aborted'}`);
         const runAiAct = async ()=>{
-            const planningModel = this.resolveModelRuntime('planning');
-            const locateModel = this.resolveModelRuntime('locate');
+            const planningModel = this.resolveModelRuntime('planning', 'planning');
+            const locateModel = this.resolveModelRuntime('locate', 'locate');
             const aiActContext = this.resolveUserContext('aiAct', opt?.context);
             const cachePrompt = (0, external_prompt_context_js_namespaceObject.buildPromptWithContext)(taskPrompt, aiActContext);
             const effort = (()=>{
@@ -637,8 +637,8 @@ class Agent {
         const plans = [
             locatePlan
         ];
-        const locateModel = this.resolveModelRuntime('locate');
-        const planningModel = this.resolveModelRuntime('planning');
+        const locateModel = this.resolveModelRuntime('locate', 'locate');
+        const planningModel = this.resolveModelRuntime('planning', 'planning');
         const { output } = await this.taskExecutor.runPlans((0, external_ui_utils_js_namespaceObject.taskTitleStr)('Locate', (0, external_ui_utils_js_namespaceObject.locateParamStr)(locateParam)), plans, planningModel, locateModel, opt?.uiContext ? {
             uiContext: opt.uiContext
         } : void 0);
@@ -933,7 +933,7 @@ class Agent {
             (0, external_device_index_js_namespaceObject.defineActionSleep)()
         ];
         this.taskExecutor = new external_tasks_js_namespaceObject.TaskExecutor(this.interface, this.service, {
-            getDefaultModelRuntime: ()=>this.resolveModelRuntime('default'),
+            getDefaultModelRuntime: ()=>this.resolveModelRuntime('default', 'general'),
             taskCache: this.taskCache,
             onTaskStart: this.callbackOnTaskStartTip.bind(this),
             replanningCycleLimit: this.opts.replanningCycleLimit,