@midscene/core 1.0.1-beta-20251028065320.0 → 1.0.1-beta-20251029093754.0

package/dist/es/ai-model/prompt/llm-section-locator.mjs

@@ -1,17 +1,20 @@
 import { PromptTemplate } from "@langchain/core/prompts";
 import { bboxDescription } from "./common.mjs";
 function systemPromptToLocateSection(vlMode) {
+    const bboxFormat = bboxDescription(vlMode);
     return `
-You goal is to find out one section containing the target element in the screenshot, put it in the \`bbox\` field. If the user describe the target element with some reference elements, you should also find the section containing the reference elements, put it in the \`references_bbox\` field.
+## Role:
+You are an AI assistant that helps identify UI elements.
 
-Usually, it should be approximately an area not more than 300x300px. Changes of the size are allowed if there are many elements to cover.
+## Objective:
+- Find a section containing the target element
+- If the description mentions reference elements, also locate sections containing those references
 
-return in this JSON format:
+## Output Format:
 \`\`\`json
 {
-  "bbox": [number, number, number, number],
+  "bbox": [number, number, number, number],  // ${bboxFormat}
   "references_bbox"?: [
-    [number, number, number, number],
     [number, number, number, number],
     ...
   ],
@@ -19,11 +22,13 @@ return in this JSON format:
 }
 \`\`\`
 
-In which, all the numbers in the \`bbox\` and \`references_bbox\` represent ${bboxDescription(vlMode)}.
-
-For example, if the user describe the target element as "the delete button on the second row with title 'Peter'", you should put the bounding box of the delete button in the \`bbox\` field, and the bounding box of the second row in the \`references_bbox\` field.
+Fields:
+* \`bbox\` - Bounding box of the section containing the target element
+* \`references_bbox\` - Optional array of bounding boxes for reference elements
+* \`error\` - Optional error message if the section cannot be found
 
-the return value should be like this:
+Example:
+If the description is "delete button on the second row with title 'Peter'", return:
 \`\`\`json
 {
   "bbox": [100, 100, 200, 200],
@@ -33,11 +38,7 @@ the return value should be like this:
 `;
 }
 const sectionLocatorInstruction = new PromptTemplate({
-    template: `Here is the target element user interested in:
-<targetDescription>
-{sectionDescription}
-</targetDescription>
-  `,
+    template: "Find section containing: {sectionDescription}",
     inputVariables: [
         "sectionDescription"
     ]

package/dist/es/ai-model/prompt/llm-section-locator.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/llm-section-locator.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/llm-section-locator.ts"],"sourcesContent":["import { PromptTemplate } from '@langchain/core/prompts';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { bboxDescription } from './common';\n\nexport function systemPromptToLocateSection(vlMode: TVlModeTypes | undefined) {\n  return `\nYou goal is to find out one section containing the target element in the screenshot, put it in the \\`bbox\\` field. If the user describe the target element with some reference elements, you should also find the section containing the reference elements, put it in the \\`references_bbox\\` field.\n\nUsually, it should be approximately an area not more than 300x300px. Changes of the size are allowed if there are many elements to cover.\n\nreturn in this JSON format:\n\\`\\`\\`json\n{\n  \"bbox\": [number, number, number, number],\n  \"references_bbox\"?: [\n    [number, number, number, number],\n    [number, number, number, number],\n    ...\n  ],\n  \"error\"?: string\n}\n\\`\\`\\`\n\nIn which, all the numbers in the \\`bbox\\` and \\`references_bbox\\` represent ${bboxDescription(vlMode)}.\n\nFor example, if the user describe the target element as \"the delete button on the second row with title 'Peter'\", you should put the bounding box of the delete button in the \\`bbox\\` field, and the bounding box of the second row in the \\`references_bbox\\` field.\n\nthe return value should be like this:\n\\`\\`\\`json\n{\n  \"bbox\": [100, 100, 200, 200],\n  \"references_bbox\": [[100, 100, 200, 200]]\n}\n\\`\\`\\`\n`;\n}\n\nexport const sectionLocatorInstruction = new PromptTemplate({\n  template: `Here is the target element user interested in:\n<targetDescription>\n{sectionDescription}\n</targetDescription>\n  `,\n  inputVariables: ['sectionDescription'],\n});\n"],"names":["systemPromptToLocateSection","vlMode","bboxDescription","sectionLocatorInstruction","PromptTemplate"],"mappings":";;AAIO,SAASA,4BAA4BC,MAAgC;IAC1E,OAAO,CAAC;;;;;;;;;;;;;;;;;;4EAkBkE,EAAEC,gBAAgBD,QAAQ;;;;;;;;;;;AAWtG,CAAC;AACD;AAEO,MAAME,4BAA4B,IAAIC,eAAe;IAC1D,UAAU,CAAC;;;;EAIX,CAAC;IACD,gBAAgB;QAAC;KAAqB;AACxC"}
+{"version":3,"file":"ai-model/prompt/llm-section-locator.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/llm-section-locator.ts"],"sourcesContent":["import { PromptTemplate } from '@langchain/core/prompts';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { bboxDescription } from './common';\n\nexport function systemPromptToLocateSection(vlMode: TVlModeTypes | undefined) {\n  const bboxFormat = bboxDescription(vlMode);\n  return `\n## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Find a section containing the target element\n- If the description mentions reference elements, also locate sections containing those references\n\n## Output Format:\n\\`\\`\\`json\n{\n  \"bbox\": [number, number, number, number],  // ${bboxFormat}\n  \"references_bbox\"?: [\n    [number, number, number, number],\n    ...\n  ],\n  \"error\"?: string\n}\n\\`\\`\\`\n\nFields:\n* \\`bbox\\` - Bounding box of the section containing the target element\n* \\`references_bbox\\` - Optional array of bounding boxes for reference elements\n* \\`error\\` - Optional error message if the section cannot be found\n\nExample:\nIf the description is \"delete button on the second row with title 'Peter'\", return:\n\\`\\`\\`json\n{\n  \"bbox\": [100, 100, 200, 200],\n  \"references_bbox\": [[100, 100, 200, 200]]\n}\n\\`\\`\\`\n`;\n}\n\nexport const sectionLocatorInstruction = new PromptTemplate({\n  template: 'Find section containing: {sectionDescription}',\n  inputVariables: ['sectionDescription'],\n});\n"],"names":["systemPromptToLocateSection","vlMode","bboxFormat","bboxDescription","sectionLocatorInstruction","PromptTemplate"],"mappings":";;AAIO,SAASA,4BAA4BC,MAAgC;IAC1E,MAAMC,aAAaC,gBAAgBF;IACnC,OAAO,CAAC;;;;;;;;;;;gDAWsC,EAAEC,WAAW;;;;;;;;;;;;;;;;;;;;;;AAsB7D,CAAC;AACD;AAEO,MAAME,4BAA4B,IAAIC,eAAe;IAC1D,UAAU;IACV,gBAAgB;QAAC;KAAqB;AACxC"}

package/dist/es/ai-model/prompt/order-sensitive-judge.mjs

@@ -0,0 +1,35 @@
+function systemPromptToJudgeOrderSensitive() {
+    return `
+## Role:
+You are an AI assistant that analyzes UI element descriptions.
+
+## Objective:
+Determine whether a given element description is order-sensitive.
+
+Order-sensitive descriptions contain phrases that specify position or sequence, such as:
+- "the first button"
+- "the second item"
+- "the third row"
+- "the last input"
+- "the 5th element"
+
+Order-insensitive descriptions do not specify position:
+- "login button"
+- "search input"
+- "submit button"
+- "user avatar"
+
+## Output Format:
+\`\`\`json
+{
+  "isOrderSensitive": boolean
+}
+\`\`\`
+
+Return true if the description is order-sensitive, false otherwise.
+`;
+}
+const orderSensitiveJudgePrompt = (description)=>`Analyze this element description: "${description}"`;
+export { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive };
+
+//# sourceMappingURL=order-sensitive-judge.mjs.map

package/dist/es/ai-model/prompt/order-sensitive-judge.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/order-sensitive-judge.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/order-sensitive-judge.ts"],"sourcesContent":["export function systemPromptToJudgeOrderSensitive() {\n  return `\n## Role:\nYou are an AI assistant that analyzes UI element descriptions.\n\n## Objective:\nDetermine whether a given element description is order-sensitive.\n\nOrder-sensitive descriptions contain phrases that specify position or sequence, such as:\n- \"the first button\"\n- \"the second item\"\n- \"the third row\"\n- \"the last input\"\n- \"the 5th element\"\n\nOrder-insensitive descriptions do not specify position:\n- \"login button\"\n- \"search input\"\n- \"submit button\"\n- \"user avatar\"\n\n## Output Format:\n\\`\\`\\`json\n{\n  \"isOrderSensitive\": boolean\n}\n\\`\\`\\`\n\nReturn true if the description is order-sensitive, false otherwise.\n`;\n}\n\nexport const orderSensitiveJudgePrompt = (description: string) => {\n  return `Analyze this element description: \"${description}\"`;\n};\n"],"names":["systemPromptToJudgeOrderSensitive","orderSensitiveJudgePrompt","description"],"mappings":"AAAO,SAASA;IACd,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4BV,CAAC;AACD;AAEO,MAAMC,4BAA4B,CAACC,cACjC,CAAC,mCAAmC,EAAEA,YAAY,CAAC,CAAC"}

package/dist/es/device/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"device/index.mjs","sources":["webpack://@midscene/core/./src/device/index.ts"],"sourcesContent":["import { getMidsceneLocationSchema } from '@/ai-model';\nimport type { DeviceAction, LocateResultElement } from '@/types';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;\n\n  abstract cacheFeatureForRect?(\n    rect: Rect,\n    opt?: { _orderSensitive: boolean },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  // @deprecated do NOT extend this method\n  abstract getContext?(): Promise<UIContext>;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\nexport const defineAction = <\n  TSchema extends z.ZodType,\n  TRuntime = z.infer<TSchema>,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema: TSchema;\n    call: (param: TRuntime) => Promise<void>;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\n// Override the inferred type to use LocateResultElement for the runtime locate field\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    call,\n  });\n};\n\n// Input\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be input')\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'append'])\n    .default('replace')\n    .optional()\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"append\" - append the value to existing content; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    call,\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe('The direction to scroll'),\n  scrollType: z\n    .enum(['once', 'untilBottom', 'untilTop', 'untilRight', 'untilLeft'])\n    .default('once')\n    .describe('The scroll type'),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe('The element to be scrolled'),\n});\nexport type ActionScrollParam = {\n  direction?: 'down' | 'up' | 'right' | 'left';\n  scrollType?: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';\n  distance?: number | null;\n  locate?: LocateResultElement;\n};\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description: 'Drag and drop the element',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a swipe gesture. You must specify either \"end\" (target location) or \"distance\" + \"direction\" - they are mutually exclusive. Use \"end\" for precise location-based swipes, or \"distance\" + \"direction\" for relative movement.',\n    paramSchema: ActionSwipeParamSchema,\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The input field to be cleared'),\n});\nexport type ActionClearInputParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: 'Clear the text content of an input field',\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    call,\n  });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n  AndroidDeviceOpt,\n  AndroidDeviceInputOpt,\n  IOSDeviceOpt,\n  IOSDeviceInputOpt,\n} from './device-options';\n"],"names":["AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","actionInputParamSchema","val","String","defineActionInput","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput"],"mappings":";;AAOO,MAAeA;AAgCtB;AAIO,MAAMC,eAAe,CAI1BC,SAaOA;AAIF,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAMO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACbI;IACF;AAIK,MAAMC,8BAA8BJ,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACbD;IACF;AAIK,MAAMG,+BAA+BN,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACbH;IACF;AAIK,MAAMK,yBAAyBR,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACbL;IACF;AAIK,MAAMO,yBAAyBV,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACW,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQV,4BACL,QAAQ,CAAC,2BACT,QAAQ;IACX,MAAMD,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAS,EACnC,OAAO,CAAC,WACR,QAAQ,GACR,QAAQ,CACP;AAEN;AAOO,MAAMa,oBAAoB,CAC/BV,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaa;QACbP;IACF;AAIK,MAAMW,iCAAiCd,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMe,4BAA4B,CACvCZ,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaiB;QACbX;IACF;AAIK,MAAMa,0BAA0BhB,EAAE,MAAM,CAAC;IAC9C,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,YAAYA,CAAC,CAADA,OACL,CAAC;QAAC;QAAQ;QAAe;QAAY;QAAc;KAAY,EACnE,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CAAC;AACd;AAQO,MAAMgB,qBAAqB,CAChCd,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACbb;IACF;AAIK,MAAMe,+BAA+BlB,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMkB,0BAA0B,CACrChB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaqB;QACbf;IACF;AAGK,MAAMiB,6BAA6BpB,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMqB,wBAAwB,CACnClB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAauB;QACbjB;IACF;AAGK,MAAMmB,yBAAyBtB,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,MAAMuB,oBAAoB,CAC/BpB,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAayB;QACbnB;IACF;AAIK,MAAMqB,8BAA8BxB,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMwB,yBAAyB,CACpCtB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAa2B;QACbrB;IACF"}
+{"version":3,"file":"device/index.mjs","sources":["webpack://@midscene/core/./src/device/index.ts"],"sourcesContent":["import { getMidsceneLocationSchema } from '@/ai-model';\nimport type { DeviceAction, LocateResultElement } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;\n\n  abstract cacheFeatureForRect?(\n    rect: Rect,\n    options?: {\n      targetDescription?: string;\n      modelConfig?: IModelConfig;\n    },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  // @deprecated do NOT extend this method\n  abstract getContext?(): Promise<UIContext>;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\nexport const defineAction = <\n  TSchema extends z.ZodType,\n  TRuntime = z.infer<TSchema>,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema: TSchema;\n    call: (param: TRuntime) => Promise<void>;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\n// Override the inferred type to use LocateResultElement for the runtime locate field\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    call,\n  });\n};\n\n// Input\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be input')\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'append'])\n    .default('replace')\n    .optional()\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"append\" - append the value to existing content; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    call,\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe('The direction to scroll'),\n  scrollType: z\n    .enum(['once', 'untilBottom', 'untilTop', 'untilRight', 'untilLeft'])\n    .default('once')\n    .describe('The scroll type'),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe('The element to be scrolled'),\n});\nexport type ActionScrollParam = {\n  direction?: 'down' | 'up' | 'right' | 'left';\n  scrollType?: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';\n  distance?: number | null;\n  locate?: LocateResultElement;\n};\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description: 'Drag and drop the element',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a swipe gesture. You must specify either \"end\" (target location) or \"distance\" + \"direction\" - they are mutually exclusive. Use \"end\" for precise location-based swipes, or \"distance\" + \"direction\" for relative movement.',\n    paramSchema: ActionSwipeParamSchema,\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The input field to be cleared'),\n});\nexport type ActionClearInputParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: 'Clear the text content of an input field',\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    call,\n  });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n  AndroidDeviceOpt,\n  AndroidDeviceInputOpt,\n  IOSDeviceOpt,\n  IOSDeviceInputOpt,\n} from './device-options';\n"],"names":["AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","actionInputParamSchema","val","String","defineActionInput","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput"],"mappings":";;AAQO,MAAeA;AAmCtB;AAIO,MAAMC,eAAe,CAI1BC,SAaOA;AAIF,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAMO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACbI;IACF;AAIK,MAAMC,8BAA8BJ,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACbD;IACF;AAIK,MAAMG,+BAA+BN,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACbH;IACF;AAIK,MAAMK,yBAAyBR,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACbL;IACF;AAIK,MAAMO,yBAAyBV,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACW,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQV,4BACL,QAAQ,CAAC,2BACT,QAAQ;IACX,MAAMD,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAS,EACnC,OAAO,CAAC,WACR,QAAQ,GACR,QAAQ,CACP;AAEN;AAOO,MAAMa,oBAAoB,CAC/BV,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaa;QACbP;IACF;AAIK,MAAMW,iCAAiCd,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMe,4BAA4B,CACvCZ,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaiB;QACbX;IACF;AAIK,MAAMa,0BAA0BhB,EAAE,MAAM,CAAC;IAC9C,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,YAAYA,CAAC,CAADA,OACL,CAAC;QAAC;QAAQ;QAAe;QAAY;QAAc;KAAY,EACnE,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CAAC;AACd;AAQO,MAAMgB,qBAAqB,CAChCd,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACbb;IACF;AAIK,MAAMe,+BAA+BlB,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMkB,0BAA0B,CACrChB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaqB;QACbf;IACF;AAGK,MAAMiB,6BAA6BpB,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMqB,wBAAwB,CACnClB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAauB;QACbjB;IACF;AAGK,MAAMmB,yBAAyBtB,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,MAAMuB,oBAAoB,CAC/BpB,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAayB;QACbnB;IACF;AAIK,MAAMqB,8BAA8BxB,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMwB,yBAAyB,CACpCtB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAa2B;QACbrB;IACF"}

package/dist/es/task-runner.mjs

@@ -1,3 +1,4 @@
+import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
@@ -9,10 +10,36 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
+const debug = getDebug('task-runner');
+const UI_CONTEXT_CACHE_TTL_MS = 300;
 class TaskRunner {
-    async captureScreenshot() {
+    async getUiContext(options) {
+        var _this_lastUiContext;
+        const now = Date.now();
+        const shouldReuse = !(null == options ? void 0 : options.forceRefresh) && this.lastUiContext && now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;
+        if (shouldReuse && (null == (_this_lastUiContext = this.lastUiContext) ? void 0 : _this_lastUiContext.context)) {
+            var _this_lastUiContext1;
+            debug(`reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`);
+            return null == (_this_lastUiContext1 = this.lastUiContext) ? void 0 : _this_lastUiContext1.context;
+        }
         try {
             const uiContext = await this.uiContextBuilder();
+            if (uiContext) this.lastUiContext = {
+                context: uiContext,
+                capturedAt: Date.now()
+            };
+            else this.lastUiContext = void 0;
+            return uiContext;
+        } catch (error) {
+            this.lastUiContext = void 0;
+            throw error;
+        }
+    }
+    async captureScreenshot() {
+        try {
+            const uiContext = await this.getUiContext({
+                forceRefresh: true
+            });
             return null == uiContext ? void 0 : uiContext.screenshotBase64;
         } catch (error) {
             console.error('error while capturing screenshot', error);
@@ -97,7 +124,7 @@ class TaskRunner {
                 if (task.subTask) {
                     uiContext = this.findPreviousNonSubTaskUIContext(taskIndex);
                     assert(uiContext, 'subTask requires uiContext from previous non-subTask task');
-                } else uiContext = await this.uiContextBuilder();
+                } else uiContext = await this.getUiContext();
                 task.uiContext = uiContext;
                 const executorContext = {
                     task,
@@ -116,7 +143,7 @@ class TaskRunner {
                 const isLastTask = taskIndex === this.tasks.length - 1;
                 if (isLastTask) {
                     const screenshot = await this.captureScreenshot();
-                    this.attachRecorderItem(task, screenshot, 'after');
+                    this.attachRecorderItem(task, screenshot, 'after-calling');
                 }
                 Object.assign(task, returnValue);
                 task.status = 'finished';
@@ -188,6 +215,7 @@ class TaskRunner {
         _define_property(this, "status", void 0);
         _define_property(this, "onTaskStart", void 0);
         _define_property(this, "uiContextBuilder", void 0);
+        _define_property(this, "lastUiContext", void 0);
         this.status = (null == options ? void 0 : options.tasks) && options.tasks.length > 0 ? 'pending' : 'init';
         this.name = name;
         this.tasks = ((null == options ? void 0 : options.tasks) || []).map((item)=>this.markTaskAsPending(item));

package/dist/es/task-runner.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"task-runner.mjs","sources":["webpack://@midscene/core/./src/task-runner.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionRecorderItem,\n  ExecutionTask,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n  PlanningActionParamError,\n  UIContext,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nexport class TaskRunner {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of runner\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly uiContextBuilder: () => Promise<UIContext>;\n\n  constructor(\n    name: string,\n    uiContextBuilder: () => Promise<UIContext>,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n    this.uiContextBuilder = uiContextBuilder;\n  }\n\n  private async captureScreenshot(): Promise<string | undefined> {\n    try {\n      const uiContext = await this.uiContextBuilder();\n      return uiContext?.screenshotBase64;\n    } catch (error) {\n      console.error('error while capturing screenshot', error);\n    }\n    return undefined;\n  }\n\n  private attachRecorderItem(\n    task: ExecutionTask,\n    contextOrScreenshot: UIContext | string | undefined,\n    phase: 'before' | 'after',\n  ): void {\n    const timing = phase;\n    const screenshot =\n      typeof contextOrScreenshot === 'string'\n        ? contextOrScreenshot\n        : contextOrScreenshot?.screenshotBase64;\n    if (!timing || !screenshot) {\n      return;\n    }\n\n    const recorderItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot,\n      timing,\n    };\n\n    if (!task.recorder) {\n      task.recorder = [recorderItem];\n      return;\n    }\n    task.recorder.push(recorderItem);\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  private findPreviousNonSubTaskUIContext(\n    currentIndex: number,\n  ): UIContext | undefined {\n    for (let i = currentIndex - 1; i >= 0; i--) {\n      const candidate = this.tasks[i];\n      if (!candidate || candidate.subTask) {\n        continue;\n      }\n      if (candidate.uiContext) {\n        return candidate.uiContext;\n      }\n    }\n    return undefined;\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `task runner is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async appendAndFlush(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    await this.append(task);\n    return this.flush();\n  }\n\n  async flush(): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for task runner, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'task runner is already running');\n    assert(this.status !== 'completed', 'task runner is already completed');\n    assert(this.status !== 'error', 'task runner is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        let uiContext: UIContext | undefined;\n        if (task.subTask) {\n          uiContext = this.findPreviousNonSubTaskUIContext(taskIndex);\n          assert(\n            uiContext,\n            'subTask requires uiContext from previous non-subTask task',\n          );\n        } else {\n          uiContext = await this.uiContextBuilder();\n        }\n        task.uiContext = uiContext;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n          uiContext,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'WaitFor' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported service subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        const isLastTask = taskIndex === this.tasks.length - 1;\n\n        if (isLastTask) {\n          const screenshot = await this.captureScreenshot();\n          this.attachRecorderItem(task, screenshot, 'after');\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n\n  async appendErrorPlan(errorMsg: string): Promise<{\n    output: undefined;\n    runner: TaskRunner;\n  }> {\n    const errorTask: ExecutionTaskActionApply<PlanningActionParamError> = {\n      type: 'Action',\n      subType: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      thought: errorMsg,\n      locate: null,\n      executor: async () => {\n        throw new Error(errorMsg || 'error without thought');\n      },\n    };\n    await this.appendAndFlush(errorTask);\n\n    return {\n      output: undefined,\n      runner: this,\n    };\n  }\n}\n"],"names":["TaskRunner","uiContext","error","console","task","contextOrScreenshot","phase","timing","screenshot","recorderItem","Date","currentIndex","i","candidate","_this_latestErrorTask","_this_latestErrorTask1","assert","Array","item","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","e","executor","param","returnValue","executorContext","isLastTask","Object","outputIndex","Math","thought","output","errorTaskIndex","dumpData","errorMsg","errorTask","Error","undefined","name","uiContextBuilder","options"],"mappings":";;;;;;;;;;;AAeO,MAAMA;IA6BX,MAAc,oBAAiD;QAC7D,IAAI;YACF,MAAMC,YAAY,MAAM,IAAI,CAAC,gBAAgB;YAC7C,OAAOA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,gBAAgB;QACpC,EAAE,OAAOC,OAAO;YACdC,QAAQ,KAAK,CAAC,oCAAoCD;QACpD;IAEF;IAEQ,mBACNE,IAAmB,EACnBC,mBAAmD,EACnDC,KAAyB,EACnB;QACN,MAAMC,SAASD;QACf,MAAME,aACJ,AAA+B,YAA/B,OAAOH,sBACHA,sBACAA,QAAAA,sBAAAA,KAAAA,IAAAA,oBAAqB,gBAAgB;QAC3C,IAAI,CAACE,UAAU,CAACC,YACd;QAGF,MAAMC,eAAsC;YAC1C,MAAM;YACN,IAAIC,KAAK,GAAG;YACZF;YACAD;QACF;QAEA,IAAI,CAACH,KAAK,QAAQ,EAAE;YAClBA,KAAK,QAAQ,GAAG;gBAACK;aAAa;YAC9B;QACF;QACAL,KAAK,QAAQ,CAAC,IAAI,CAACK;IACrB;IAEQ,kBAAkBL,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQ;YACR,GAAGA,IAAI;QACT;IACF;IAEQ,gCACNO,YAAoB,EACG;QACvB,IAAK,IAAIC,IAAID,eAAe,GAAGC,KAAK,GAAGA,IAAK;YAC1C,MAAMC,YAAY,IAAI,CAAC,KAAK,CAACD,EAAE;YAC/B,IAAI,AAACC,cAAaA,UAAU,OAAO,EAGnC;gBAAA,IAAIA,UAAU,SAAS,EACrB,OAAOA,UAAU,SAAS;YAC5B;QACF;IAEF;IAEA,MAAM,OAAOT,IAA+C,EAAiB;YAGbU,uBAAkCC;QAFhGC,OACE,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACX,CAAC,yDAAyD,EAAE,QAAAF,CAAAA,wBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,sBAAwB,KAAK,CAAC,EAAE,EAAE,QAAAC,CAAAA,yBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,uBAAwB,UAAU,EAAE;QAEpI,IAAIE,MAAM,OAAO,CAACb,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACc,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACd;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;IAElB;IAEA,MAAM,eACJA,IAA+C,EACS;QACxD,MAAM,IAAI,CAAC,MAAM,CAACA;QAClB,OAAO,IAAI,CAAC,KAAK;IACnB;IAEA,MAAM,QAAgE;QACpE,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDD,QAAQ,IAAI,CACV;QAIJa,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QACpCA,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM,EAAc;QAEhC,MAAMG,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACf,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIe,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMhB,OAAO,IAAI,CAAC,KAAK,CAACgB,UAAU;YAClCJ,OACEZ,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAOM,KAAK,GAAG;YACjB;YACA,IAAI;gBACFN,KAAK,MAAM,GAAG;gBACd,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOmB,GAAG;oBACVpB,QAAQ,KAAK,CAAC,wBAAwBoB;gBACxC;gBACAP,OACE;oBAAC;oBAAW;oBAAU;iBAAW,CAAC,OAAO,CAACZ,KAAK,IAAI,KAAK,GACxD,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEoB,QAAQ,EAAEC,KAAK,EAAE,GAAGrB;gBAC5BY,OAAOQ,UAAU,CAAC,oCAAoC,EAAEpB,KAAK,IAAI,EAAE;gBAEnE,IAAIsB;gBACJ,IAAIzB;gBACJ,IAAIG,KAAK,OAAO,EAAE;oBAChBH,YAAY,IAAI,CAAC,+BAA+B,CAACmB;oBACjDJ,OACEf,WACA;gBAEJ,OACEA,YAAY,MAAM,IAAI,CAAC,gBAAgB;gBAEzCG,KAAK,SAAS,GAAGH;gBACjB,MAAM0B,kBAAmC;oBACvCvB;oBACA,SAASkB,QAAAA,qBAAAA,KAAAA,IAAAA,mBAAoB,OAAO;oBACpCrB;gBACF;gBAEA,IAAIG,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BY,OACEZ,AAAiB,aAAjBA,KAAK,OAAO,IACVA,AAAiB,YAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;oBACzC,IAAIvB,AAAiB,aAAjBA,KAAK,OAAO,EACdkB,qBACEI,QAAAA,cAAAA,KAAAA,IAAAA,YACC,MAAM;gBAEb,OAAO,IAAItB,AAAc,aAAdA,KAAK,IAAI,IAAiBA,AAAc,eAAdA,KAAK,IAAI,EAC5CsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;qBACpC;oBACLxB,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEC,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;gBAC3C;gBAEA,MAAMC,aAAaR,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;gBAErD,IAAIQ,YAAY;oBACd,MAAMpB,aAAa,MAAM,IAAI,CAAC,iBAAiB;oBAC/C,IAAI,CAAC,kBAAkB,CAACJ,MAAMI,YAAY;gBAC5C;gBAEAqB,OAAO,MAAM,CAACzB,MAAMsB;gBACpBtB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGM,KAAK,GAAG;gBAC1BN,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtDgB;YACF,EAAE,OAAOG,GAAQ;gBACfF,wBAAwB;gBACxBjB,KAAK,KAAK,GAAGmB;gBACbnB,KAAK,YAAY,GACfmB,AAAAA,CAAAA,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO,AAAD,KAAM,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEnB,KAAK,UAAU,GAAGmB,EAAE,KAAK;gBAEzBnB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGM,KAAK,GAAG;gBAC1BN,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD;YACF;QACF;QAGA,IAAK,IAAIQ,IAAIQ,YAAY,GAAGR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAGzB,IAAIS,uBACF,IAAI,CAAC,MAAM,GAAG;aAEd,IAAI,CAAC,MAAM,GAAG;QAGhB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMS,cAAcC,KAAK,GAAG,CAACX,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEY,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAET,MAAMC,iBAAiB,IAAI,CAAC,KAAK,CAAC,SAAS,CACzC,CAAC9B,OAASA,AAAgB,aAAhBA,KAAK,MAAM;QAEvB,IAAI8B,kBAAkB,GACpB,OAAO,IAAI,CAAC,KAAK,CAACA,eAAe;QAEnC,OAAO;IACT;IAEA,OAAsB;QACpB,MAAMC,WAA0B;YAC9B,SAASzB,KAAK,GAAG;YACjB,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;QACA,OAAOyB;IACT;IAEA,MAAM,gBAAgBC,QAAgB,EAGnC;QACD,MAAMC,YAAgE;YACpE,MAAM;YACN,SAAS;YACT,OAAO;gBACL,SAASD;YACX;YACA,SAASA;YACT,QAAQ;YACR,UAAU;gBACR,MAAM,IAAIE,MAAMF,YAAY;YAC9B;QACF;QACA,MAAM,IAAI,CAAC,cAAc,CAACC;QAE1B,OAAO;YACL,QAAQE;YACR,QAAQ,IAAI;QACd;IACF;IAlSA,YACEC,IAAY,EACZC,gBAA0C,EAC1CC,OAEC,CACD;QAjBF;QAEA;QAGA;QAEA;QAEA,uBAAiB,oBAAjB;QASE,IAAI,CAAC,MAAM,GACTA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAKA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAGF;QACZ,IAAI,CAAC,KAAK,GAAIE,AAAAA,CAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAK,EAAC,EAAG,GAAG,CAAC,CAACxB,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGwB,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,WAAW;QACvC,IAAI,CAAC,gBAAgB,GAAGD;IAC1B;AAoRF"}
+{"version":3,"file":"task-runner.mjs","sources":["webpack://@midscene/core/./src/task-runner.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionRecorderItem,\n  ExecutionTask,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n  PlanningActionParamError,\n  UIContext,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\n\nconst debug = getDebug('task-runner');\nconst UI_CONTEXT_CACHE_TTL_MS = 300;\n\nexport class TaskRunner {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of runner\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly uiContextBuilder: () => Promise<UIContext>;\n\n  constructor(\n    name: string,\n    uiContextBuilder: () => Promise<UIContext>,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n    this.uiContextBuilder = uiContextBuilder;\n  }\n\n  private lastUiContext?: {\n    context: UIContext;\n    capturedAt: number;\n  };\n\n  private async getUiContext(options?: { forceRefresh?: boolean }): Promise<\n    UIContext | undefined\n  > {\n    const now = Date.now();\n    const shouldReuse =\n      !options?.forceRefresh &&\n      this.lastUiContext &&\n      now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;\n\n    if (shouldReuse && this.lastUiContext?.context) {\n      debug(\n        `reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`,\n      );\n      return this.lastUiContext?.context;\n    }\n\n    try {\n      const uiContext = await this.uiContextBuilder();\n      if (uiContext) {\n        this.lastUiContext = {\n          context: uiContext,\n          capturedAt: Date.now(),\n        };\n      } else {\n        this.lastUiContext = undefined;\n      }\n      return uiContext;\n    } catch (error) {\n      this.lastUiContext = undefined;\n      throw error;\n    }\n  }\n\n  private async captureScreenshot(): Promise<string | undefined> {\n    try {\n      const uiContext = await this.getUiContext({ forceRefresh: true });\n      return uiContext?.screenshotBase64;\n    } catch (error) {\n      console.error('error while capturing screenshot', error);\n    }\n    return undefined;\n  }\n\n  private attachRecorderItem(\n    task: ExecutionTask,\n    contextOrScreenshot: UIContext | string | undefined,\n    phase: 'after-calling',\n  ): void {\n    const timing = phase;\n    const screenshot =\n      typeof contextOrScreenshot === 'string'\n        ? contextOrScreenshot\n        : contextOrScreenshot?.screenshotBase64;\n    if (!timing || !screenshot) {\n      return;\n    }\n\n    const recorderItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot,\n      timing,\n    };\n\n    if (!task.recorder) {\n      task.recorder = [recorderItem];\n      return;\n    }\n    task.recorder.push(recorderItem);\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  private findPreviousNonSubTaskUIContext(\n    currentIndex: number,\n  ): UIContext | undefined {\n    for (let i = currentIndex - 1; i >= 0; i--) {\n      const candidate = this.tasks[i];\n      if (!candidate || candidate.subTask) {\n        continue;\n      }\n      if (candidate.uiContext) {\n        return candidate.uiContext;\n      }\n    }\n    return undefined;\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `task runner is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async appendAndFlush(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    await this.append(task);\n    return this.flush();\n  }\n\n  async flush(): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for task runner, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'task runner is already running');\n    assert(this.status !== 'completed', 'task runner is already completed');\n    assert(this.status !== 'error', 'task runner is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        let uiContext: UIContext | undefined;\n        if (task.subTask) {\n          uiContext = this.findPreviousNonSubTaskUIContext(taskIndex);\n          assert(\n            uiContext,\n            'subTask requires uiContext from previous non-subTask task',\n          );\n        } else {\n          uiContext = await this.getUiContext();\n        }\n        task.uiContext = uiContext;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n          uiContext,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'WaitFor' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported service subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        const isLastTask = taskIndex === this.tasks.length - 1;\n\n        if (isLastTask) {\n          const screenshot = await this.captureScreenshot();\n          this.attachRecorderItem(task, screenshot, 'after-calling');\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n\n  async appendErrorPlan(errorMsg: string): Promise<{\n    output: undefined;\n    runner: TaskRunner;\n  }> {\n    const errorTask: ExecutionTaskActionApply<PlanningActionParamError> = {\n      type: 'Action',\n      subType: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      thought: errorMsg,\n      locate: null,\n      executor: async () => {\n        throw new Error(errorMsg || 'error without thought');\n      },\n    };\n    await this.appendAndFlush(errorTask);\n\n    return {\n      output: undefined,\n      runner: this,\n    };\n  }\n}\n"],"names":["debug","getDebug","UI_CONTEXT_CACHE_TTL_MS","TaskRunner","options","_this_lastUiContext","now","Date","shouldReuse","_this_lastUiContext1","uiContext","undefined","error","console","task","contextOrScreenshot","phase","timing","screenshot","recorderItem","currentIndex","i","candidate","_this_latestErrorTask","_this_latestErrorTask1","assert","Array","item","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","e","executor","param","returnValue","executorContext","isLastTask","Object","outputIndex","Math","thought","output","errorTaskIndex","dumpData","errorMsg","errorTask","Error","name","uiContextBuilder"],"mappings":";;;;;;;;;;;;AAgBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,0BAA0B;AAEzB,MAAMC;IAkCX,MAAc,aAAaC,OAAoC,EAE7D;YAOmBC;QANnB,MAAMC,MAAMC,KAAK,GAAG;QACpB,MAAMC,cACJ,CAACJ,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,YAAY,AAAD,KACrB,IAAI,CAAC,aAAa,IAClBE,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,IAAIJ;QAEzC,IAAIM,eAAAA,SAAeH,CAAAA,sBAAAA,IAAI,CAAC,aAAa,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,OAAO,AAAD,GAAG;gBAIvCI;YAHPT,MACE,CAAC,gCAAgC,EAAEM,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAEhF,OAAO,QAAAG,CAAAA,uBAAAA,IAAI,CAAC,aAAa,AAAD,IAAjBA,KAAAA,IAAAA,qBAAoB,OAAO;QACpC;QAEA,IAAI;YACF,MAAMC,YAAY,MAAM,IAAI,CAAC,gBAAgB;YAC7C,IAAIA,WACF,IAAI,CAAC,aAAa,GAAG;gBACnB,SAASA;gBACT,YAAYH,KAAK,GAAG;YACtB;iBAEA,IAAI,CAAC,aAAa,GAAGI;YAEvB,OAAOD;QACT,EAAE,OAAOE,OAAO;YACd,IAAI,CAAC,aAAa,GAAGD;YACrB,MAAMC;QACR;IACF;IAEA,MAAc,oBAAiD;QAC7D,IAAI;YACF,MAAMF,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;gBAAE,cAAc;YAAK;YAC/D,OAAOA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,gBAAgB;QACpC,EAAE,OAAOE,OAAO;YACdC,QAAQ,KAAK,CAAC,oCAAoCD;QACpD;IAEF;IAEQ,mBACNE,IAAmB,EACnBC,mBAAmD,EACnDC,KAAsB,EAChB;QACN,MAAMC,SAASD;QACf,MAAME,aACJ,AAA+B,YAA/B,OAAOH,sBACHA,sBACAA,QAAAA,sBAAAA,KAAAA,IAAAA,oBAAqB,gBAAgB;QAC3C,IAAI,CAACE,UAAU,CAACC,YACd;QAGF,MAAMC,eAAsC;YAC1C,MAAM;YACN,IAAIZ,KAAK,GAAG;YACZW;YACAD;QACF;QAEA,IAAI,CAACH,KAAK,QAAQ,EAAE;YAClBA,KAAK,QAAQ,GAAG;gBAACK;aAAa;YAC9B;QACF;QACAL,KAAK,QAAQ,CAAC,IAAI,CAACK;IACrB;IAEQ,kBAAkBL,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQ;YACR,GAAGA,IAAI;QACT;IACF;IAEQ,gCACNM,YAAoB,EACG;QACvB,IAAK,IAAIC,IAAID,eAAe,GAAGC,KAAK,GAAGA,IAAK;YAC1C,MAAMC,YAAY,IAAI,CAAC,KAAK,CAACD,EAAE;YAC/B,IAAI,AAACC,cAAaA,UAAU,OAAO,EAGnC;gBAAA,IAAIA,UAAU,SAAS,EACrB,OAAOA,UAAU,SAAS;YAC5B;QACF;IAEF;IAEA,MAAM,OAAOR,IAA+C,EAAiB;YAGbS,uBAAkCC;QAFhGC,OACE,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACX,CAAC,yDAAyD,EAAE,QAAAF,CAAAA,wBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,sBAAwB,KAAK,CAAC,EAAE,EAAE,QAAAC,CAAAA,yBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,uBAAwB,UAAU,EAAE;QAEpI,IAAIE,MAAM,OAAO,CAACZ,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACa,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACb;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;IAElB;IAEA,MAAM,eACJA,IAA+C,EACS;QACxD,MAAM,IAAI,CAAC,MAAM,CAACA;QAClB,OAAO,IAAI,CAAC,KAAK;IACnB;IAEA,MAAM,QAAgE;QACpE,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDD,QAAQ,IAAI,CACV;QAIJY,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QACpCA,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM,EAAc;QAEhC,MAAMG,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACd,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIc,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMf,OAAO,IAAI,CAAC,KAAK,CAACe,UAAU;YAClCJ,OACEX,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAOP,KAAK,GAAG;YACjB;YACA,IAAI;gBACFO,KAAK,MAAM,GAAG;gBACd,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOkB,GAAG;oBACVnB,QAAQ,KAAK,CAAC,wBAAwBmB;gBACxC;gBACAP,OACE;oBAAC;oBAAW;oBAAU;iBAAW,CAAC,OAAO,CAACX,KAAK,IAAI,KAAK,GACxD,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEmB,QAAQ,EAAEC,KAAK,EAAE,GAAGpB;gBAC5BW,OAAOQ,UAAU,CAAC,oCAAoC,EAAEnB,KAAK,IAAI,EAAE;gBAEnE,IAAIqB;gBACJ,IAAIzB;gBACJ,IAAII,KAAK,OAAO,EAAE;oBAChBJ,YAAY,IAAI,CAAC,+BAA+B,CAACmB;oBACjDJ,OACEf,WACA;gBAEJ,OACEA,YAAY,MAAM,IAAI,CAAC,YAAY;gBAErCI,KAAK,SAAS,GAAGJ;gBACjB,MAAM0B,kBAAmC;oBACvCtB;oBACA,SAASiB,QAAAA,qBAAAA,KAAAA,IAAAA,mBAAoB,OAAO;oBACpCrB;gBACF;gBAEA,IAAII,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BW,OACEX,AAAiB,aAAjBA,KAAK,OAAO,IACVA,AAAiB,YAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDqB,cAAc,MAAMrB,KAAK,QAAQ,CAACoB,OAAOE;oBACzC,IAAItB,AAAiB,aAAjBA,KAAK,OAAO,EACdiB,qBACEI,QAAAA,cAAAA,KAAAA,IAAAA,YACC,MAAM;gBAEb,OAAO,IAAIrB,AAAc,aAAdA,KAAK,IAAI,IAAiBA,AAAc,eAAdA,KAAK,IAAI,EAC5CqB,cAAc,MAAMrB,KAAK,QAAQ,CAACoB,OAAOE;qBACpC;oBACLvB,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEC,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEqB,cAAc,MAAMrB,KAAK,QAAQ,CAACoB,OAAOE;gBAC3C;gBAEA,MAAMC,aAAaR,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;gBAErD,IAAIQ,YAAY;oBACd,MAAMnB,aAAa,MAAM,IAAI,CAAC,iBAAiB;oBAC/C,IAAI,CAAC,kBAAkB,CAACJ,MAAMI,YAAY;gBAC5C;gBAEAoB,OAAO,MAAM,CAACxB,MAAMqB;gBACpBrB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGP,KAAK,GAAG;gBAC1BO,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtDe;YACF,EAAE,OAAOG,GAAQ;gBACfF,wBAAwB;gBACxBhB,KAAK,KAAK,GAAGkB;gBACblB,KAAK,YAAY,GACfkB,AAAAA,CAAAA,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO,AAAD,KAAM,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnElB,KAAK,UAAU,GAAGkB,EAAE,KAAK;gBAEzBlB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGP,KAAK,GAAG;gBAC1BO,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD;YACF;QACF;QAGA,IAAK,IAAIO,IAAIQ,YAAY,GAAGR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAGzB,IAAIS,uBACF,IAAI,CAAC,MAAM,GAAG;aAEd,IAAI,CAAC,MAAM,GAAG;QAGhB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMS,cAAcC,KAAK,GAAG,CAACX,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEY,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAET,MAAMC,iBAAiB,IAAI,CAAC,KAAK,CAAC,SAAS,CACzC,CAAC7B,OAASA,AAAgB,aAAhBA,KAAK,MAAM;QAEvB,IAAI6B,kBAAkB,GACpB,OAAO,IAAI,CAAC,KAAK,CAACA,eAAe;QAEnC,OAAO;IACT;IAEA,OAAsB;QACpB,MAAMC,WAA0B;YAC9B,SAASrC,KAAK,GAAG;YACjB,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;QACA,OAAOqC;IACT;IAEA,MAAM,gBAAgBC,QAAgB,EAGnC;QACD,MAAMC,YAAgE;YACpE,MAAM;YACN,SAAS;YACT,OAAO;gBACL,SAASD;YACX;YACA,SAASA;YACT,QAAQ;YACR,UAAU;gBACR,MAAM,IAAIE,MAAMF,YAAY;YAC9B;QACF;QACA,MAAM,IAAI,CAAC,cAAc,CAACC;QAE1B,OAAO;YACL,QAAQnC;YACR,QAAQ,IAAI;QACd;IACF;IAxUA,YACEqC,IAAY,EACZC,gBAA0C,EAC1C7C,OAEC,CACD;QAjBF;QAEA;QAGA;QAEA;QAEA,uBAAiB,oBAAjB;QAmBA,uBAAQ,iBAAR;QAVE,IAAI,CAAC,MAAM,GACTA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAKA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAG4C;QACZ,IAAI,CAAC,KAAK,GAAI5C,AAAAA,CAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAK,EAAC,EAAG,GAAG,CAAC,CAACuB,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGvB,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,WAAW;QACvC,IAAI,CAAC,gBAAgB,GAAG6C;IAC1B;AA0TF"}

package/dist/es/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"types.mjs","sources":["webpack://@midscene/core/./src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type {\n  CreateOpenAIClientFn,\n  TModelConfigFn,\n} from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './ai-model/common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\nexport type { LocateResultElement };\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  abstract screenshotBase64: string;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n}\n\nexport class ServiceError extends Error {\n  dump: ServiceDump;\n\n  constructor(message: string, dump: ServiceDump) {\n    super(message);\n    this.name = 'ServiceError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  subTask?: boolean;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActionContext?: string;\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<T = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<T>;\n  call: (param: T, context: ExecutorContext) => Promise<void> | void;\n}\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext;\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfigFn;\n  cache?: Cache;\n  replanningCycleLimit?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["AIResponseFormat","UIContext","ServiceError","Error","message","dump"],"mappings":";AAAqD;;;;;;;;;;AA0C9C,IAAKA,yBAAgBA,WAAAA,GAAAA,SAAhBA,gBAAgB;;;WAAhBA;;AA2EL,MAAeC;AAMtB;AAsEO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF"}
+{"version":3,"file":"types.mjs","sources":["webpack://@midscene/core/./src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type {\n  CreateOpenAIClientFn,\n  TModelConfigFn,\n} from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './ai-model/common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\nexport type { LocateResultElement };\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  abstract screenshotBase64: string;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n}\n\nexport class ServiceError extends Error {\n  dump: ServiceDump;\n\n  constructor(message: string, dump: ServiceDump) {\n    super(message);\n    this.name = 'ServiceError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  subTask?: boolean;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActionContext?: string;\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<T = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<T>;\n  /** Optional delay (ms) after runner execution. This is useful to wait for the UI to update. Defaults to 300ms. Set to 0 to skip. */\n  delayAfterRunner?: number;\n  call: (param: T, context: ExecutorContext) => Promise<void> | void;\n}\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext;\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfigFn;\n  cache?: Cache;\n  replanningCycleLimit?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["AIResponseFormat","UIContext","ServiceError","Error","message","dump"],"mappings":";AAAqD;;;;;;;;;;AA0C9C,IAAKA,yBAAgBA,WAAAA,GAAAA,SAAhBA,gBAAgB;;;WAAhBA;;AA0EL,MAAeC;AAMtB;AAsEO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF"}