@midscene/core 1.8.10 → 1.9.0

package/dist/lib/device/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"device/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/device/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getMidsceneLocationSchema } from '@/common';\nimport type {\n  ActionScrollParam,\n  DeviceAction,\n  LocateResultElement,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport interface FileChooserHandler {\n  accept(files: string[]): Promise<void>;\n}\n\nexport interface MjpegStreamFrame {\n  /** Raw base64-encoded image bytes WITHOUT a `data:image/...;base64,` prefix. */\n  data: string;\n  contentType?: string;\n}\n\nexport interface MjpegStreamHandle {\n  stop(): void | Promise<void>;\n}\n\nexport interface MjpegStreamOptions {\n  signal?: AbortSignal;\n  onFrame(frame: MjpegStreamFrame): void;\n  onError?(error: unknown): void;\n}\n\n/** A point in device-pixel coordinates on the screen. */\nexport interface PointerPoint {\n  x: number;\n  y: number;\n}\n\nexport interface PointerInputPrimitives {\n  tap(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n  doubleClick?(p: PointerPoint): Promise<void>;\n  rightClick?(p: PointerPoint): Promise<void>;\n  hover?(p: PointerPoint): Promise<void>;\n  longPress?(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n  dragAndDrop?(from: PointerPoint, to: PointerPoint): Promise<void>;\n}\n\nexport interface TouchInputPrimitives {\n  swipe(\n    start: PointerPoint,\n    end: PointerPoint,\n    opts?: { duration?: number; repeat?: number },\n  ): Promise<void>;\n  pinch?(\n    center: PointerPoint,\n    opts: { startDistance: number; endDistance: number; duration: number },\n  ): Promise<void>;\n}\n\nexport interface KeyboardInputPrimitives {\n  keyboardPress(keyName: string, opts?: { target?: unknown }): Promise<void>;\n  cursorMove?(direction: 'left' | 'right', times?: number): Promise<void>;\n  typeText(\n    value: string,\n    opts?: {\n      autoDismissKeyboard?: boolean;\n      keyboardDismissStrategy?: 'esc-first' | 'back-first';\n      target?: unknown;\n      replace?: boolean;\n      focusOnly?: boolean;\n    },\n  ): Promise<void>;\n  clearInput(target?: unknown): Promise<void>;\n}\n\nexport interface ScrollInputPrimitives {\n  scroll(param: ActionScrollParam): Promise<void>;\n}\n\nexport interface SystemInputPrimitives {\n  backButton?(): Promise<void>;\n  homeButton?(): Promise<void>;\n  recentAppsButton?(): Promise<void>;\n}\n\nexport interface InputPrimitives {\n  pointer?: PointerInputPrimitives;\n  keyboard?: KeyboardInputPrimitives;\n  touch?: TouchInputPrimitives;\n  scroll?: ScrollInputPrimitives;\n  system?: SystemInputPrimitives;\n}\n\nexport interface MobileInputPrimitives extends InputPrimitives {\n  pointer: PointerInputPrimitives & {\n    doubleClick(p: PointerPoint): Promise<void>;\n    longPress(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n    dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n  };\n  keyboard: KeyboardInputPrimitives;\n  touch: TouchInputPrimitives;\n}\n\nexport interface BrowserInputPrimitives extends InputPrimitives {\n  pointer: PointerInputPrimitives & {\n    doubleClick(p: PointerPoint): Promise<void>;\n    rightClick(p: PointerPoint): Promise<void>;\n    hover(p: PointerPoint): Promise<void>;\n    dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n    longPress(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n  };\n  keyboard: KeyboardInputPrimitives;\n  scroll: ScrollInputPrimitives;\n  touch: TouchInputPrimitives;\n}\n\nexport interface ComputerInputPrimitives extends InputPrimitives {\n  pointer: PointerInputPrimitives & {\n    doubleClick(p: PointerPoint): Promise<void>;\n    rightClick(p: PointerPoint): Promise<void>;\n    hover(p: PointerPoint): Promise<void>;\n    dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n  };\n  keyboard: KeyboardInputPrimitives;\n  scroll: ScrollInputPrimitives;\n}\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[];\n\n  abstract cacheFeatureForPoint?(\n    center: [number, number],\n    options?: {\n      targetDescription?: string;\n      modelConfig?: IModelConfig;\n    },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // for web only\n  registerFileChooserListener?(\n    handler: (chooser: FileChooserHandler) => Promise<void>,\n  ): Promise<{ dispose: () => void; getError: () => Error | undefined }>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  /**\n   * Get the current device-local time as a formatted string.\n   * Prefer this for user-visible time because timestamps alone do not preserve\n   * the target device's timezone when formatted on the host machine.\n   */\n  getDeviceLocalTimeString?(format?: string): Promise<string>;\n\n  /** URL of native MJPEG stream for real-time screen preview (e.g. WDA MJPEG server) */\n  mjpegStreamUrl?: string;\n\n  /**\n   * Optional in-process MJPEG frame producer. Implementations can push raw\n   * base64 frames here when there is no standalone native MJPEG URL, e.g.\n   * Chromium CDP Page.startScreencast for web previews.\n   */\n  startMjpegStream?(\n    options: MjpegStreamOptions,\n  ): MjpegStreamHandle | undefined | Promise<MjpegStreamHandle | undefined>;\n\n  /**\n   * Optional hook used after keyboard-only actions to force a fresh frame on\n   * the active MJPEG stream. Implementations should be a no-op when no stream\n   * is active.\n   */\n  flushPendingVisualUpdate?(): Promise<void>;\n\n  /**\n   * Optional navigation state probe for browser-like interfaces, used to drive\n   * loading indicators in playground UIs. Returning `undefined` means the\n   * interface does not expose this concept.\n   */\n  navigationState?(): Promise<{ isLoading: boolean }>;\n\n  /**\n   * Low-level device input surface. Platform implementations expose transport\n   * primitives here; higher-level AI actions and manual pointer dispatch should\n   * adapt to this instead of duplicating platform gesture logic.\n   */\n  inputPrimitives?: InputPrimitives;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\n// TReturn allows specifying the return type of the action\nexport const defineAction = <\n  TSchema extends z.ZodType | undefined = undefined,\n  TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined,\n  TReturn = any,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema?: TSchema;\n    call: (param: TRuntime) => Promise<TReturn> | TReturn;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime, TReturn>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime, TReturn> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\nfunction pointFromLocate(\n  locate: LocateResultElement | undefined,\n  missingMessage: string,\n): PointerPoint {\n  if (!locate) {\n    throw new Error(missingMessage);\n  }\n  return { x: locate.center[0], y: locate.center[1] };\n}\n\nfunction defineLocatedPointAction<\n  TSchema extends z.ZodType,\n  TParam extends { locate: LocateResultElement },\n>(config: {\n  name: string;\n  description: string;\n  interfaceAlias?: string;\n  paramSchema: TSchema;\n  sample: DeviceAction<TParam>['sample'];\n  missingLocateMessage: string;\n  call: (point: PointerPoint, param: TParam) => Promise<void>;\n}): DeviceAction<TParam> {\n  return defineAction<TSchema, TParam>({\n    name: config.name,\n    description: config.description,\n    interfaceAlias: config.interfaceAlias,\n    paramSchema: config.paramSchema,\n    sample: config.sample,\n    call: async (param) => {\n      await config.call(\n        pointFromLocate(param.locate, config.missingLocateMessage),\n        param,\n      );\n    },\n  });\n}\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  tap: PointerInputPrimitives['tap'],\n): DeviceAction<ActionTapParam> => {\n  return defineLocatedPointAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    sample: {\n      locate: { prompt: 'the \"Submit\" button' },\n    },\n    missingLocateMessage: 'Element not found, cannot tap',\n    call: async (point) => {\n      await tap(point);\n    },\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  rightClick: NonNullable<PointerInputPrimitives['rightClick']>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineLocatedPointAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    sample: {\n      locate: { prompt: 'the file icon on the desktop' },\n    },\n    missingLocateMessage: 'Element not found, cannot right click',\n    call: async (point) => {\n      await rightClick(point);\n    },\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  doubleClick: NonNullable<PointerInputPrimitives['doubleClick']>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineLocatedPointAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    sample: {\n      locate: { prompt: 'the folder icon' },\n    },\n    missingLocateMessage: 'Element not found, cannot double click',\n    call: async (point) => {\n      await doubleClick(point);\n    },\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  hover: NonNullable<PointerInputPrimitives['hover']>,\n): DeviceAction<ActionHoverParam> => {\n  return defineLocatedPointAction<\n    typeof actionHoverParamSchema,\n    ActionHoverParam\n  >({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    sample: {\n      locate: { prompt: 'the navigation menu item \"Products\"' },\n    },\n    missingLocateMessage: 'Element not found, cannot hover',\n    call: async (point) => {\n      await hover(point);\n    },\n  });\n};\n\n// Input\nconst inputLocateDescription =\n  'the position of the placeholder or text content in the target input field. If there is no content, locate the center of the input field.';\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace mode, only the inserted characters for typeOnly mode, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe(inputLocateDescription)\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'typeOnly'])\n    .default('replace')\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"typeOnly\" - type the value directly without clearing the field first, and should be set explicitly for incremental edits after moving the cursor; \"clear\" - clear the field without inputting new text.',\n    ),\n  autoDismissKeyboard: z\n    .boolean()\n    .optional()\n    .describe(\n      'If true, the keyboard will be dismissed after the input is completed. Do not set it unless the user asks you to do so.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'typeOnly' | 'append';\n  autoDismissKeyboard?: boolean;\n};\n\nexport const defineActionInput = (\n  keyboard: KeyboardInputPrimitives,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    sample: {\n      value: 'test@example.com',\n      locate: { prompt: 'the email input field' },\n    },\n    call: async (param) => {\n      // backward compat: convert deprecated 'append' to 'typeOnly'\n      if ((param.mode as string) === 'append') {\n        param.mode = 'typeOnly';\n      }\n\n      if (param.mode === 'clear') {\n        await keyboard.clearInput(param.locate);\n        return;\n      }\n\n      if (!param || !param.value) {\n        return;\n      }\n\n      await keyboard.typeText(param.value, {\n        target: param.locate,\n        replace: param.mode !== 'typeOnly',\n        autoDismissKeyboard: param.autoDismissKeyboard,\n      });\n    },\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  keyboardPress: KeyboardInputPrimitives['keyboardPress'],\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    sample: {\n      keyName: 'Enter',\n    },\n    call: async (param) => {\n      await keyboardPress(param.keyName, {\n        target: param.locate,\n      });\n    },\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  scrollType: z\n    .enum([\n      'singleAction',\n      'scrollToBottom',\n      'scrollToTop',\n      'scrollToRight',\n      'scrollToLeft',\n    ])\n    .default('singleAction')\n    .describe(\n      'The scroll behavior: \"singleAction\" for a single scroll action, \"scrollToBottom\" for scrolling all the way to the bottom by rapidly scrolling 5-10 times (skipping intermediate content until reaching the bottom), \"scrollToTop\" for scrolling all the way to the top by rapidly scrolling 5-10 times (skipping intermediate content until reaching the top), \"scrollToRight\" for scrolling all the way to the right by rapidly scrolling multiple times, \"scrollToLeft\" for scrolling all the way to the left by rapidly scrolling multiple times',\n    ),\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe(\n      'The direction to scroll. Only effective when scrollType is \"singleAction\".',\n    ),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Describe the target element to be scrolled on, like \"the table\" or \"the list\" or \"the content area\" or \"the scrollable area\". Do NOT provide a general intent like \"scroll to find some element\"',\n    ),\n});\n\nexport const defineActionScroll = (\n  scroll: ScrollInputPrimitives['scroll'],\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or a scrollable element to browse content. This is the preferred way to scroll on all platforms, including mobile. Supports scrollToBottom/scrollToTop for boundary navigation. Default: direction `down`, scrollType `singleAction`, distance `null`.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    sample: {\n      direction: 'down',\n      scrollType: 'singleAction',\n      locate: { prompt: 'the center of the product list area' },\n    },\n    call: async (param) => {\n      await scroll(param);\n    },\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  dragAndDrop: NonNullable<PointerInputPrimitives['dragAndDrop']>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description:\n      'Pick up a specific UI element and move it to a new position (e.g., reorder a card, move a file into a folder, sort list items). The element itself moves with your finger/mouse.',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    sample: {\n      from: { prompt: 'the \"report.pdf\" file icon' },\n      to: { prompt: 'the upload drop zone' },\n    },\n    call: async (param) => {\n      const from = param.from;\n      const to = param.to;\n      if (!from) {\n        throw new Error('missing \"from\" param for drag and drop');\n      }\n      if (!to) {\n        throw new Error('missing \"to\" param for drag and drop');\n      }\n      await dragAndDrop(\n        { x: from.center[0], y: from.center[1] },\n        { x: to.center[0], y: to.center[1] },\n      );\n    },\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  longPress: NonNullable<PointerInputPrimitives['longPress']>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineLocatedPointAction<\n    typeof ActionLongPressParamSchema,\n    ActionLongPressParam\n  >({\n    name: 'LongPress',\n    description: 'Long press the element',\n    interfaceAlias: 'aiLongPress',\n    paramSchema: ActionLongPressParamSchema,\n    sample: {\n      locate: { prompt: 'the message bubble' },\n    },\n    missingLocateMessage: 'LongPress requires an element to be located',\n    call: async (point, param) => {\n      await longPress(point, { duration: param.duration });\n    },\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport function normalizeMobileSwipeParam(\n  param: ActionSwipeParam,\n  screenSize: { width: number; height: number },\n): {\n  startPoint: { x: number; y: number };\n  endPoint: { x: number; y: number };\n  duration: number;\n  repeatCount: number;\n} {\n  const { width, height } = screenSize;\n  const { start, end } = param;\n\n  const startPoint = start\n    ? { x: start.center[0], y: start.center[1] }\n    : { x: width / 2, y: height / 2 };\n\n  let endPoint: { x: number; y: number };\n\n  if (end) {\n    endPoint = { x: end.center[0], y: end.center[1] };\n  } else if (param.distance) {\n    const direction = param.direction;\n    if (!direction) {\n      throw new Error('direction is required for swipe gesture');\n    }\n    endPoint = {\n      x:\n        startPoint.x +\n        (direction === 'right'\n          ? param.distance\n          : direction === 'left'\n            ? -param.distance\n            : 0),\n      y:\n        startPoint.y +\n        (direction === 'down'\n          ? param.distance\n          : direction === 'up'\n            ? -param.distance\n            : 0),\n    };\n  } else {\n    throw new Error(\n      'Either end or distance must be specified for swipe gesture',\n    );\n  }\n\n  endPoint.x = Math.max(0, Math.min(endPoint.x, width));\n  endPoint.y = Math.max(0, Math.min(endPoint.y, height));\n\n  const duration = param.duration ?? 300;\n\n  let repeatCount = typeof param.repeat === 'number' ? param.repeat : 1;\n  if (repeatCount === 0) {\n    repeatCount = 10;\n  }\n\n  return { startPoint, endPoint, duration, repeatCount };\n}\n\nexport const defineActionSwipe = (config: {\n  swipe: TouchInputPrimitives['swipe'];\n  size(): Promise<Size>;\n}): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a touch gesture for interactions beyond regular scrolling (e.g., flip pages in a carousel, dismiss a notification, swipe-to-delete a list item). For regular content scrolling, use Scroll instead. Use \"distance\" + \"direction\" for relative movement, or \"end\" for precise endpoint.',\n    paramSchema: ActionSwipeParamSchema,\n    sample: {\n      start: { prompt: 'center of the notification' },\n      end: { prompt: 'upper edge of the screen' },\n    },\n    call: async (param) => {\n      const { startPoint, endPoint, duration, repeatCount } =\n        normalizeMobileSwipeParam(param, await config.size());\n      for (let i = 0; i < repeatCount; i++) {\n        await config.swipe(startPoint, endPoint, { duration });\n      }\n    },\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The input field to be cleared')\n    .optional(),\n});\nexport type ActionClearInputParam = {\n  locate?: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  clearInput: KeyboardInputPrimitives['clearInput'],\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: inputLocateDescription,\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    sample: {\n      locate: { prompt: 'the search input field' },\n    },\n    call: async (param) => {\n      await clearInput(param.locate);\n    },\n  });\n};\n\n// CursorMove\nexport const actionCursorMoveParamSchema = z.object({\n  direction: z\n    .enum(['left', 'right'])\n    .describe('The direction to move the cursor'),\n  times: z\n    .number()\n    .int()\n    .min(1)\n    .default(1)\n    .describe(\n      'The number of times to move the cursor in the specified direction',\n    ),\n});\nexport type ActionCursorMoveParam = {\n  direction: 'left' | 'right';\n  times?: number;\n};\n\nexport const defineActionCursorMove = (config: {\n  keyboard: Pick<KeyboardInputPrimitives, 'keyboardPress' | 'cursorMove'>;\n  sleep?(timeMs: number): Promise<void>;\n}): DeviceAction<ActionCursorMoveParam> => {\n  return defineAction<\n    typeof actionCursorMoveParamSchema,\n    ActionCursorMoveParam\n  >({\n    name: 'CursorMove',\n    description:\n      'Move the text cursor (caret) left or right within an input field or text area. Use this to reposition the cursor without selecting text.',\n    paramSchema: actionCursorMoveParamSchema,\n    sample: {\n      direction: 'left',\n      times: 3,\n    },\n    call: async (param) => {\n      const times = param.times ?? 1;\n      if (config.keyboard.cursorMove) {\n        await config.keyboard.cursorMove(param.direction, times);\n        return;\n      }\n\n      const wait =\n        config.sleep ??\n        ((timeMs: number) =>\n          new Promise<void>((resolve) => setTimeout(resolve, timeMs)));\n      const arrowKey = param.direction === 'left' ? 'ArrowLeft' : 'ArrowRight';\n      for (let i = 0; i < times; i++) {\n        await config.keyboard.keyboardPress(arrowKey);\n        await wait(100);\n      }\n    },\n  });\n};\n\n// Pinch\nexport const ActionPinchParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'The element to pinch on. If not specified, the center of the screen will be used',\n    ),\n  direction: z\n    .enum(['in', 'out'])\n    .describe(\n      'Pinch direction. \"in\" = pinch fingers together (zoom out / shrink), \"out\" = spread fingers apart (zoom in / enlarge).',\n    ),\n  distance: z\n    .number()\n    .positive()\n    .optional()\n    .describe(\n      'How far each finger moves in pixels. Defaults to a quarter of the shorter screen dimension.',\n    ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Duration of the pinch gesture in milliseconds'),\n});\n\nexport type ActionPinchParam = {\n  locate?: LocateResultElement;\n  direction: 'in' | 'out';\n  distance?: number;\n  duration?: number;\n};\n\nexport const defineActionPinch = (config: {\n  pinch: TouchInputPrimitives['pinch'];\n  size(): Promise<Size>;\n}): DeviceAction<ActionPinchParam> | undefined => {\n  if (!config.pinch) {\n    return undefined;\n  }\n\n  return defineAction<typeof ActionPinchParamSchema, ActionPinchParam>({\n    name: 'Pinch',\n    description:\n      'Perform a two-finger pinch gesture. Use direction \"in\" to pinch fingers together (zoom out), or \"out\" to spread fingers apart (zoom in). Optionally specify distance for how far each finger moves.',\n    interfaceAlias: 'aiPinch',\n    paramSchema: ActionPinchParamSchema,\n    sample: {\n      locate: { prompt: 'the map area' },\n      direction: 'out',\n      distance: 200,\n    },\n    call: async (param) => {\n      const { centerX, centerY, startDistance, endDistance, duration } =\n        normalizePinchParam(param, await config.size());\n      await config.pinch?.(\n        { x: centerX, y: centerY },\n        { startDistance, endDistance, duration },\n      );\n    },\n  });\n};\n\nexport function normalizePinchParam(\n  param: ActionPinchParam,\n  screenSize: { width: number; height: number },\n): {\n  centerX: number;\n  centerY: number;\n  startDistance: number;\n  endDistance: number;\n  duration: number;\n} {\n  const { width, height } = screenSize;\n  const element = param.locate;\n  const centerX = element\n    ? Math.round(element.center[0])\n    : Math.round(width / 2);\n  const centerY = element\n    ? Math.round(element.center[1])\n    : Math.round(height / 2);\n  const duration = param.duration ?? 500;\n\n  const baseDistance = Math.round(Math.min(width, height) / 4);\n  const fingerDistance = param.distance ?? baseDistance;\n\n  const startDistance = baseDistance;\n  const endDistance =\n    param.direction === 'out'\n      ? baseDistance + fingerDistance\n      : Math.max(10, baseDistance - fingerDistance);\n\n  return { centerX, centerY, startDistance, endDistance, duration };\n}\n\nexport interface MobileInputActionContext {\n  input: MobileInputPrimitives;\n  size(): Promise<Size>;\n  sleep?(timeMs: number): Promise<void>;\n  getDefaultAutoDismissKeyboard?(): boolean | undefined;\n  systemActions?: SystemInputActionOptions;\n}\n\nexport interface SystemInputActionConfig {\n  name: string;\n  description: string;\n  interfaceAlias?: string;\n  delayBeforeRunner?: number;\n  delayAfterRunner?: number;\n}\n\nexport interface SystemInputActionOptions {\n  backButton?: SystemInputActionConfig;\n  homeButton?: SystemInputActionConfig;\n  recentAppsButton?: SystemInputActionConfig;\n}\n\nexport interface InputPrimitiveActionOptions {\n  size?: () => Promise<Size>;\n  sleep?: (timeMs: number) => Promise<void>;\n  includeSwipe?: boolean;\n  includePinch?: boolean;\n  systemActions?: SystemInputActionOptions;\n}\n\nfunction defineSystemInputAction(\n  config: SystemInputActionConfig,\n  call: () => Promise<void>,\n): DeviceAction<undefined, void> {\n  return defineAction<undefined, undefined, void>({\n    name: config.name,\n    description: config.description,\n    interfaceAlias: config.interfaceAlias,\n    delayBeforeRunner: config.delayBeforeRunner,\n    delayAfterRunner: config.delayAfterRunner,\n    call,\n  });\n}\n\nexport function defineActionsFromInputPrimitives(\n  input: InputPrimitives,\n  options: InputPrimitiveActionOptions = {},\n): DeviceAction<any>[] {\n  const actions: Array<DeviceAction<any> | undefined> = [];\n  const { pointer, keyboard, scroll, touch, system } = input;\n\n  if (pointer) {\n    actions.push(defineActionTap(pointer.tap));\n    if (pointer.doubleClick) {\n      actions.push(defineActionDoubleClick(pointer.doubleClick));\n    }\n    if (pointer.rightClick) {\n      actions.push(defineActionRightClick(pointer.rightClick));\n    }\n    if (pointer.hover) {\n      actions.push(defineActionHover(pointer.hover));\n    }\n    if (pointer.dragAndDrop) {\n      actions.push(defineActionDragAndDrop(pointer.dragAndDrop));\n    }\n    if (pointer.longPress) {\n      actions.push(defineActionLongPress(pointer.longPress));\n    }\n  }\n\n  if (keyboard) {\n    actions.push(\n      defineActionInput(keyboard),\n      defineActionClearInput(keyboard.clearInput),\n      defineActionKeyboardPress(keyboard.keyboardPress),\n      defineActionCursorMove({ keyboard, sleep: options.sleep }),\n    );\n  }\n\n  if (scroll) {\n    actions.push(defineActionScroll(scroll.scroll));\n  }\n\n  if (touch?.swipe && options.size && options.includeSwipe !== false) {\n    actions.push(defineActionSwipe({ swipe: touch.swipe, size: options.size }));\n  }\n\n  if (touch?.pinch && options.size && options.includePinch !== false) {\n    actions.push(defineActionPinch({ pinch: touch.pinch, size: options.size }));\n  }\n\n  if (system && options.systemActions) {\n    const { systemActions } = options;\n    if (system.backButton && systemActions.backButton) {\n      actions.push(\n        defineSystemInputAction(systemActions.backButton, system.backButton),\n      );\n    }\n    if (system.homeButton && systemActions.homeButton) {\n      actions.push(\n        defineSystemInputAction(systemActions.homeButton, system.homeButton),\n      );\n    }\n    if (system.recentAppsButton && systemActions.recentAppsButton) {\n      actions.push(\n        defineSystemInputAction(\n          systemActions.recentAppsButton,\n          system.recentAppsButton,\n        ),\n      );\n    }\n  }\n\n  return actions.filter((action): action is DeviceAction<any> =>\n    Boolean(action),\n  );\n}\n\nexport function createDefaultMobileActions(\n  context: MobileInputActionContext,\n): DeviceAction<any>[] {\n  return defineActionsFromInputPrimitives(context.input, {\n    size: context.size,\n    sleep: context.sleep,\n    systemActions: context.systemActions,\n  });\n}\n\n// Sleep\nexport const ActionSleepParamSchema = z.object({\n  timeMs: z\n    .number()\n    .default(1000)\n    .optional()\n    .describe('Sleep duration in milliseconds, defaults to 1000ms (1 second)'),\n});\n\nexport type ActionSleepParam = {\n  timeMs?: number;\n};\n\nexport const defineActionSleep = (): DeviceAction<ActionSleepParam> => {\n  return defineAction<typeof ActionSleepParamSchema, ActionSleepParam>({\n    name: 'Sleep',\n    description:\n      'Wait for a specified duration before continuing. Defaults to 1 second (1000ms) if not specified.',\n    paramSchema: ActionSleepParamSchema,\n    sample: {\n      timeMs: 2000,\n    },\n    call: async (param) => {\n      const duration = param?.timeMs ?? 1000;\n      getDebug('device:common-action')(`Sleeping for ${duration}ms`);\n      await new Promise((resolve) => setTimeout(resolve, duration));\n    },\n  });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n  AndroidDeviceOpt,\n  AndroidDeviceInputOpt,\n  IOSDeviceOpt,\n  IOSDeviceInputOpt,\n  HarmonyDeviceOpt,\n  HarmonyDeviceInputOpt,\n} from './device-options';\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","AbstractInterface","defineAction","config","pointFromLocate","locate","missingMessage","Error","defineLocatedPointAction","param","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","tap","point","actionRightClickParamSchema","defineActionRightClick","rightClick","actionDoubleClickParamSchema","defineActionDoubleClick","doubleClick","actionHoverParamSchema","defineActionHover","hover","inputLocateDescription","actionInputParamSchema","val","String","defineActionInput","keyboard","actionKeyboardPressParamSchema","defineActionKeyboardPress","keyboardPress","actionScrollParamSchema","defineActionScroll","scroll","actionDragAndDropParamSchema","defineActionDragAndDrop","dragAndDrop","from","to","ActionLongPressParamSchema","defineActionLongPress","longPress","ActionSwipeParamSchema","normalizeMobileSwipeParam","screenSize","width","height","start","end","startPoint","endPoint","direction","Math","duration","repeatCount","defineActionSwipe","i","actionClearInputParamSchema","defineActionClearInput","clearInput","actionCursorMoveParamSchema","defineActionCursorMove","times","wait","timeMs","Promise","resolve","setTimeout","arrowKey","ActionPinchParamSchema","defineActionPinch","centerX","centerY","startDistance","endDistance","normalizePinchParam","element","baseDistance","fingerDistance","defineSystemInputAction","call","defineActionsFromInputPrimitives","input","options","actions","pointer","touch","system","systemActions","action","Boolean","createDefaultMobileActions","context","ActionSleepParamSchema","defineActionSleep","getDebug"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC0HO,MAAeI;;QA8CpB;QA8BA;;AACF;AAKO,MAAMC,eAAe,CAK1BC,SAaOA;AAGT,SAASC,gBACPC,MAAuC,EACvCC,cAAsB;IAEtB,IAAI,CAACD,QACH,MAAM,IAAIE,MAAMD;IAElB,OAAO;QAAE,GAAGD,OAAO,MAAM,CAAC,EAAE;QAAE,GAAGA,OAAO,MAAM,CAAC,EAAE;IAAC;AACpD;AAEA,SAASG,yBAGPL,MAQD;IACC,OAAOD,aAA8B;QACnC,MAAMC,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW;QAC/B,gBAAgBA,OAAO,cAAc;QACrC,aAAaA,OAAO,WAAW;QAC/B,QAAQA,OAAO,MAAM;QACrB,MAAM,OAAOM;YACX,MAAMN,OAAO,IAAI,CACfC,gBAAgBK,MAAM,MAAM,EAAEN,OAAO,oBAAoB,GACzDM;QAEJ;IACF;AACF;AAGO,MAAMC,uBAAuBC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC3C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMC,kBAAkB,CAC7BC,MAEON,yBAAsE;QAC3E,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsB;QAC1C;QACA,sBAAsB;QACtB,MAAM,OAAOK;YACX,MAAMD,IAAIC;QACZ;IACF;AAIK,MAAMC,8BAA8BL,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMK,yBAAyB,CACpCC,aAEOV,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaQ;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAA+B;QACnD;QACA,sBAAsB;QACtB,MAAM,OAAOD;YACX,MAAMG,WAAWH;QACnB;IACF;AAIK,MAAMI,+BAA+BR,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMQ,0BAA0B,CACrCC,cAEOb,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAkB;QACtC;QACA,sBAAsB;QACtB,MAAM,OAAOJ;YACX,MAAMM,YAAYN;QACpB;IACF;AAIK,MAAMO,yBAAyBX,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMW,oBAAoB,CAC/BC,QAEOhB,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAac;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACA,sBAAsB;QACtB,MAAM,OAAOP;YACX,MAAMS,MAAMT;QACd;IACF;AAIF,MAAMU,yBACJ;AACK,MAAMC,yBAAyBf,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOA,6BAAAA,CAAAA,CAAAA,KACC,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,MAAQ;KAAG,EAC9B,SAAS,CAAC,CAACgB,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQf,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAACa,wBACT,QAAQ;IACX,MAAMd,6BAAAA,CAAAA,CAAAA,OACC,CAAC;QAAC;QAAW;QAAS;KAAW,EACrC,OAAO,CAAC,WACR,QAAQ,CACP;IAEJ,qBAAqBA,6BAAAA,CAAAA,CAAAA,OACX,GACP,QAAQ,GACR,QAAQ,CACP;AAEN;AAQO,MAAMkB,oBAAoB,CAC/BC,WAEO5B,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAawB;QACb,QAAQ;YACN,OAAO;YACP,QAAQ;gBAAE,QAAQ;YAAwB;QAC5C;QACA,MAAM,OAAOjB;YAEX,IAAKA,AAA0B,aAA1BA,MAAM,IAAI,EACbA,MAAM,IAAI,GAAG;YAGf,IAAIA,AAAe,YAAfA,MAAM,IAAI,EAAc,YAC1B,MAAMqB,SAAS,UAAU,CAACrB,MAAM,MAAM;YAIxC,IAAI,CAACA,SAAS,CAACA,MAAM,KAAK,EACxB;YAGF,MAAMqB,SAAS,QAAQ,CAACrB,MAAM,KAAK,EAAE;gBACnC,QAAQA,MAAM,MAAM;gBACpB,SAASA,AAAe,eAAfA,MAAM,IAAI;gBACnB,qBAAqBA,MAAM,mBAAmB;YAChD;QACF;IACF;AAIK,MAAMsB,iCAAiCpB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACrD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,6BAAAA,CAAAA,CAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMqB,4BAA4B,CACvCC,gBAEO/B,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAa6B;QACb,QAAQ;YACN,SAAS;QACX;QACA,MAAM,OAAOtB;YACX,MAAMwB,cAAcxB,MAAM,OAAO,EAAE;gBACjC,QAAQA,MAAM,MAAM;YACtB;QACF;IACF;AAIK,MAAMyB,0BAA0BvB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,YAAYA,6BAAAA,CAAAA,CAAAA,OACL,CAAC;QACJ;QACA;QACA;QACA;QACA;KACD,EACA,OAAO,CAAC,gBACR,QAAQ,CACP;IAEJ,WAAWA,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMuB,qBAAqB,CAChCC,SAEOlC,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAagC;QACb,QAAQ;YACN,WAAW;YACX,YAAY;YACZ,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACA,MAAM,OAAOzB;YACX,MAAM2B,OAAO3B;QACf;IACF;AAIK,MAAM4B,+BAA+B1B,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,MAAMC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;IAC3C,IAAIA,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAM0B,0BAA0B,CACrCC,cAEOrC,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamC;QACb,QAAQ;YACN,MAAM;gBAAE,QAAQ;YAA6B;YAC7C,IAAI;gBAAE,QAAQ;YAAuB;QACvC;QACA,MAAM,OAAO5B;YACX,MAAM+B,OAAO/B,MAAM,IAAI;YACvB,MAAMgC,KAAKhC,MAAM,EAAE;YACnB,IAAI,CAAC+B,MACH,MAAM,IAAIjC,MAAM;YAElB,IAAI,CAACkC,IACH,MAAM,IAAIlC,MAAM;YAElB,MAAMgC,YACJ;gBAAE,GAAGC,KAAK,MAAM,CAAC,EAAE;gBAAE,GAAGA,KAAK,MAAM,CAAC,EAAE;YAAC,GACvC;gBAAE,GAAGC,GAAG,MAAM,CAAC,EAAE;gBAAE,GAAGA,GAAG,MAAM,CAAC,EAAE;YAAC;QAEvC;IACF;AAGK,MAAMC,6BAA6B/B,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;IAEF,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMgC,wBAAwB,CACnCC,YAEOpC,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAakC;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAqB;QACzC;QACA,sBAAsB;QACtB,MAAM,OAAO3B,OAAON;YAClB,MAAMmC,UAAU7B,OAAO;gBAAE,UAAUN,MAAM,QAAQ;YAAC;QACpD;IACF;AAGK,MAAMoC,yBAAyBlC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,6BAAAA,CAAAA,CAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,SAASmC,0BACdrC,KAAuB,EACvBsC,UAA6C;IAO7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAM,EAAEG,KAAK,EAAEC,GAAG,EAAE,GAAG1C;IAEvB,MAAM2C,aAAaF,QACf;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;IAAC,IACzC;QAAE,GAAGF,QAAQ;QAAG,GAAGC,SAAS;IAAE;IAElC,IAAII;IAEJ,IAAIF,KACFE,WAAW;QAAE,GAAGF,IAAI,MAAM,CAAC,EAAE;QAAE,GAAGA,IAAI,MAAM,CAAC,EAAE;IAAC;SAC3C,IAAI1C,MAAM,QAAQ,EAAE;QACzB,MAAM6C,YAAY7C,MAAM,SAAS;QACjC,IAAI,CAAC6C,WACH,MAAM,IAAI/C,MAAM;QAElB8C,WAAW;YACT,GACED,WAAW,CAAC,GACXE,CAAAA,AAAc,YAAdA,YACG7C,MAAM,QAAQ,GACd6C,AAAc,WAAdA,YACE,CAAC7C,MAAM,QAAQ,GACf;YACR,GACE2C,WAAW,CAAC,GACXE,CAAAA,AAAc,WAAdA,YACG7C,MAAM,QAAQ,GACd6C,AAAc,SAAdA,YACE,CAAC7C,MAAM,QAAQ,GACf;QACV;IACF,OACE,MAAM,IAAIF,MACR;IAIJ8C,SAAS,CAAC,GAAGE,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACF,SAAS,CAAC,EAAEL;IAC9CK,SAAS,CAAC,GAAGE,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACF,SAAS,CAAC,EAAEJ;IAE9C,MAAMO,WAAW/C,MAAM,QAAQ,IAAI;IAEnC,IAAIgD,cAAc,AAAwB,YAAxB,OAAOhD,MAAM,MAAM,GAAgBA,MAAM,MAAM,GAAG;IACpE,IAAIgD,AAAgB,MAAhBA,aACFA,cAAc;IAGhB,OAAO;QAAEL;QAAYC;QAAUG;QAAUC;IAAY;AACvD;AAEO,MAAMC,oBAAoB,CAACvD,SAIzBD,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa2C;QACb,QAAQ;YACN,OAAO;gBAAE,QAAQ;YAA6B;YAC9C,KAAK;gBAAE,QAAQ;YAA2B;QAC5C;QACA,MAAM,OAAOpC;YACX,MAAM,EAAE2C,UAAU,EAAEC,QAAQ,EAAEG,QAAQ,EAAEC,WAAW,EAAE,GACnDX,0BAA0BrC,OAAO,MAAMN,OAAO,IAAI;YACpD,IAAK,IAAIwD,IAAI,GAAGA,IAAIF,aAAaE,IAC/B,MAAMxD,OAAO,KAAK,CAACiD,YAAYC,UAAU;gBAAEG;YAAS;QAExD;IACF;AAIK,MAAMI,8BAA8BjD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,iCACT,QAAQ;AACb;AAKO,MAAMiD,yBAAyB,CACpCC,aAEO5D,aAGL;QACA,MAAM;QACN,aAAauB;QACb,gBAAgB;QAChB,aAAamC;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAyB;QAC7C;QACA,MAAM,OAAOnD;YACX,MAAMqD,WAAWrD,MAAM,MAAM;QAC/B;IACF;AAIK,MAAMsD,8BAA8BpD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,WAAWA,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAQ;KAAQ,EACtB,QAAQ,CAAC;IACZ,OAAOA,6BAAAA,CAAAA,CAAAA,MACE,GACN,GAAG,GACH,GAAG,CAAC,GACJ,OAAO,CAAC,GACR,QAAQ,CACP;AAEN;AAMO,MAAMqD,yBAAyB,CAAC7D,SAI9BD,aAGL;QACA,MAAM;QACN,aACE;QACF,aAAa6D;QACb,QAAQ;YACN,WAAW;YACX,OAAO;QACT;QACA,MAAM,OAAOtD;YACX,MAAMwD,QAAQxD,MAAM,KAAK,IAAI;YAC7B,IAAIN,OAAO,QAAQ,CAAC,UAAU,EAAE,YAC9B,MAAMA,OAAO,QAAQ,CAAC,UAAU,CAACM,MAAM,SAAS,EAAEwD;YAIpD,MAAMC,OACJ/D,OAAO,KAAK,IACV,EAAAgE,SACA,IAAIC,QAAc,CAACC,UAAYC,WAAWD,SAASF,QAAO;YAC9D,MAAMI,WAAW9D,AAAoB,WAApBA,MAAM,SAAS,GAAc,cAAc;YAC5D,IAAK,IAAIkD,IAAI,GAAGA,IAAIM,OAAON,IAAK;gBAC9B,MAAMxD,OAAO,QAAQ,CAAC,aAAa,CAACoE;gBACpC,MAAML,KAAK;YACb;QACF;IACF;AAIK,MAAMM,yBAAyB7D,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAM;KAAM,EAClB,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AASO,MAAM8D,oBAAoB,CAACtE;IAIhC,IAAI,CAACA,OAAO,KAAK,EACf;IAGF,OAAOD,aAA8D;QACnE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAasE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAe;YACjC,WAAW;YACX,UAAU;QACZ;QACA,MAAM,OAAO/D;YACX,MAAM,EAAEiE,OAAO,EAAEC,OAAO,EAAEC,aAAa,EAAEC,WAAW,EAAErB,QAAQ,EAAE,GAC9DsB,oBAAoBrE,OAAO,MAAMN,OAAO,IAAI;YAC9C,MAAMA,OAAO,KAAK,GAChB;gBAAE,GAAGuE;gBAAS,GAAGC;YAAQ,GACzB;gBAAEC;gBAAeC;gBAAarB;YAAS;QAE3C;IACF;AACF;AAEO,SAASsB,oBACdrE,KAAuB,EACvBsC,UAA6C;IAQ7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAMgC,UAAUtE,MAAM,MAAM;IAC5B,MAAMiE,UAAUK,UACZxB,KAAK,KAAK,CAACwB,QAAQ,MAAM,CAAC,EAAE,IAC5BxB,KAAK,KAAK,CAACP,QAAQ;IACvB,MAAM2B,UAAUI,UACZxB,KAAK,KAAK,CAACwB,QAAQ,MAAM,CAAC,EAAE,IAC5BxB,KAAK,KAAK,CAACN,SAAS;IACxB,MAAMO,WAAW/C,MAAM,QAAQ,IAAI;IAEnC,MAAMuE,eAAezB,KAAK,KAAK,CAACA,KAAK,GAAG,CAACP,OAAOC,UAAU;IAC1D,MAAMgC,iBAAiBxE,MAAM,QAAQ,IAAIuE;IAEzC,MAAMJ,gBAAgBI;IACtB,MAAMH,cACJpE,AAAoB,UAApBA,MAAM,SAAS,GACXuE,eAAeC,iBACf1B,KAAK,GAAG,CAAC,IAAIyB,eAAeC;IAElC,OAAO;QAAEP;QAASC;QAASC;QAAeC;QAAarB;IAAS;AAClE;AAgCA,SAAS0B,wBACP/E,MAA+B,EAC/BgF,IAAyB;IAEzB,OAAOjF,aAAyC;QAC9C,MAAMC,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW;QAC/B,gBAAgBA,OAAO,cAAc;QACrC,mBAAmBA,OAAO,iBAAiB;QAC3C,kBAAkBA,OAAO,gBAAgB;QACzCgF;IACF;AACF;AAEO,SAASC,iCACdC,KAAsB,EACtBC,UAAuC,CAAC,CAAC;IAEzC,MAAMC,UAAgD,EAAE;IACxD,MAAM,EAAEC,OAAO,EAAE1D,QAAQ,EAAEM,MAAM,EAAEqD,KAAK,EAAEC,MAAM,EAAE,GAAGL;IAErD,IAAIG,SAAS;QACXD,QAAQ,IAAI,CAAC1E,gBAAgB2E,QAAQ,GAAG;QACxC,IAAIA,QAAQ,WAAW,EACrBD,QAAQ,IAAI,CAACnE,wBAAwBoE,QAAQ,WAAW;QAE1D,IAAIA,QAAQ,UAAU,EACpBD,QAAQ,IAAI,CAACtE,uBAAuBuE,QAAQ,UAAU;QAExD,IAAIA,QAAQ,KAAK,EACfD,QAAQ,IAAI,CAAChE,kBAAkBiE,QAAQ,KAAK;QAE9C,IAAIA,QAAQ,WAAW,EACrBD,QAAQ,IAAI,CAACjD,wBAAwBkD,QAAQ,WAAW;QAE1D,IAAIA,QAAQ,SAAS,EACnBD,QAAQ,IAAI,CAAC5C,sBAAsB6C,QAAQ,SAAS;IAExD;IAEA,IAAI1D,UACFyD,QAAQ,IAAI,CACV1D,kBAAkBC,WAClB+B,uBAAuB/B,SAAS,UAAU,GAC1CE,0BAA0BF,SAAS,aAAa,GAChDkC,uBAAuB;QAAElC;QAAU,OAAOwD,QAAQ,KAAK;IAAC;IAI5D,IAAIlD,QACFmD,QAAQ,IAAI,CAACpD,mBAAmBC,OAAO,MAAM;IAG/C,IAAIqD,OAAO,SAASH,QAAQ,IAAI,IAAIA,AAAyB,UAAzBA,QAAQ,YAAY,EACtDC,QAAQ,IAAI,CAAC7B,kBAAkB;QAAE,OAAO+B,MAAM,KAAK;QAAE,MAAMH,QAAQ,IAAI;IAAC;IAG1E,IAAIG,OAAO,SAASH,QAAQ,IAAI,IAAIA,AAAyB,UAAzBA,QAAQ,YAAY,EACtDC,QAAQ,IAAI,CAACd,kBAAkB;QAAE,OAAOgB,MAAM,KAAK;QAAE,MAAMH,QAAQ,IAAI;IAAC;IAG1E,IAAII,UAAUJ,QAAQ,aAAa,EAAE;QACnC,MAAM,EAAEK,aAAa,EAAE,GAAGL;QAC1B,IAAII,OAAO,UAAU,IAAIC,cAAc,UAAU,EAC/CJ,QAAQ,IAAI,CACVL,wBAAwBS,cAAc,UAAU,EAAED,OAAO,UAAU;QAGvE,IAAIA,OAAO,UAAU,IAAIC,cAAc,UAAU,EAC/CJ,QAAQ,IAAI,CACVL,wBAAwBS,cAAc,UAAU,EAAED,OAAO,UAAU;QAGvE,IAAIA,OAAO,gBAAgB,IAAIC,cAAc,gBAAgB,EAC3DJ,QAAQ,IAAI,CACVL,wBACES,cAAc,gBAAgB,EAC9BD,OAAO,gBAAgB;IAI/B;IAEA,OAAOH,QAAQ,MAAM,CAAC,CAACK,SACrBC,QAAQD;AAEZ;AAEO,SAASE,2BACdC,OAAiC;IAEjC,OAAOX,iCAAiCW,QAAQ,KAAK,EAAE;QACrD,MAAMA,QAAQ,IAAI;QAClB,OAAOA,QAAQ,KAAK;QACpB,eAAeA,QAAQ,aAAa;IACtC;AACF;AAGO,MAAMC,yBAAyBrF,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQA,6BAAAA,CAAAA,CAAAA,MACC,GACN,OAAO,CAAC,MACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMsF,oBAAoB,IACxB/F,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa8F;QACb,QAAQ;YACN,QAAQ;QACV;QACA,MAAM,OAAOvF;YACX,MAAM+C,WAAW/C,OAAO,UAAU;YAClCyF,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,wBAAwB,CAAC,aAAa,EAAE1C,SAAS,EAAE,CAAC;YAC7D,MAAM,IAAIY,QAAQ,CAACC,UAAYC,WAAWD,SAASb;QACrD;IACF"}
+{"version":3,"file":"device/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/device/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ModelRuntime } from '@/ai-model/models';\nimport { getMidsceneLocationSchema } from '@/common';\nimport type {\n  ActionScrollParam,\n  DeviceAction,\n  LocateResultElement,\n} from '@/types';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport interface FileChooserHandler {\n  accept(files: string[]): Promise<void>;\n}\n\nexport interface MjpegStreamFrame {\n  /** Raw base64-encoded image bytes WITHOUT a `data:image/...;base64,` prefix. */\n  data: string;\n  contentType?: string;\n}\n\nexport interface MjpegStreamHandle {\n  stop(): void | Promise<void>;\n}\n\nexport interface MjpegStreamOptions {\n  signal?: AbortSignal;\n  onFrame(frame: MjpegStreamFrame): void;\n  onError?(error: unknown): void;\n}\n\n/** A point in device-pixel coordinates on the screen. */\nexport interface PointerPoint {\n  x: number;\n  y: number;\n}\n\nexport interface PointerInputPrimitives {\n  tap(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n  doubleClick?(p: PointerPoint): Promise<void>;\n  rightClick?(p: PointerPoint): Promise<void>;\n  hover?(p: PointerPoint): Promise<void>;\n  longPress?(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n  dragAndDrop?(from: PointerPoint, to: PointerPoint): Promise<void>;\n}\n\nexport interface TouchInputPrimitives {\n  swipe(\n    start: PointerPoint,\n    end: PointerPoint,\n    opts?: { duration?: number; repeat?: number },\n  ): Promise<void>;\n  pinch?(\n    center: PointerPoint,\n    opts: { startDistance: number; endDistance: number; duration: number },\n  ): Promise<void>;\n}\n\nexport interface KeyboardInputPrimitives {\n  keyboardPress(keyName: string, opts?: { target?: unknown }): Promise<void>;\n  cursorMove?(direction: 'left' | 'right', times?: number): Promise<void>;\n  typeText(\n    value: string,\n    opts?: {\n      autoDismissKeyboard?: boolean;\n      keyboardDismissStrategy?: 'esc-first' | 'back-first';\n      target?: unknown;\n      replace?: boolean;\n      focusOnly?: boolean;\n    },\n  ): Promise<void>;\n  clearInput(target?: unknown): Promise<void>;\n}\n\nexport interface ScrollInputPrimitives {\n  scroll(param: ActionScrollParam): Promise<void>;\n}\n\nexport interface SystemInputPrimitives {\n  backButton?(): Promise<void>;\n  homeButton?(): Promise<void>;\n  recentAppsButton?(): Promise<void>;\n}\n\nexport interface InputPrimitives {\n  pointer?: PointerInputPrimitives;\n  keyboard?: KeyboardInputPrimitives;\n  touch?: TouchInputPrimitives;\n  scroll?: ScrollInputPrimitives;\n  system?: SystemInputPrimitives;\n}\n\nexport interface MobileInputPrimitives extends InputPrimitives {\n  pointer: PointerInputPrimitives & {\n    doubleClick(p: PointerPoint): Promise<void>;\n    longPress(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n    dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n  };\n  keyboard: KeyboardInputPrimitives;\n  touch: TouchInputPrimitives;\n}\n\nexport interface BrowserInputPrimitives extends InputPrimitives {\n  pointer: PointerInputPrimitives & {\n    doubleClick(p: PointerPoint): Promise<void>;\n    rightClick(p: PointerPoint): Promise<void>;\n    hover(p: PointerPoint): Promise<void>;\n    dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n    longPress(p: PointerPoint, opts?: { duration?: number }): Promise<void>;\n  };\n  keyboard: KeyboardInputPrimitives;\n  scroll: ScrollInputPrimitives;\n  touch: TouchInputPrimitives;\n}\n\nexport interface ComputerInputPrimitives extends InputPrimitives {\n  pointer: PointerInputPrimitives & {\n    doubleClick(p: PointerPoint): Promise<void>;\n    rightClick(p: PointerPoint): Promise<void>;\n    hover(p: PointerPoint): Promise<void>;\n    dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;\n  };\n  keyboard: KeyboardInputPrimitives;\n  scroll: ScrollInputPrimitives;\n}\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[];\n\n  abstract cacheFeatureForPoint?(\n    center: [number, number],\n    options?: {\n      targetDescription?: string;\n      modelRuntime?: ModelRuntime;\n    },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // for web only\n  registerFileChooserListener?(\n    handler: (chooser: FileChooserHandler) => Promise<void>,\n  ): Promise<{ dispose: () => void; getError: () => Error | undefined }>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  /**\n   * Get the current device-local time as a formatted string.\n   * Prefer this for user-visible time because timestamps alone do not preserve\n   * the target device's timezone when formatted on the host machine.\n   */\n  getDeviceLocalTimeString?(format?: string): Promise<string>;\n\n  /** URL of native MJPEG stream for real-time screen preview (e.g. WDA MJPEG server) */\n  mjpegStreamUrl?: string;\n\n  /**\n   * Optional in-process MJPEG frame producer. Implementations can push raw\n   * base64 frames here when there is no standalone native MJPEG URL, e.g.\n   * Chromium CDP Page.startScreencast for web previews.\n   */\n  startMjpegStream?(\n    options: MjpegStreamOptions,\n  ): MjpegStreamHandle | undefined | Promise<MjpegStreamHandle | undefined>;\n\n  /**\n   * Optional hook used after keyboard-only actions to force a fresh frame on\n   * the active MJPEG stream. Implementations should be a no-op when no stream\n   * is active.\n   */\n  flushPendingVisualUpdate?(): Promise<void>;\n\n  /**\n   * Optional navigation state probe for browser-like interfaces, used to drive\n   * loading indicators in playground UIs. Returning `undefined` means the\n   * interface does not expose this concept.\n   */\n  navigationState?(): Promise<{ isLoading: boolean }>;\n\n  /**\n   * Low-level device input surface. Platform implementations expose transport\n   * primitives here; higher-level AI actions and manual pointer dispatch should\n   * adapt to this instead of duplicating platform gesture logic.\n   */\n  inputPrimitives?: InputPrimitives;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\n// TReturn allows specifying the return type of the action\nexport const defineAction = <\n  TSchema extends z.ZodType | undefined = undefined,\n  TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined,\n  TReturn = any,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema?: TSchema;\n    call: (param: TRuntime) => Promise<TReturn> | TReturn;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime, TReturn>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime, TReturn> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\nfunction pointFromLocate(\n  locate: LocateResultElement | undefined,\n  missingMessage: string,\n): PointerPoint {\n  if (!locate) {\n    throw new Error(missingMessage);\n  }\n  return { x: locate.center[0], y: locate.center[1] };\n}\n\nfunction defineLocatedPointAction<\n  TSchema extends z.ZodType,\n  TParam extends { locate: LocateResultElement },\n>(config: {\n  name: string;\n  description: string;\n  interfaceAlias?: string;\n  paramSchema: TSchema;\n  sample: DeviceAction<TParam>['sample'];\n  missingLocateMessage: string;\n  call: (point: PointerPoint, param: TParam) => Promise<void>;\n}): DeviceAction<TParam> {\n  return defineAction<TSchema, TParam>({\n    name: config.name,\n    description: config.description,\n    interfaceAlias: config.interfaceAlias,\n    paramSchema: config.paramSchema,\n    sample: config.sample,\n    call: async (param) => {\n      await config.call(\n        pointFromLocate(param.locate, config.missingLocateMessage),\n        param,\n      );\n    },\n  });\n}\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  tap: PointerInputPrimitives['tap'],\n): DeviceAction<ActionTapParam> => {\n  return defineLocatedPointAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    sample: {\n      locate: { prompt: 'the \"Submit\" button' },\n    },\n    missingLocateMessage: 'Element not found, cannot tap',\n    call: async (point) => {\n      await tap(point);\n    },\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  rightClick: NonNullable<PointerInputPrimitives['rightClick']>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineLocatedPointAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    sample: {\n      locate: { prompt: 'the file icon on the desktop' },\n    },\n    missingLocateMessage: 'Element not found, cannot right click',\n    call: async (point) => {\n      await rightClick(point);\n    },\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  doubleClick: NonNullable<PointerInputPrimitives['doubleClick']>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineLocatedPointAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    sample: {\n      locate: { prompt: 'the folder icon' },\n    },\n    missingLocateMessage: 'Element not found, cannot double click',\n    call: async (point) => {\n      await doubleClick(point);\n    },\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  hover: NonNullable<PointerInputPrimitives['hover']>,\n): DeviceAction<ActionHoverParam> => {\n  return defineLocatedPointAction<\n    typeof actionHoverParamSchema,\n    ActionHoverParam\n  >({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    sample: {\n      locate: { prompt: 'the navigation menu item \"Products\"' },\n    },\n    missingLocateMessage: 'Element not found, cannot hover',\n    call: async (point) => {\n      await hover(point);\n    },\n  });\n};\n\n// Input\nconst inputLocateDescription =\n  'the position of the placeholder or text content in the target input field. If there is no content, locate the center of the input field.';\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace mode, only the inserted characters for typeOnly mode, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe(inputLocateDescription)\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'typeOnly'])\n    .default('replace')\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"typeOnly\" - type the value directly without clearing the field first, and should be set explicitly for incremental edits after moving the cursor; \"clear\" - clear the field without inputting new text.',\n    ),\n  autoDismissKeyboard: z\n    .boolean()\n    .optional()\n    .describe(\n      'If true, the keyboard will be dismissed after the input is completed. Do not set it unless the user asks you to do so.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'typeOnly' | 'append';\n  autoDismissKeyboard?: boolean;\n};\n\nexport const defineActionInput = (\n  keyboard: KeyboardInputPrimitives,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    sample: {\n      value: 'test@example.com',\n      locate: { prompt: 'the email input field' },\n    },\n    call: async (param) => {\n      // backward compat: convert deprecated 'append' to 'typeOnly'\n      if ((param.mode as string) === 'append') {\n        param.mode = 'typeOnly';\n      }\n\n      if (param.mode === 'clear') {\n        await keyboard.clearInput(param.locate);\n        return;\n      }\n\n      if (!param || !param.value) {\n        return;\n      }\n\n      await keyboard.typeText(param.value, {\n        target: param.locate,\n        replace: param.mode !== 'typeOnly',\n        autoDismissKeyboard: param.autoDismissKeyboard,\n      });\n    },\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  keyboardPress: KeyboardInputPrimitives['keyboardPress'],\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    sample: {\n      keyName: 'Enter',\n    },\n    call: async (param) => {\n      await keyboardPress(param.keyName, {\n        target: param.locate,\n      });\n    },\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  scrollType: z\n    .enum([\n      'singleAction',\n      'scrollToBottom',\n      'scrollToTop',\n      'scrollToRight',\n      'scrollToLeft',\n    ])\n    .default('singleAction')\n    .describe(\n      'The scroll behavior: \"singleAction\" for a single scroll action, \"scrollToBottom\" for scrolling all the way to the bottom by rapidly scrolling 5-10 times (skipping intermediate content until reaching the bottom), \"scrollToTop\" for scrolling all the way to the top by rapidly scrolling 5-10 times (skipping intermediate content until reaching the top), \"scrollToRight\" for scrolling all the way to the right by rapidly scrolling multiple times, \"scrollToLeft\" for scrolling all the way to the left by rapidly scrolling multiple times',\n    ),\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe(\n      'The direction to scroll. Only effective when scrollType is \"singleAction\".',\n    ),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Describe the target element to be scrolled on, like \"the table\" or \"the list\" or \"the content area\" or \"the scrollable area\". Do NOT provide a general intent like \"scroll to find some element\"',\n    ),\n});\n\nexport const defineActionScroll = (\n  scroll: ScrollInputPrimitives['scroll'],\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or a scrollable element to browse content. This is the preferred way to scroll on all platforms, including mobile. Supports scrollToBottom/scrollToTop for boundary navigation. Default: direction `down`, scrollType `singleAction`, distance `null`.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    sample: {\n      direction: 'down',\n      scrollType: 'singleAction',\n      locate: { prompt: 'the center of the product list area' },\n    },\n    call: async (param) => {\n      await scroll(param);\n    },\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  dragAndDrop: NonNullable<PointerInputPrimitives['dragAndDrop']>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description:\n      'Pick up a specific UI element and move it to a new position (e.g., reorder a card, move a file into a folder, sort list items). The element itself moves with your finger/mouse.',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    sample: {\n      from: { prompt: 'the \"report.pdf\" file icon' },\n      to: { prompt: 'the upload drop zone' },\n    },\n    call: async (param) => {\n      const from = param.from;\n      const to = param.to;\n      if (!from) {\n        throw new Error('missing \"from\" param for drag and drop');\n      }\n      if (!to) {\n        throw new Error('missing \"to\" param for drag and drop');\n      }\n      await dragAndDrop(\n        { x: from.center[0], y: from.center[1] },\n        { x: to.center[0], y: to.center[1] },\n      );\n    },\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  longPress: NonNullable<PointerInputPrimitives['longPress']>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineLocatedPointAction<\n    typeof ActionLongPressParamSchema,\n    ActionLongPressParam\n  >({\n    name: 'LongPress',\n    description: 'Long press the element',\n    interfaceAlias: 'aiLongPress',\n    paramSchema: ActionLongPressParamSchema,\n    sample: {\n      locate: { prompt: 'the message bubble' },\n    },\n    missingLocateMessage: 'LongPress requires an element to be located',\n    call: async (point, param) => {\n      await longPress(point, { duration: param.duration });\n    },\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport function normalizeMobileSwipeParam(\n  param: ActionSwipeParam,\n  screenSize: { width: number; height: number },\n): {\n  startPoint: { x: number; y: number };\n  endPoint: { x: number; y: number };\n  duration: number;\n  repeatCount: number;\n} {\n  const { width, height } = screenSize;\n  const { start, end } = param;\n\n  const startPoint = start\n    ? { x: start.center[0], y: start.center[1] }\n    : { x: width / 2, y: height / 2 };\n\n  let endPoint: { x: number; y: number };\n\n  if (end) {\n    endPoint = { x: end.center[0], y: end.center[1] };\n  } else if (param.distance) {\n    const direction = param.direction;\n    if (!direction) {\n      throw new Error('direction is required for swipe gesture');\n    }\n    endPoint = {\n      x:\n        startPoint.x +\n        (direction === 'right'\n          ? param.distance\n          : direction === 'left'\n            ? -param.distance\n            : 0),\n      y:\n        startPoint.y +\n        (direction === 'down'\n          ? param.distance\n          : direction === 'up'\n            ? -param.distance\n            : 0),\n    };\n  } else {\n    throw new Error(\n      'Either end or distance must be specified for swipe gesture',\n    );\n  }\n\n  endPoint.x = Math.max(0, Math.min(endPoint.x, width));\n  endPoint.y = Math.max(0, Math.min(endPoint.y, height));\n\n  const duration = param.duration ?? 300;\n\n  let repeatCount = typeof param.repeat === 'number' ? param.repeat : 1;\n  if (repeatCount === 0) {\n    repeatCount = 10;\n  }\n\n  return { startPoint, endPoint, duration, repeatCount };\n}\n\nexport const defineActionSwipe = (config: {\n  swipe: TouchInputPrimitives['swipe'];\n  size(): Promise<Size>;\n}): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a touch gesture for interactions beyond regular scrolling (e.g., flip pages in a carousel, dismiss a notification, swipe-to-delete a list item). For regular content scrolling, use Scroll instead. Use \"distance\" + \"direction\" for relative movement, or \"end\" for precise endpoint.',\n    paramSchema: ActionSwipeParamSchema,\n    sample: {\n      start: { prompt: 'center of the notification' },\n      end: { prompt: 'upper edge of the screen' },\n    },\n    call: async (param) => {\n      const { startPoint, endPoint, duration, repeatCount } =\n        normalizeMobileSwipeParam(param, await config.size());\n      for (let i = 0; i < repeatCount; i++) {\n        await config.swipe(startPoint, endPoint, { duration });\n      }\n    },\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The input field to be cleared')\n    .optional(),\n});\nexport type ActionClearInputParam = {\n  locate?: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  clearInput: KeyboardInputPrimitives['clearInput'],\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: inputLocateDescription,\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    sample: {\n      locate: { prompt: 'the search input field' },\n    },\n    call: async (param) => {\n      await clearInput(param.locate);\n    },\n  });\n};\n\n// CursorMove\nexport const actionCursorMoveParamSchema = z.object({\n  direction: z\n    .enum(['left', 'right'])\n    .describe('The direction to move the cursor'),\n  times: z\n    .number()\n    .int()\n    .min(1)\n    .default(1)\n    .describe(\n      'The number of times to move the cursor in the specified direction',\n    ),\n});\nexport type ActionCursorMoveParam = {\n  direction: 'left' | 'right';\n  times?: number;\n};\n\nexport const defineActionCursorMove = (config: {\n  keyboard: Pick<KeyboardInputPrimitives, 'keyboardPress' | 'cursorMove'>;\n  sleep?(timeMs: number): Promise<void>;\n}): DeviceAction<ActionCursorMoveParam> => {\n  return defineAction<\n    typeof actionCursorMoveParamSchema,\n    ActionCursorMoveParam\n  >({\n    name: 'CursorMove',\n    description:\n      'Move the text cursor (caret) left or right within an input field or text area. Use this to reposition the cursor without selecting text.',\n    paramSchema: actionCursorMoveParamSchema,\n    sample: {\n      direction: 'left',\n      times: 3,\n    },\n    call: async (param) => {\n      const times = param.times ?? 1;\n      if (config.keyboard.cursorMove) {\n        await config.keyboard.cursorMove(param.direction, times);\n        return;\n      }\n\n      const wait =\n        config.sleep ??\n        ((timeMs: number) =>\n          new Promise<void>((resolve) => setTimeout(resolve, timeMs)));\n      const arrowKey = param.direction === 'left' ? 'ArrowLeft' : 'ArrowRight';\n      for (let i = 0; i < times; i++) {\n        await config.keyboard.keyboardPress(arrowKey);\n        await wait(100);\n      }\n    },\n  });\n};\n\n// Pinch\nexport const ActionPinchParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'The element to pinch on. If not specified, the center of the screen will be used',\n    ),\n  direction: z\n    .enum(['in', 'out'])\n    .describe(\n      'Pinch direction. \"in\" = pinch fingers together (zoom out / shrink), \"out\" = spread fingers apart (zoom in / enlarge).',\n    ),\n  distance: z\n    .number()\n    .positive()\n    .optional()\n    .describe(\n      'How far each finger moves in pixels. Defaults to a quarter of the shorter screen dimension.',\n    ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Duration of the pinch gesture in milliseconds'),\n});\n\nexport type ActionPinchParam = {\n  locate?: LocateResultElement;\n  direction: 'in' | 'out';\n  distance?: number;\n  duration?: number;\n};\n\nexport const defineActionPinch = (config: {\n  pinch: TouchInputPrimitives['pinch'];\n  size(): Promise<Size>;\n}): DeviceAction<ActionPinchParam> | undefined => {\n  if (!config.pinch) {\n    return undefined;\n  }\n\n  return defineAction<typeof ActionPinchParamSchema, ActionPinchParam>({\n    name: 'Pinch',\n    description:\n      'Perform a two-finger pinch gesture. Use direction \"in\" to pinch fingers together (zoom out), or \"out\" to spread fingers apart (zoom in). Optionally specify distance for how far each finger moves.',\n    interfaceAlias: 'aiPinch',\n    paramSchema: ActionPinchParamSchema,\n    sample: {\n      locate: { prompt: 'the map area' },\n      direction: 'out',\n      distance: 200,\n    },\n    call: async (param) => {\n      const { centerX, centerY, startDistance, endDistance, duration } =\n        normalizePinchParam(param, await config.size());\n      await config.pinch?.(\n        { x: centerX, y: centerY },\n        { startDistance, endDistance, duration },\n      );\n    },\n  });\n};\n\nexport function normalizePinchParam(\n  param: ActionPinchParam,\n  screenSize: { width: number; height: number },\n): {\n  centerX: number;\n  centerY: number;\n  startDistance: number;\n  endDistance: number;\n  duration: number;\n} {\n  const { width, height } = screenSize;\n  const element = param.locate;\n  const centerX = element\n    ? Math.round(element.center[0])\n    : Math.round(width / 2);\n  const centerY = element\n    ? Math.round(element.center[1])\n    : Math.round(height / 2);\n  const duration = param.duration ?? 500;\n\n  const baseDistance = Math.round(Math.min(width, height) / 4);\n  const fingerDistance = param.distance ?? baseDistance;\n\n  const startDistance = baseDistance;\n  const endDistance =\n    param.direction === 'out'\n      ? baseDistance + fingerDistance\n      : Math.max(10, baseDistance - fingerDistance);\n\n  return { centerX, centerY, startDistance, endDistance, duration };\n}\n\nexport interface MobileInputActionContext {\n  input: MobileInputPrimitives;\n  size(): Promise<Size>;\n  sleep?(timeMs: number): Promise<void>;\n  getDefaultAutoDismissKeyboard?(): boolean | undefined;\n  systemActions?: SystemInputActionOptions;\n}\n\nexport interface SystemInputActionConfig {\n  name: string;\n  description: string;\n  interfaceAlias?: string;\n  delayBeforeRunner?: number;\n  delayAfterRunner?: number;\n}\n\nexport interface SystemInputActionOptions {\n  backButton?: SystemInputActionConfig;\n  homeButton?: SystemInputActionConfig;\n  recentAppsButton?: SystemInputActionConfig;\n}\n\nexport interface InputPrimitiveActionOptions {\n  size?: () => Promise<Size>;\n  sleep?: (timeMs: number) => Promise<void>;\n  includeSwipe?: boolean;\n  includePinch?: boolean;\n  systemActions?: SystemInputActionOptions;\n}\n\nfunction defineSystemInputAction(\n  config: SystemInputActionConfig,\n  call: () => Promise<void>,\n): DeviceAction<undefined, void> {\n  return defineAction<undefined, undefined, void>({\n    name: config.name,\n    description: config.description,\n    interfaceAlias: config.interfaceAlias,\n    delayBeforeRunner: config.delayBeforeRunner,\n    delayAfterRunner: config.delayAfterRunner,\n    call,\n  });\n}\n\nexport function defineActionsFromInputPrimitives(\n  input: InputPrimitives,\n  options: InputPrimitiveActionOptions = {},\n): DeviceAction<any>[] {\n  const actions: Array<DeviceAction<any> | undefined> = [];\n  const { pointer, keyboard, scroll, touch, system } = input;\n\n  if (pointer) {\n    actions.push(defineActionTap(pointer.tap));\n    if (pointer.doubleClick) {\n      actions.push(defineActionDoubleClick(pointer.doubleClick));\n    }\n    if (pointer.rightClick) {\n      actions.push(defineActionRightClick(pointer.rightClick));\n    }\n    if (pointer.hover) {\n      actions.push(defineActionHover(pointer.hover));\n    }\n    if (pointer.dragAndDrop) {\n      actions.push(defineActionDragAndDrop(pointer.dragAndDrop));\n    }\n    if (pointer.longPress) {\n      actions.push(defineActionLongPress(pointer.longPress));\n    }\n  }\n\n  if (keyboard) {\n    actions.push(\n      defineActionInput(keyboard),\n      defineActionClearInput(keyboard.clearInput),\n      defineActionKeyboardPress(keyboard.keyboardPress),\n      defineActionCursorMove({ keyboard, sleep: options.sleep }),\n    );\n  }\n\n  if (scroll) {\n    actions.push(defineActionScroll(scroll.scroll));\n  }\n\n  if (touch?.swipe && options.size && options.includeSwipe !== false) {\n    actions.push(defineActionSwipe({ swipe: touch.swipe, size: options.size }));\n  }\n\n  if (touch?.pinch && options.size && options.includePinch !== false) {\n    actions.push(defineActionPinch({ pinch: touch.pinch, size: options.size }));\n  }\n\n  if (system && options.systemActions) {\n    const { systemActions } = options;\n    if (system.backButton && systemActions.backButton) {\n      actions.push(\n        defineSystemInputAction(systemActions.backButton, system.backButton),\n      );\n    }\n    if (system.homeButton && systemActions.homeButton) {\n      actions.push(\n        defineSystemInputAction(systemActions.homeButton, system.homeButton),\n      );\n    }\n    if (system.recentAppsButton && systemActions.recentAppsButton) {\n      actions.push(\n        defineSystemInputAction(\n          systemActions.recentAppsButton,\n          system.recentAppsButton,\n        ),\n      );\n    }\n  }\n\n  return actions.filter((action): action is DeviceAction<any> =>\n    Boolean(action),\n  );\n}\n\nexport function createDefaultMobileActions(\n  context: MobileInputActionContext,\n): DeviceAction<any>[] {\n  return defineActionsFromInputPrimitives(context.input, {\n    size: context.size,\n    sleep: context.sleep,\n    systemActions: context.systemActions,\n  });\n}\n\n// Sleep\nexport const ActionSleepParamSchema = z.object({\n  timeMs: z\n    .number()\n    .default(1000)\n    .optional()\n    .describe('Sleep duration in milliseconds, defaults to 1000ms (1 second)'),\n});\n\nexport type ActionSleepParam = {\n  timeMs?: number;\n};\n\nexport const defineActionSleep = (): DeviceAction<ActionSleepParam> => {\n  return defineAction<typeof ActionSleepParamSchema, ActionSleepParam>({\n    name: 'Sleep',\n    description:\n      'Wait for a specified duration before continuing. Defaults to 1 second (1000ms) if not specified.',\n    paramSchema: ActionSleepParamSchema,\n    sample: {\n      timeMs: 2000,\n    },\n    call: async (param) => {\n      const duration = param?.timeMs ?? 1000;\n      getDebug('device:common-action')(`Sleeping for ${duration}ms`);\n      await new Promise((resolve) => setTimeout(resolve, duration));\n    },\n  });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n  AndroidDeviceOpt,\n  AndroidDeviceInputOpt,\n  IOSDeviceOpt,\n  IOSDeviceInputOpt,\n  HarmonyDeviceOpt,\n  HarmonyDeviceInputOpt,\n} from './device-options';\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","AbstractInterface","defineAction","config","pointFromLocate","locate","missingMessage","Error","defineLocatedPointAction","param","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","tap","point","actionRightClickParamSchema","defineActionRightClick","rightClick","actionDoubleClickParamSchema","defineActionDoubleClick","doubleClick","actionHoverParamSchema","defineActionHover","hover","inputLocateDescription","actionInputParamSchema","val","String","defineActionInput","keyboard","actionKeyboardPressParamSchema","defineActionKeyboardPress","keyboardPress","actionScrollParamSchema","defineActionScroll","scroll","actionDragAndDropParamSchema","defineActionDragAndDrop","dragAndDrop","from","to","ActionLongPressParamSchema","defineActionLongPress","longPress","ActionSwipeParamSchema","normalizeMobileSwipeParam","screenSize","width","height","start","end","startPoint","endPoint","direction","Math","duration","repeatCount","defineActionSwipe","i","actionClearInputParamSchema","defineActionClearInput","clearInput","actionCursorMoveParamSchema","defineActionCursorMove","times","wait","timeMs","Promise","resolve","setTimeout","arrowKey","ActionPinchParamSchema","defineActionPinch","centerX","centerY","startDistance","endDistance","normalizePinchParam","element","baseDistance","fingerDistance","defineSystemInputAction","call","defineActionsFromInputPrimitives","input","options","actions","pointer","touch","system","systemActions","action","Boolean","createDefaultMobileActions","context","ActionSleepParamSchema","defineActionSleep","getDebug"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC0HO,MAAeI;;QA8CpB;QA8BA;;AACF;AAKO,MAAMC,eAAe,CAK1BC,SAaOA;AAGT,SAASC,gBACPC,MAAuC,EACvCC,cAAsB;IAEtB,IAAI,CAACD,QACH,MAAM,IAAIE,MAAMD;IAElB,OAAO;QAAE,GAAGD,OAAO,MAAM,CAAC,EAAE;QAAE,GAAGA,OAAO,MAAM,CAAC,EAAE;IAAC;AACpD;AAEA,SAASG,yBAGPL,MAQD;IACC,OAAOD,aAA8B;QACnC,MAAMC,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW;QAC/B,gBAAgBA,OAAO,cAAc;QACrC,aAAaA,OAAO,WAAW;QAC/B,QAAQA,OAAO,MAAM;QACrB,MAAM,OAAOM;YACX,MAAMN,OAAO,IAAI,CACfC,gBAAgBK,MAAM,MAAM,EAAEN,OAAO,oBAAoB,GACzDM;QAEJ;IACF;AACF;AAGO,MAAMC,uBAAuBC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC3C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMC,kBAAkB,CAC7BC,MAEON,yBAAsE;QAC3E,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsB;QAC1C;QACA,sBAAsB;QACtB,MAAM,OAAOK;YACX,MAAMD,IAAIC;QACZ;IACF;AAIK,MAAMC,8BAA8BL,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMK,yBAAyB,CACpCC,aAEOV,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaQ;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAA+B;QACnD;QACA,sBAAsB;QACtB,MAAM,OAAOD;YACX,MAAMG,WAAWH;QACnB;IACF;AAIK,MAAMI,+BAA+BR,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMQ,0BAA0B,CACrCC,cAEOb,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAkB;QACtC;QACA,sBAAsB;QACtB,MAAM,OAAOJ;YACX,MAAMM,YAAYN;QACpB;IACF;AAIK,MAAMO,yBAAyBX,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMW,oBAAoB,CAC/BC,QAEOhB,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAac;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACA,sBAAsB;QACtB,MAAM,OAAOP;YACX,MAAMS,MAAMT;QACd;IACF;AAIF,MAAMU,yBACJ;AACK,MAAMC,yBAAyBf,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOA,6BAAAA,CAAAA,CAAAA,KACC,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,MAAQ;KAAG,EAC9B,SAAS,CAAC,CAACgB,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQf,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAACa,wBACT,QAAQ;IACX,MAAMd,6BAAAA,CAAAA,CAAAA,OACC,CAAC;QAAC;QAAW;QAAS;KAAW,EACrC,OAAO,CAAC,WACR,QAAQ,CACP;IAEJ,qBAAqBA,6BAAAA,CAAAA,CAAAA,OACX,GACP,QAAQ,GACR,QAAQ,CACP;AAEN;AAQO,MAAMkB,oBAAoB,CAC/BC,WAEO5B,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAawB;QACb,QAAQ;YACN,OAAO;YACP,QAAQ;gBAAE,QAAQ;YAAwB;QAC5C;QACA,MAAM,OAAOjB;YAEX,IAAKA,AAA0B,aAA1BA,MAAM,IAAI,EACbA,MAAM,IAAI,GAAG;YAGf,IAAIA,AAAe,YAAfA,MAAM,IAAI,EAAc,YAC1B,MAAMqB,SAAS,UAAU,CAACrB,MAAM,MAAM;YAIxC,IAAI,CAACA,SAAS,CAACA,MAAM,KAAK,EACxB;YAGF,MAAMqB,SAAS,QAAQ,CAACrB,MAAM,KAAK,EAAE;gBACnC,QAAQA,MAAM,MAAM;gBACpB,SAASA,AAAe,eAAfA,MAAM,IAAI;gBACnB,qBAAqBA,MAAM,mBAAmB;YAChD;QACF;IACF;AAIK,MAAMsB,iCAAiCpB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACrD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,6BAAAA,CAAAA,CAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMqB,4BAA4B,CACvCC,gBAEO/B,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAa6B;QACb,QAAQ;YACN,SAAS;QACX;QACA,MAAM,OAAOtB;YACX,MAAMwB,cAAcxB,MAAM,OAAO,EAAE;gBACjC,QAAQA,MAAM,MAAM;YACtB;QACF;IACF;AAIK,MAAMyB,0BAA0BvB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,YAAYA,6BAAAA,CAAAA,CAAAA,OACL,CAAC;QACJ;QACA;QACA;QACA;QACA;KACD,EACA,OAAO,CAAC,gBACR,QAAQ,CACP;IAEJ,WAAWA,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMuB,qBAAqB,CAChCC,SAEOlC,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAagC;QACb,QAAQ;YACN,WAAW;YACX,YAAY;YACZ,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACA,MAAM,OAAOzB;YACX,MAAM2B,OAAO3B;QACf;IACF;AAIK,MAAM4B,+BAA+B1B,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,MAAMC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;IAC3C,IAAIA,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAM0B,0BAA0B,CACrCC,cAEOrC,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamC;QACb,QAAQ;YACN,MAAM;gBAAE,QAAQ;YAA6B;YAC7C,IAAI;gBAAE,QAAQ;YAAuB;QACvC;QACA,MAAM,OAAO5B;YACX,MAAM+B,OAAO/B,MAAM,IAAI;YACvB,MAAMgC,KAAKhC,MAAM,EAAE;YACnB,IAAI,CAAC+B,MACH,MAAM,IAAIjC,MAAM;YAElB,IAAI,CAACkC,IACH,MAAM,IAAIlC,MAAM;YAElB,MAAMgC,YACJ;gBAAE,GAAGC,KAAK,MAAM,CAAC,EAAE;gBAAE,GAAGA,KAAK,MAAM,CAAC,EAAE;YAAC,GACvC;gBAAE,GAAGC,GAAG,MAAM,CAAC,EAAE;gBAAE,GAAGA,GAAG,MAAM,CAAC,EAAE;YAAC;QAEvC;IACF;AAGK,MAAMC,6BAA6B/B,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;IAEF,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMgC,wBAAwB,CACnCC,YAEOpC,yBAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAakC;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAqB;QACzC;QACA,sBAAsB;QACtB,MAAM,OAAO3B,OAAON;YAClB,MAAMmC,UAAU7B,OAAO;gBAAE,UAAUN,MAAM,QAAQ;YAAC;QACpD;IACF;AAGK,MAAMoC,yBAAyBlC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,6BAAAA,CAAAA,CAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,SAASmC,0BACdrC,KAAuB,EACvBsC,UAA6C;IAO7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAM,EAAEG,KAAK,EAAEC,GAAG,EAAE,GAAG1C;IAEvB,MAAM2C,aAAaF,QACf;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;IAAC,IACzC;QAAE,GAAGF,QAAQ;QAAG,GAAGC,SAAS;IAAE;IAElC,IAAII;IAEJ,IAAIF,KACFE,WAAW;QAAE,GAAGF,IAAI,MAAM,CAAC,EAAE;QAAE,GAAGA,IAAI,MAAM,CAAC,EAAE;IAAC;SAC3C,IAAI1C,MAAM,QAAQ,EAAE;QACzB,MAAM6C,YAAY7C,MAAM,SAAS;QACjC,IAAI,CAAC6C,WACH,MAAM,IAAI/C,MAAM;QAElB8C,WAAW;YACT,GACED,WAAW,CAAC,GACXE,CAAAA,AAAc,YAAdA,YACG7C,MAAM,QAAQ,GACd6C,AAAc,WAAdA,YACE,CAAC7C,MAAM,QAAQ,GACf;YACR,GACE2C,WAAW,CAAC,GACXE,CAAAA,AAAc,WAAdA,YACG7C,MAAM,QAAQ,GACd6C,AAAc,SAAdA,YACE,CAAC7C,MAAM,QAAQ,GACf;QACV;IACF,OACE,MAAM,IAAIF,MACR;IAIJ8C,SAAS,CAAC,GAAGE,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACF,SAAS,CAAC,EAAEL;IAC9CK,SAAS,CAAC,GAAGE,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACF,SAAS,CAAC,EAAEJ;IAE9C,MAAMO,WAAW/C,MAAM,QAAQ,IAAI;IAEnC,IAAIgD,cAAc,AAAwB,YAAxB,OAAOhD,MAAM,MAAM,GAAgBA,MAAM,MAAM,GAAG;IACpE,IAAIgD,AAAgB,MAAhBA,aACFA,cAAc;IAGhB,OAAO;QAAEL;QAAYC;QAAUG;QAAUC;IAAY;AACvD;AAEO,MAAMC,oBAAoB,CAACvD,SAIzBD,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa2C;QACb,QAAQ;YACN,OAAO;gBAAE,QAAQ;YAA6B;YAC9C,KAAK;gBAAE,QAAQ;YAA2B;QAC5C;QACA,MAAM,OAAOpC;YACX,MAAM,EAAE2C,UAAU,EAAEC,QAAQ,EAAEG,QAAQ,EAAEC,WAAW,EAAE,GACnDX,0BAA0BrC,OAAO,MAAMN,OAAO,IAAI;YACpD,IAAK,IAAIwD,IAAI,GAAGA,IAAIF,aAAaE,IAC/B,MAAMxD,OAAO,KAAK,CAACiD,YAAYC,UAAU;gBAAEG;YAAS;QAExD;IACF;AAIK,MAAMI,8BAA8BjD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,iCACT,QAAQ;AACb;AAKO,MAAMiD,yBAAyB,CACpCC,aAEO5D,aAGL;QACA,MAAM;QACN,aAAauB;QACb,gBAAgB;QAChB,aAAamC;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAyB;QAC7C;QACA,MAAM,OAAOnD;YACX,MAAMqD,WAAWrD,MAAM,MAAM;QAC/B;IACF;AAIK,MAAMsD,8BAA8BpD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,WAAWA,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAQ;KAAQ,EACtB,QAAQ,CAAC;IACZ,OAAOA,6BAAAA,CAAAA,CAAAA,MACE,GACN,GAAG,GACH,GAAG,CAAC,GACJ,OAAO,CAAC,GACR,QAAQ,CACP;AAEN;AAMO,MAAMqD,yBAAyB,CAAC7D,SAI9BD,aAGL;QACA,MAAM;QACN,aACE;QACF,aAAa6D;QACb,QAAQ;YACN,WAAW;YACX,OAAO;QACT;QACA,MAAM,OAAOtD;YACX,MAAMwD,QAAQxD,MAAM,KAAK,IAAI;YAC7B,IAAIN,OAAO,QAAQ,CAAC,UAAU,EAAE,YAC9B,MAAMA,OAAO,QAAQ,CAAC,UAAU,CAACM,MAAM,SAAS,EAAEwD;YAIpD,MAAMC,OACJ/D,OAAO,KAAK,IACV,EAAAgE,SACA,IAAIC,QAAc,CAACC,UAAYC,WAAWD,SAASF,QAAO;YAC9D,MAAMI,WAAW9D,AAAoB,WAApBA,MAAM,SAAS,GAAc,cAAc;YAC5D,IAAK,IAAIkD,IAAI,GAAGA,IAAIM,OAAON,IAAK;gBAC9B,MAAMxD,OAAO,QAAQ,CAAC,aAAa,CAACoE;gBACpC,MAAML,KAAK;YACb;QACF;IACF;AAIK,MAAMM,yBAAyB7D,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAM;KAAM,EAClB,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AASO,MAAM8D,oBAAoB,CAACtE;IAIhC,IAAI,CAACA,OAAO,KAAK,EACf;IAGF,OAAOD,aAA8D;QACnE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAasE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAe;YACjC,WAAW;YACX,UAAU;QACZ;QACA,MAAM,OAAO/D;YACX,MAAM,EAAEiE,OAAO,EAAEC,OAAO,EAAEC,aAAa,EAAEC,WAAW,EAAErB,QAAQ,EAAE,GAC9DsB,oBAAoBrE,OAAO,MAAMN,OAAO,IAAI;YAC9C,MAAMA,OAAO,KAAK,GAChB;gBAAE,GAAGuE;gBAAS,GAAGC;YAAQ,GACzB;gBAAEC;gBAAeC;gBAAarB;YAAS;QAE3C;IACF;AACF;AAEO,SAASsB,oBACdrE,KAAuB,EACvBsC,UAA6C;IAQ7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAMgC,UAAUtE,MAAM,MAAM;IAC5B,MAAMiE,UAAUK,UACZxB,KAAK,KAAK,CAACwB,QAAQ,MAAM,CAAC,EAAE,IAC5BxB,KAAK,KAAK,CAACP,QAAQ;IACvB,MAAM2B,UAAUI,UACZxB,KAAK,KAAK,CAACwB,QAAQ,MAAM,CAAC,EAAE,IAC5BxB,KAAK,KAAK,CAACN,SAAS;IACxB,MAAMO,WAAW/C,MAAM,QAAQ,IAAI;IAEnC,MAAMuE,eAAezB,KAAK,KAAK,CAACA,KAAK,GAAG,CAACP,OAAOC,UAAU;IAC1D,MAAMgC,iBAAiBxE,MAAM,QAAQ,IAAIuE;IAEzC,MAAMJ,gBAAgBI;IACtB,MAAMH,cACJpE,AAAoB,UAApBA,MAAM,SAAS,GACXuE,eAAeC,iBACf1B,KAAK,GAAG,CAAC,IAAIyB,eAAeC;IAElC,OAAO;QAAEP;QAASC;QAASC;QAAeC;QAAarB;IAAS;AAClE;AAgCA,SAAS0B,wBACP/E,MAA+B,EAC/BgF,IAAyB;IAEzB,OAAOjF,aAAyC;QAC9C,MAAMC,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW;QAC/B,gBAAgBA,OAAO,cAAc;QACrC,mBAAmBA,OAAO,iBAAiB;QAC3C,kBAAkBA,OAAO,gBAAgB;QACzCgF;IACF;AACF;AAEO,SAASC,iCACdC,KAAsB,EACtBC,UAAuC,CAAC,CAAC;IAEzC,MAAMC,UAAgD,EAAE;IACxD,MAAM,EAAEC,OAAO,EAAE1D,QAAQ,EAAEM,MAAM,EAAEqD,KAAK,EAAEC,MAAM,EAAE,GAAGL;IAErD,IAAIG,SAAS;QACXD,QAAQ,IAAI,CAAC1E,gBAAgB2E,QAAQ,GAAG;QACxC,IAAIA,QAAQ,WAAW,EACrBD,QAAQ,IAAI,CAACnE,wBAAwBoE,QAAQ,WAAW;QAE1D,IAAIA,QAAQ,UAAU,EACpBD,QAAQ,IAAI,CAACtE,uBAAuBuE,QAAQ,UAAU;QAExD,IAAIA,QAAQ,KAAK,EACfD,QAAQ,IAAI,CAAChE,kBAAkBiE,QAAQ,KAAK;QAE9C,IAAIA,QAAQ,WAAW,EACrBD,QAAQ,IAAI,CAACjD,wBAAwBkD,QAAQ,WAAW;QAE1D,IAAIA,QAAQ,SAAS,EACnBD,QAAQ,IAAI,CAAC5C,sBAAsB6C,QAAQ,SAAS;IAExD;IAEA,IAAI1D,UACFyD,QAAQ,IAAI,CACV1D,kBAAkBC,WAClB+B,uBAAuB/B,SAAS,UAAU,GAC1CE,0BAA0BF,SAAS,aAAa,GAChDkC,uBAAuB;QAAElC;QAAU,OAAOwD,QAAQ,KAAK;IAAC;IAI5D,IAAIlD,QACFmD,QAAQ,IAAI,CAACpD,mBAAmBC,OAAO,MAAM;IAG/C,IAAIqD,OAAO,SAASH,QAAQ,IAAI,IAAIA,AAAyB,UAAzBA,QAAQ,YAAY,EACtDC,QAAQ,IAAI,CAAC7B,kBAAkB;QAAE,OAAO+B,MAAM,KAAK;QAAE,MAAMH,QAAQ,IAAI;IAAC;IAG1E,IAAIG,OAAO,SAASH,QAAQ,IAAI,IAAIA,AAAyB,UAAzBA,QAAQ,YAAY,EACtDC,QAAQ,IAAI,CAACd,kBAAkB;QAAE,OAAOgB,MAAM,KAAK;QAAE,MAAMH,QAAQ,IAAI;IAAC;IAG1E,IAAII,UAAUJ,QAAQ,aAAa,EAAE;QACnC,MAAM,EAAEK,aAAa,EAAE,GAAGL;QAC1B,IAAII,OAAO,UAAU,IAAIC,cAAc,UAAU,EAC/CJ,QAAQ,IAAI,CACVL,wBAAwBS,cAAc,UAAU,EAAED,OAAO,UAAU;QAGvE,IAAIA,OAAO,UAAU,IAAIC,cAAc,UAAU,EAC/CJ,QAAQ,IAAI,CACVL,wBAAwBS,cAAc,UAAU,EAAED,OAAO,UAAU;QAGvE,IAAIA,OAAO,gBAAgB,IAAIC,cAAc,gBAAgB,EAC3DJ,QAAQ,IAAI,CACVL,wBACES,cAAc,gBAAgB,EAC9BD,OAAO,gBAAgB;IAI/B;IAEA,OAAOH,QAAQ,MAAM,CAAC,CAACK,SACrBC,QAAQD;AAEZ;AAEO,SAASE,2BACdC,OAAiC;IAEjC,OAAOX,iCAAiCW,QAAQ,KAAK,EAAE;QACrD,MAAMA,QAAQ,IAAI;QAClB,OAAOA,QAAQ,KAAK;QACpB,eAAeA,QAAQ,aAAa;IACtC;AACF;AAGO,MAAMC,yBAAyBrF,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQA,6BAAAA,CAAAA,CAAAA,MACC,GACN,OAAO,CAAC,MACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMsF,oBAAoB,IACxB/F,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa8F;QACb,QAAQ;YACN,QAAQ;QACV;QACA,MAAM,OAAOvF;YACX,MAAM+C,WAAW/C,OAAO,UAAU;YAClCyF,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,wBAAwB,CAAC,aAAa,EAAE1C,SAAS,EAAE,CAAC;YAC7D,MAAM,IAAIY,QAAQ,CAACC,UAAYC,WAAWD,SAASb;QACrD;IACF"}

package/dist/lib/service/index.js

@@ -26,7 +26,7 @@ __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
     default: ()=>Service
 });
-const util_js_namespaceObject = require("../ai-model/auto-glm/util.js");
+const errors_js_namespaceObject = require("../ai-model/errors.js");
 const inspect_js_namespaceObject = require("../ai-model/inspect.js");
 const describe_js_namespaceObject = require("../ai-model/prompt/describe.js");
 const index_js_namespaceObject = require("../ai-model/service-caller/index.js");
@@ -48,62 +48,27 @@ function _define_property(obj, key, value) {
 }
 const debug = (0, logger_namespaceObject.getDebug)('ai:service');
 class Service {
-    async locate(query, opt, modelConfig, abortSignal) {
+    async locate(query, opt, modelRuntime, abortSignal) {
+        const { config: modelConfig } = modelRuntime;
         const queryPrompt = 'string' == typeof query ? query : query.prompt;
         (0, utils_namespaceObject.assert)(queryPrompt, 'query is required for locate');
         (0, utils_namespaceObject.assert)('object' == typeof query, 'query should be an object for locate');
-        const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;
-        let searchAreaPrompt;
-        if (query.deepLocate && !hasPlanLocatedElement) searchAreaPrompt = query.prompt;
-        const { modelFamily } = modelConfig;
-        if (searchAreaPrompt && !modelFamily) {
-            console.warn('The "deepLocate" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config');
-            searchAreaPrompt = void 0;
-        }
-        if (searchAreaPrompt && (0, util_js_namespaceObject.isAutoGLM)(modelFamily)) {
-            console.warn('The "deepLocate" feature is not supported with AutoGLM.');
-            searchAreaPrompt = void 0;
-        }
+        if (!modelConfig.modelFamily) throw new Error(errors_js_namespaceObject.defaultModelFamilyRequiredForLocateMessage);
         const context = opt?.context || await this.contextRetrieverFn();
-        let searchArea;
-        let searchAreaRawResponse;
-        let searchAreaUsage;
-        let searchAreaResponse;
-        if (query.deepLocate && hasPlanLocatedElement) {
-            const searchAreaConfig = await (0, inspect_js_namespaceObject.buildSearchAreaConfig)({
-                context,
-                baseRect: opt.planLocatedElement.rect,
-                modelFamily
-            });
-            searchArea = searchAreaConfig.rect;
-            searchAreaRawResponse = JSON.stringify({
-                source: 'plan-located-element',
-                rect: opt.planLocatedElement.rect
-            });
-            searchAreaResponse = {
-                rect: searchArea,
-                imageBase64: searchAreaConfig.imageBase64,
-                scale: searchAreaConfig.scale,
-                rawResponse: searchAreaRawResponse
-            };
-        } else if (searchAreaPrompt) {
-            searchAreaResponse = await (0, inspect_js_namespaceObject.AiLocateSection)({
-                context,
-                sectionDescription: searchAreaPrompt,
-                modelConfig,
-                abortSignal
-            });
-            (0, utils_namespaceObject.assert)(searchAreaResponse.rect, `cannot find search area for "${searchAreaPrompt}"${searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''}`);
-            searchAreaRawResponse = searchAreaResponse.rawResponse;
-            searchAreaUsage = searchAreaResponse.usage;
-            searchArea = searchAreaResponse.rect;
-        }
+        const searchArea = await this.resolveLocateSearchArea({
+            query,
+            queryPrompt,
+            opt,
+            context,
+            modelRuntime,
+            abortSignal
+        });
         const startTime = Date.now();
         const { parseResult, rect, rawResponse, usage, reasoning_content } = await (0, inspect_js_namespaceObject.AiLocateElement)({
             context,
             targetElementDescription: queryPrompt,
-            searchConfig: searchAreaResponse,
-            modelConfig,
+            searchConfig: searchArea.config,
+            modelRuntime,
             abortSignal
         });
         const timeCost = Date.now() - startTime;
@@ -113,9 +78,9 @@ class Service {
             rawResponse: JSON.stringify(rawResponse),
             formatResponse: JSON.stringify(parseResult),
             usage,
-            searchArea,
-            searchAreaRawResponse,
-            searchAreaUsage,
+            searchArea: searchArea.trace.sourceRect,
+            searchAreaRawResponse: searchArea.trace.rawResponse,
+            searchAreaUsage: searchArea.trace.usage,
             reasoning_content
         };
         let errorLog;
@@ -125,25 +90,25 @@ class Service {
             userQuery: {
                 element: queryPrompt
             },
-            matchedElement: [],
             matchedRect: rect,
             data: null,
             taskInfo,
-            deepLocate: !!searchArea,
+            deepLocate: !!searchArea.trace.sourceRect,
             error: errorLog
         };
-        const elements = parseResult.elements || [];
+        const element = parseResult.element;
         const dump = (0, external_utils_js_namespaceObject.createServiceDump)({
             ...dumpData,
-            matchedElement: elements
+            matchedElement: element ? [
+                element
+            ] : []
         });
         if (errorLog) throw new external_types_js_namespaceObject.ServiceError(errorLog, dump);
-        if (elements.length > 1) throw new external_types_js_namespaceObject.ServiceError(`locate: multiple elements found, length = ${elements.length}`, dump);
-        if (1 === elements.length) return {
+        if (element) return {
             element: {
-                center: elements[0].center,
-                rect: elements[0].rect,
-                description: elements[0].description
+                center: element.center,
+                rect: element.rect,
+                description: element.description
             },
             rect,
             dump
@@ -154,7 +119,72 @@ class Service {
             dump
         };
     }
-    async extract(dataDemand, modelConfig, opt, pageDescription, multimodalPrompt, context) {
+    async resolveLocateSearchArea(options) {
+        const { query, queryPrompt, opt, context, modelRuntime, abortSignal } = options;
+        const { adapter } = modelRuntime;
+        const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;
+        if (!query.deepLocate) return {
+            trace: {}
+        };
+        if (hasPlanLocatedElement) {
+            const config = await (0, inspect_js_namespaceObject.buildSearchAreaConfig)({
+                context,
+                baseRect: opt.planLocatedElement.rect
+            });
+            return {
+                config,
+                trace: {
+                    sourceRect: config.sourceRect,
+                    rawResponse: JSON.stringify({
+                        source: 'plan-located-element',
+                        rect: opt.planLocatedElement.rect
+                    })
+                }
+            };
+        }
+        if (adapter.locate.supportsSearchArea) {
+            const searchAreaResponse = await (0, inspect_js_namespaceObject.AiLocateSection)({
+                context,
+                sectionDescription: queryPrompt,
+                modelRuntime,
+                abortSignal
+            });
+            const { searchAreaConfig } = searchAreaResponse;
+            (0, utils_namespaceObject.assert)(searchAreaConfig, `cannot find search area for "${queryPrompt}"${searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''}`);
+            return {
+                config: searchAreaConfig,
+                trace: {
+                    sourceRect: searchAreaConfig.sourceRect,
+                    rawResponse: searchAreaResponse.rawResponse,
+                    usage: searchAreaResponse.usage
+                }
+            };
+        }
+        const firstPassLocateResult = await (0, inspect_js_namespaceObject.AiLocateElement)({
+            context,
+            targetElementDescription: queryPrompt,
+            modelRuntime,
+            abortSignal
+        });
+        (0, utils_namespaceObject.assert)(firstPassLocateResult.rect, `cannot find search area for "${queryPrompt}"${firstPassLocateResult.parseResult.errors?.length ? `: ${firstPassLocateResult.parseResult.errors.join('\n')}` : ''}`);
+        const config = await (0, inspect_js_namespaceObject.buildSearchAreaConfig)({
+            context,
+            baseRect: firstPassLocateResult.rect
+        });
+        return {
+            config,
+            trace: {
+                sourceRect: config.sourceRect,
+                rawResponse: JSON.stringify({
+                    source: 'deep-locate-first-pass',
+                    rect: firstPassLocateResult.rect,
+                    rawResponse: firstPassLocateResult.rawResponse
+                }),
+                usage: firstPassLocateResult.usage
+            }
+        };
+    }
+    async extract(dataDemand, modelRuntime, opt, pageDescription, multimodalPrompt, context) {
         (0, utils_namespaceObject.assert)(context, 'context is required for extract');
         (0, utils_namespaceObject.assert)('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
         const startTime = Date.now();
@@ -168,7 +198,7 @@ class Service {
                 dataQuery: dataDemand,
                 multimodalPrompt,
                 extractOption: opt,
-                modelConfig,
+                modelRuntime,
                 pageDescription
             });
             parseResult = result.parseResult;
@@ -189,7 +219,6 @@ class Service {
                     userQuery: {
                         dataDemand
                     },
-                    matchedElement: [],
                     data: null,
                     taskInfo,
                     error: error.message
@@ -214,7 +243,6 @@ class Service {
             userQuery: {
                 dataDemand
             },
-            matchedElement: [],
             data: null,
             taskInfo,
             error: errorLog
@@ -233,13 +261,12 @@ class Service {
             dump
         };
     }
-    async describe(target, modelConfig, opt) {
+    async describe(target, modelRuntime, opt) {
         (0, utils_namespaceObject.assert)(target, 'target is required for service.describe');
         const context = await this.contextRetrieverFn();
         const { shotSize } = context;
         const screenshotBase64 = context.screenshot.base64;
         (0, utils_namespaceObject.assert)(screenshotBase64, 'screenshot is required for service.describe');
-        const { modelFamily } = modelConfig;
         const systemPrompt = (0, describe_js_namespaceObject.elementDescriberInstruction)();
         const defaultRectSize = 30;
         const targetRect = Array.isArray(target) ? {
@@ -261,7 +288,7 @@ class Service {
         if (opt?.deepLocate) {
             const searchArea = (0, external_common_js_namespaceObject.expandSearchArea)(targetRect, shotSize);
             debug('describe: cropping to searchArea', searchArea);
-            const croppedResult = await (0, img_namespaceObject.cropByRect)(imagePayload, searchArea, 'qwen2.5-vl' === modelFamily);
+            const croppedResult = await (0, img_namespaceObject.cropByRect)(imagePayload, searchArea);
             imagePayload = croppedResult.imageBase64;
         }
         const msgs = [
@@ -282,7 +309,7 @@ class Service {
                 ]
             }
         ];
-        const res = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, modelConfig);
+        const res = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, modelRuntime);
         const { content } = res;
         (0, utils_namespaceObject.assert)(!content.error, `describe failed: ${content.error}`);
         (0, utils_namespaceObject.assert)(content.description, 'failed to describe the element');

package/dist/lib/service/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"service/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  AiLocateSection,\n  buildSearchAreaConfig,\n} from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport {\n  AIResponseParseError,\n  callAIWithObjectResponse,\n} from '@/ai-model/service-caller';\nimport { type AIArgs, expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  PlanningLocateParam,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: PlanningLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n    let searchAreaPrompt;\n    if (query.deepLocate && !hasPlanLocatedElement) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { modelFamily } = modelConfig;\n\n    if (searchAreaPrompt && !modelFamily) {\n      console.warn(\n        'The \"deepLocate\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    if (searchAreaPrompt && isAutoGLM(modelFamily)) {\n      console.warn('The \"deepLocate\" feature is not supported with AutoGLM.');\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (query.deepLocate && hasPlanLocatedElement) {\n      const searchAreaConfig = await buildSearchAreaConfig({\n        context,\n        baseRect: opt.planLocatedElement!.rect,\n        modelFamily,\n      });\n      searchArea = searchAreaConfig.rect;\n\n      searchAreaRawResponse = JSON.stringify({\n        source: 'plan-located-element',\n        rect: opt.planLocatedElement!.rect,\n      });\n      searchAreaResponse = {\n        rect: searchArea,\n        imageBase64: searchAreaConfig.imageBase64,\n        scale: searchAreaConfig.scale,\n        rawResponse: searchAreaRawResponse,\n      };\n    } else if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n        abortSignal,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage, reasoning_content } =\n      await AiLocateElement({\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchAreaResponse,\n        modelConfig,\n        abortSignal,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new ServiceError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          description: elements[0]!.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelConfig,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          matchedElement: [],\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepLocate?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { modelFamily } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        modelFamily === 'qwen2.5-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n      msgs,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Service","query","opt","modelConfig","abortSignal","queryPrompt","assert","hasPlanLocatedElement","searchAreaPrompt","modelFamily","console","undefined","isAutoGLM","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","searchAreaConfig","buildSearchAreaConfig","JSON","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;AC2CA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACzBC,WAAyB,EACM;QAC/B,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QAEpBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMM,wBAAwB,CAAC,CAACL,KAAK,oBAAoB;QAEzD,IAAIM;QACJ,IAAIP,MAAM,UAAU,IAAI,CAACM,uBACvBC,mBAAmBP,MAAM,MAAM;QAGjC,MAAM,EAAEQ,WAAW,EAAE,GAAGN;QAExB,IAAIK,oBAAoB,CAACC,aAAa;YACpCC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,IAAIH,oBAAoBI,AAAAA,IAAAA,wBAAAA,SAAAA,AAAAA,EAAUH,cAAc;YAC9CC,QAAQ,IAAI,CAAC;YACbF,mBAAmBG;QACrB;QAEA,MAAME,UAAUX,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIY;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIhB,MAAM,UAAU,IAAIM,uBAAuB;YAC7C,MAAMW,mBAAmB,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;gBACnDN;gBACA,UAAUX,IAAI,kBAAkB,CAAE,IAAI;gBACtCO;YACF;YACAK,aAAaI,iBAAiB,IAAI;YAElCH,wBAAwBK,KAAK,SAAS,CAAC;gBACrC,QAAQ;gBACR,MAAMlB,IAAI,kBAAkB,CAAE,IAAI;YACpC;YACAe,qBAAqB;gBACnB,MAAMH;gBACN,aAAaI,iBAAiB,WAAW;gBACzC,OAAOA,iBAAiB,KAAK;gBAC7B,aAAaH;YACf;QACF,OAAO,IAAIP,kBAAkB;YAC3BS,qBAAqB,MAAMI,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBACzCR;gBACA,oBAAoBL;gBACpBL;gBACAC;YACF;YACAE,IAAAA,sBAAAA,MAAAA,AAAAA,EACEW,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAET,iBAAiB,CAAC,EAChDS,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAMK,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAEC,iBAAiB,EAAE,GAChE,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YACpBhB;YACA,0BAA0BR;YAC1B,cAAcY;YACdd;YACAC;QACF;QAEF,MAAM0B,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaV,KAAK,SAAS,CAACM;YAC5B,gBAAgBN,KAAK,SAAS,CAACI;YAC/BG;YACAb;YACAC;YACAC;YACAY;QACF;QAEA,IAAII;QACJ,IAAIR,YAAY,MAAM,EAAE,QACtBQ,WAAW,CAAC,4BAA4B,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS5B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAaoB;YACb,MAAM;YACNM;YACA,YAAY,CAAC,CAACjB;YACd,OAAOkB;QACT;QAEA,MAAME,WAAWV,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMW,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,kCAAAA,YAAYA,CACpB,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,aAAaA,QAAQ,CAAC,EAAE,CAAE,WAAW;YACvC;YACAT;YACAU;QACF;QAGF,OAAO;YACL,SAAS;YACTV;YACAU;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BnC,WAAyB,EACzBD,GAA0B,EAC1BqC,eAAwB,EACxBC,gBAAoC,EACpC3B,OAAmB,EACe;QAClCP,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOO,SAAS;QAChBP,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAOgC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMhB,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMa,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,oBAAAA,AAAAA,EAAwB;gBAC3C7B;gBACA,WAAWyB;gBACXE;gBACA,eAAetC;gBACfC;gBACAoC;YACF;YACAf,cAAciB,OAAO,WAAW;YAChCf,cAAce,OAAO,WAAW;YAChCd,QAAQc,OAAO,KAAK;YACpBb,oBAAoBa,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,yBAAAA,oBAAoBA,EAAE;gBAEzC,MAAMd,WAAWP,KAAK,GAAG,KAAKD;gBAC9B,MAAMS,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAaa,MAAM,WAAW;oBAC9B,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMR,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEE;oBAAW;oBACxB,gBAAgB,EAAE;oBAClB,MAAM;oBACNP;oBACA,OAAOY,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIN,kCAAAA,YAAYA,CAACM,MAAM,OAAO,EAAER;YACxC;YACA,MAAMQ;QACR;QAEA,MAAMb,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZJ;YACA,gBAAgBN,KAAK,SAAS,CAACI;YAC/BG;YACAC;QACF;QAEA,IAAII;QACJ,IAAIR,YAAY,MAAM,EAAE,QACtBQ,WAAW,CAAC,qBAAqB,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNP;YACA,OAAOC;QACT;QAEA,MAAM,EAAEa,IAAI,EAAEC,OAAO,EAAE,GAAGtB,eAAe,CAAC;QAE1C,MAAMW,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACXY;QACF;QAEA,IAAIb,YAAY,CAACa,MACf,MAAM,IAAIR,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,OAAO;YACLU;YACAC;YACAnB;YACAC;YACAO;QACF;IACF;IAEA,MAAM,SACJY,MAA+B,EAC/B5C,WAAyB,EACzBD,GAEC,EACwD;QACzDI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOyC,QAAQ;QACf,MAAMlC,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEmC,QAAQ,EAAE,GAAGnC;QACrB,MAAMoC,mBAAmBpC,QAAQ,UAAU,CAAC,MAAM;QAClDP,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO2C,kBAAkB;QAEzB,MAAM,EAAExC,WAAW,EAAE,GAAGN;QACxB,MAAM+C,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAInD,KAAK,YAAY;YACnB,MAAMY,aAAa4C,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;YAMhDlD,MAAM,oCAAoCgB;YAC1C,MAAM6C,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BJ,cACA1C,YACAL,AAAgB,iBAAhBA;YAEF+C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChBF,MACA1D;QAGF,MAAM,EAAE6D,OAAO,EAAE,GAAGF;QACpBxD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC0D,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1D1D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO0D,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAlWA,YACEnD,OAA2D,EAC3DX,GAAoB,CACpB;QAPF;QAEA;QAMEI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOO,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMoD,QAAQ,OAAO,CAACpD;QAGlD,IAAI,AAAyB,WAAlBX,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAqVF"}
+{"version":3,"file":"service/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { defaultModelFamilyRequiredForLocateMessage } from '@/ai-model/errors';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  AiLocateSection,\n  buildSearchAreaConfig,\n} from '@/ai-model/inspect';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport {\n  AIResponseParseError,\n  callAIWithObjectResponse,\n} from '@/ai-model/service-caller';\nimport type { AIArgs } from '@/ai-model/types';\nimport type { SearchAreaConfig } from '@/ai-model/workflows/inspect/types';\nimport { expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  PlanningLocateParam,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\ninterface LocateSearchAreaResult {\n  config?: SearchAreaConfig;\n  trace: {\n    sourceRect?: Rect;\n    rawResponse?: string;\n    usage?: AIUsageInfo;\n  };\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: PlanningLocateParam,\n    opt: LocateOpts,\n    modelRuntime: ModelRuntime,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const { config: modelConfig } = modelRuntime;\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    if (!modelConfig.modelFamily) {\n      throw new Error(defaultModelFamilyRequiredForLocateMessage);\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    const searchArea = await this.resolveLocateSearchArea({\n      query,\n      queryPrompt,\n      opt,\n      context,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage, reasoning_content } =\n      await AiLocateElement({\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchArea.config,\n        modelRuntime,\n        abortSignal,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea: searchArea.trace.sourceRect,\n      searchAreaRawResponse: searchArea.trace.rawResponse,\n      searchAreaUsage: searchArea.trace.usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea.trace.sourceRect,\n      error: errorLog,\n    };\n\n    const element = parseResult.element;\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: element ? [element] : [],\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (element) {\n      return {\n        element: {\n          center: element.center,\n          rect: element.rect,\n          description: element.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  private async resolveLocateSearchArea(options: {\n    query: PlanningLocateParam;\n    queryPrompt: TUserPrompt;\n    opt: LocateOpts;\n    context: UIContext;\n    modelRuntime: ModelRuntime;\n    abortSignal?: AbortSignal;\n  }): Promise<LocateSearchAreaResult> {\n    const { query, queryPrompt, opt, context, modelRuntime, abortSignal } =\n      options;\n    const { adapter } = modelRuntime;\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n    if (!query.deepLocate) {\n      return { trace: {} };\n    }\n\n    if (hasPlanLocatedElement) {\n      const config = await buildSearchAreaConfig({\n        context,\n        baseRect: opt.planLocatedElement!.rect,\n      });\n\n      return {\n        config,\n        trace: {\n          sourceRect: config.sourceRect,\n          rawResponse: JSON.stringify({\n            source: 'plan-located-element',\n            rect: opt.planLocatedElement!.rect,\n          }),\n        },\n      };\n    }\n\n    if (adapter.locate.supportsSearchArea) {\n      const searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: queryPrompt,\n        modelRuntime,\n        abortSignal,\n      });\n      const { searchAreaConfig } = searchAreaResponse;\n      assert(\n        searchAreaConfig,\n        `cannot find search area for \"${queryPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n\n      return {\n        config: searchAreaConfig,\n        trace: {\n          sourceRect: searchAreaConfig.sourceRect,\n          rawResponse: searchAreaResponse.rawResponse,\n          usage: searchAreaResponse.usage,\n        },\n      };\n    }\n\n    const firstPassLocateResult = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      modelRuntime,\n      abortSignal,\n    });\n    assert(\n      firstPassLocateResult.rect,\n      `cannot find search area for \"${queryPrompt}\"${\n        firstPassLocateResult.parseResult.errors?.length\n          ? `: ${firstPassLocateResult.parseResult.errors.join('\\n')}`\n          : ''\n      }`,\n    );\n\n    const config = await buildSearchAreaConfig({\n      context,\n      baseRect: firstPassLocateResult.rect,\n    });\n\n    return {\n      config,\n      trace: {\n        sourceRect: config.sourceRect,\n        rawResponse: JSON.stringify({\n          source: 'deep-locate-first-pass',\n          rect: firstPassLocateResult.rect,\n          rawResponse: firstPassLocateResult.rawResponse,\n        }),\n        usage: firstPassLocateResult.usage,\n      },\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelRuntime,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelRuntime: ModelRuntime,\n    opt?: {\n      deepLocate?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      // Describe is not a coordinate-parsing flow, so it does not need image\n      // padding for bbox normalization.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(imagePayload, searchArea);\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n      msgs,\n      modelRuntime,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Service","query","opt","modelRuntime","abortSignal","modelConfig","queryPrompt","assert","Error","defaultModelFamilyRequiredForLocateMessage","context","searchArea","startTime","Date","parseResult","rect","rawResponse","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","element","dump","createServiceDump","ServiceError","options","adapter","hasPlanLocatedElement","config","buildSearchAreaConfig","searchAreaResponse","AiLocateSection","searchAreaConfig","firstPassLocateResult","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;ACsDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,YAA0B,EAC1BC,WAAyB,EACM;QAC/B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGF;QAChC,MAAMG,cAAc,AAAiB,YAAjB,OAAOL,QAAqBA,QAAQA,MAAM,MAAM;QACpEM,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QAEpBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAON,OAAoB;QAElC,IAAI,CAACI,YAAY,WAAW,EAC1B,MAAM,IAAIG,MAAMC,0BAAAA,0CAA0CA;QAG5D,MAAMC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,MAAMS,aAAa,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACpDV;YACAK;YACAJ;YACAQ;YACAP;YACAC;QACF;QAEA,MAAMQ,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAEC,iBAAiB,EAAE,GAChE,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YACpBT;YACA,0BAA0BJ;YAC1B,cAAcK,WAAW,MAAM;YAC/BR;YACAC;QACF;QAEF,MAAMgB,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACA,YAAYN,WAAW,KAAK,CAAC,UAAU;YACvC,uBAAuBA,WAAW,KAAK,CAAC,WAAW;YACnD,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACvCO;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,4BAA4B,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASlB;YACX;YACA,aAAaS;YACb,MAAM;YACNM;YACA,YAAY,CAAC,CAACV,WAAW,KAAK,CAAC,UAAU;YACzC,OAAOY;QACT;QAEA,MAAME,UAAUX,YAAY,OAAO;QAEnC,MAAMY,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC,UAAU;gBAACA;aAAQ,GAAG,EAAE;QAC1C;QAEA,IAAIF,UACF,MAAM,IAAIK,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,IAAID,SACF,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,MAAM;gBACtB,MAAMA,QAAQ,IAAI;gBAClB,aAAaA,QAAQ,WAAW;YAClC;YACAV;YACAW;QACF;QAGF,OAAO;YACL,SAAS;YACTX;YACAW;QACF;IACF;IAEA,MAAc,wBAAwBG,OAOrC,EAAmC;QAClC,MAAM,EAAE5B,KAAK,EAAEK,WAAW,EAAEJ,GAAG,EAAEQ,OAAO,EAAEP,YAAY,EAAEC,WAAW,EAAE,GACnEyB;QACF,MAAM,EAAEC,OAAO,EAAE,GAAG3B;QACpB,MAAM4B,wBAAwB,CAAC,CAAC7B,KAAK,oBAAoB;QAEzD,IAAI,CAACD,MAAM,UAAU,EACnB,OAAO;YAAE,OAAO,CAAC;QAAE;QAGrB,IAAI8B,uBAAuB;YACzB,MAAMC,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;gBACzCvB;gBACA,UAAUR,IAAI,kBAAkB,CAAE,IAAI;YACxC;YAEA,OAAO;gBACL8B;gBACA,OAAO;oBACL,YAAYA,OAAO,UAAU;oBAC7B,aAAaV,KAAK,SAAS,CAAC;wBAC1B,QAAQ;wBACR,MAAMpB,IAAI,kBAAkB,CAAE,IAAI;oBACpC;gBACF;YACF;QACF;QAEA,IAAI4B,QAAQ,MAAM,CAAC,kBAAkB,EAAE;YACrC,MAAMI,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBAC/CzB;gBACA,oBAAoBJ;gBACpBH;gBACAC;YACF;YACA,MAAM,EAAEgC,gBAAgB,EAAE,GAAGF;YAC7B3B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE6B,kBACA,CAAC,6BAA6B,EAAE9B,YAAY,CAAC,EAC3C4B,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAGJ,OAAO;gBACL,QAAQE;gBACR,OAAO;oBACL,YAAYA,iBAAiB,UAAU;oBACvC,aAAaF,mBAAmB,WAAW;oBAC3C,OAAOA,mBAAmB,KAAK;gBACjC;YACF;QACF;QAEA,MAAMG,wBAAwB,MAAMlB,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YAClDT;YACA,0BAA0BJ;YAC1BH;YACAC;QACF;QACAG,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8B,sBAAsB,IAAI,EAC1B,CAAC,6BAA6B,EAAE/B,YAAY,CAAC,EAC3C+B,sBAAsB,WAAW,CAAC,MAAM,EAAE,SACtC,CAAC,EAAE,EAAEA,sBAAsB,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,GAC1D,IACJ;QAGJ,MAAML,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;YACzCvB;YACA,UAAU2B,sBAAsB,IAAI;QACtC;QAEA,OAAO;YACLL;YACA,OAAO;gBACL,YAAYA,OAAO,UAAU;gBAC7B,aAAaV,KAAK,SAAS,CAAC;oBAC1B,QAAQ;oBACR,MAAMe,sBAAsB,IAAI;oBAChC,aAAaA,sBAAsB,WAAW;gBAChD;gBACA,OAAOA,sBAAsB,KAAK;YACpC;QACF;IACF;IAEA,MAAM,QACJC,UAA+B,EAC/BnC,YAA0B,EAC1BD,GAA0B,EAC1BqC,eAAwB,EACxBC,gBAAoC,EACpC9B,OAAmB,EACe;QAClCH,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,SAAS;QAChBH,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAO+B,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAM1B,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMuB,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,oBAAAA,AAAAA,EAAwB;gBAC3ChC;gBACA,WAAW4B;gBACXE;gBACA,eAAetC;gBACfC;gBACAoC;YACF;YACAzB,cAAc2B,OAAO,WAAW;YAChCzB,cAAcyB,OAAO,WAAW;YAChCxB,QAAQwB,OAAO,KAAK;YACpBvB,oBAAoBuB,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,yBAAAA,oBAAoBA,EAAE;gBAEzC,MAAMxB,WAAWP,KAAK,GAAG,KAAKD;gBAC9B,MAAMS,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAauB,MAAM,WAAW;oBAC9B,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMjB,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEW;oBAAW;oBACxB,MAAM;oBACNjB;oBACA,OAAOsB,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIf,kCAAAA,YAAYA,CAACe,MAAM,OAAO,EAAEjB;YACxC;YACA,MAAMiB;QACR;QAEA,MAAMvB,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZJ;YACA,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,qBAAqB,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTc;YACF;YACA,MAAM;YACNjB;YACA,OAAOE;QACT;QAEA,MAAM,EAAEsB,IAAI,EAAEC,OAAO,EAAE,GAAGhC,eAAe,CAAC;QAE1C,MAAMY,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACXqB;QACF;QAEA,IAAItB,YAAY,CAACsB,MACf,MAAM,IAAIjB,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,OAAO;YACLmB;YACAC;YACA7B;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJqB,MAA+B,EAC/B5C,YAA0B,EAC1BD,GAEC,EACwD;QACzDK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOwC,QAAQ;QACf,MAAMrC,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEsC,QAAQ,EAAE,GAAGtC;QACrB,MAAMuC,mBAAmBvC,QAAQ,UAAU,CAAC,MAAM;QAClDH,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO0C,kBAAkB;QACzB,MAAMC,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAInD,KAAK,YAAY;YACnB,MAAMS,aAAa+C,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;YAQhDlD,MAAM,oCAAoCa;YAC1C,MAAMgD,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAAWJ,cAAc7C;YACrD6C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChBF,MACA1D;QAGF,MAAM,EAAE6D,OAAO,EAAE,GAAGF;QACpBvD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACyD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DzD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOyD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAjYA,YACEtD,OAA2D,EAC3DR,GAAoB,CACpB;QAPF;QAEA;QAMEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMuD,QAAQ,OAAO,CAACvD;QAGlD,IAAI,AAAyB,WAAlBR,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAoXF"}