@midscene/core 1.9.6 → 1.9.7

package/dist/lib/recorder-ui-describer.js.map

@@ -1 +1 @@
-{"version":3,"file":"recorder-ui-describer.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/recorder-ui-describer.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { IModelConfig } from '@midscene/shared/env';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport type {\n  MidsceneRecorderEvent,\n  MidsceneRecorderPageInfo,\n  MidsceneRecorderTarget,\n} from '@midscene/shared/recorder';\nimport { RECORDER_UI_DESCRIBER_SYSTEM_PROMPT } from './ai-model/prompt/recorder-ui-describer';\nimport { callAIWithObjectResponse } from './ai-model/service-caller';\nimport type { Rect } from './types';\n\nexport interface DescribeRecorderUIEventInput {\n  event: MidsceneRecorderEvent;\n  target?: MidsceneRecorderTarget;\n}\n\nexport interface DescribeRecorderUIEventOptions {\n  maxRetries?: number;\n  retryDelayMs?: number;\n  concurrency?: number;\n}\n\nexport interface DescribeRecorderUIEventResult {\n  event: MidsceneRecorderEvent;\n  usedFallback: boolean;\n  error?: string;\n}\n\ninterface RecorderUIEventAIResponse {\n  elementDescription?: string;\n  replayInstruction?: string;\n  actionSummary?: string;\n  scrollDestinationDescription?: string;\n  confidence?: 'high' | 'medium' | 'low';\n  error?: string;\n}\n\nconst RECORDER_UI_DESCRIBER_DEFAULT_RETRIES = 2;\nconst RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS = 200;\nconst RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY = 2;\n\nfunction delay(ms: number) {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nfunction clamp(value: number, min: number, max: number) {\n  return Math.min(Math.max(value, min), max);\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction isPendingDescription(value?: string) {\n  return value?.trim() === 'AI is analyzing element...';\n}\n\nfunction getRecorderEventScreenshot(event: MidsceneRecorderEvent) {\n  return (\n    event.screenshotWithBox || event.screenshotBefore || event.screenshotAfter\n  );\n}\n\nfunction getRecorderEventAfterScreenshot(event: MidsceneRecorderEvent) {\n  return event.screenshotAfter || event.screenshotWithBox;\n}\n\nfunction normalizeActionType(event: MidsceneRecorderEvent) {\n  return event.actionType?.trim();\n}\n\nfunction getPlatformId(target?: MidsceneRecorderTarget) {\n  return target?.platformId?.toLowerCase();\n}\n\nfunction getPlatformSurface(target?: MidsceneRecorderTarget) {\n  switch (getPlatformId(target)) {\n    case 'web':\n      return 'current web page';\n    case 'android':\n    case 'ios':\n    case 'harmony':\n      return 'current mobile screen';\n    case 'computer':\n      return 'current desktop screen';\n    default:\n      return 'current UI';\n  }\n}\n\nfunction getPlatformGuidance(target?: MidsceneRecorderTarget) {\n  switch (getPlatformId(target)) {\n    case 'web':\n      return 'For web targets, use web UI terms such as button, input, link, menu item, tab, dialog, aria-label, placeholder, and form section when visible or inferable.';\n    case 'android':\n    case 'ios':\n    case 'harmony':\n      return 'For mobile targets, use mobile UI terms such as tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, and screen section.';\n    case 'computer':\n      return 'For desktop/computer targets, use desktop UI terms such as menu item, toolbar button, dialog field, sidebar item, window control, file row, and application region.';\n    default:\n      return 'Use platform-neutral UI terms such as control, field, item, icon button, list item, region, panel, and page section.';\n  }\n}\n\nfunction getPointerActionVerb(event: MidsceneRecorderEvent) {\n  switch (normalizeActionType(event)) {\n    case 'Tap':\n      return 'Tap';\n    case 'DoubleClick':\n      return 'Double click';\n    case 'LongPress':\n      return 'Long press';\n    case 'RightClick':\n      return 'Right click';\n    default:\n      return 'Click';\n  }\n}\n\nfunction getDragActionVerb(event: MidsceneRecorderEvent) {\n  switch (normalizeActionType(event)) {\n    case 'Swipe':\n      return 'Swipe';\n    case 'DragAndDrop':\n      return 'Drag';\n    default:\n      return 'Drag';\n  }\n}\n\nfunction getActionReplayInstruction(\n  event: MidsceneRecorderEvent,\n  elementDescription: string,\n) {\n  if (event.type === 'click') {\n    const verb = getPointerActionVerb(event);\n    if (verb === 'Long press') {\n      return `${verb} the element described as \"${elementDescription}\".`;\n    }\n    return `${verb} on the element described as \"${elementDescription}\".`;\n  }\n\n  if (event.type === 'drag') {\n    const verb = getDragActionVerb(event);\n    return `${verb} through the area described as \"${elementDescription}\".`;\n  }\n\n  return undefined;\n}\n\nfunction getActionSummaryVerb(event: MidsceneRecorderEvent) {\n  if (event.type === 'click') {\n    return getPointerActionVerb(event);\n  }\n  if (event.type === 'drag') {\n    return getDragActionVerb(event);\n  }\n  if (event.type === 'keydown') {\n    return 'Press';\n  }\n  return undefined;\n}\n\nfunction getCanonicalReplayInstruction(\n  event: MidsceneRecorderEvent,\n  elementDescription: string,\n) {\n  switch (event.type) {\n    case 'click':\n    case 'drag':\n    case 'input':\n    case 'keydown':\n      return getFallbackReplayInstruction(event, elementDescription);\n    default:\n      return undefined;\n  }\n}\n\nfunction getCanonicalActionSummary(\n  event: MidsceneRecorderEvent,\n  elementDescription: string,\n) {\n  switch (event.type) {\n    case 'click':\n    case 'drag':\n    case 'input':\n    case 'keydown':\n      return getFallbackActionSummary(event, elementDescription);\n    default:\n      return undefined;\n  }\n}\n\nfunction pointToRect(\n  x: number,\n  y: number,\n  size: number,\n  pageInfo: MidsceneRecorderPageInfo,\n): Rect {\n  const width = pageInfo.width || size;\n  const height = pageInfo.height || size;\n  const left = clamp(Math.floor(x - size / 2), 0, Math.max(width - 1, 0));\n  const top = clamp(Math.floor(y - size / 2), 0, Math.max(height - 1, 0));\n  return {\n    left,\n    top,\n    width: Math.min(size, Math.max(width - left, 1)),\n    height: Math.min(size, Math.max(height - top, 1)),\n  };\n}\n\nfunction getPointRectSize(event: MidsceneRecorderEvent) {\n  switch (event.type) {\n    case 'scroll':\n      return 96;\n    case 'drag':\n      return 64;\n    default:\n      return 36;\n  }\n}\n\nexport function getRecorderUIEventTargetRect(\n  event: MidsceneRecorderEvent,\n): Rect | null {\n  const rect = event.elementRect;\n  if (!rect) {\n    return null;\n  }\n\n  if (\n    isFiniteNumber(rect.width) &&\n    rect.width > 0 &&\n    isFiniteNumber(rect.height) &&\n    rect.height > 0 &&\n    (isFiniteNumber(rect.left) || isFiniteNumber(rect.top))\n  ) {\n    return {\n      left: rect.left || 0,\n      top: rect.top || 0,\n      width: rect.width,\n      height: rect.height,\n    };\n  }\n\n  if (isFiniteNumber(rect.x) && isFiniteNumber(rect.y)) {\n    return pointToRect(rect.x, rect.y, getPointRectSize(event), event.pageInfo);\n  }\n\n  return null;\n}\n\nfunction getFallbackDescription(\n  event: MidsceneRecorderEvent,\n  target?: MidsceneRecorderTarget,\n) {\n  const pageContext = getPageSemanticContext(event);\n  const surface = getPlatformSurface(target);\n\n  switch (event.type) {\n    case 'navigation':\n      return event.url || event.value || event.actionType || 'navigation';\n    case 'scroll':\n      return pageContext\n        ? `${pageContext} scrollable content`\n        : `scrollable content on the ${surface}`;\n    case 'drag':\n      return pageContext\n        ? `gesture area in ${pageContext}`\n        : `gesture area on the ${surface}`;\n    case 'input':\n      return pageContext\n        ? `input field in ${pageContext}`\n        : `input field on the ${surface}`;\n    case 'keydown':\n      return pageContext\n        ? `focused control in ${pageContext}`\n        : `focused control on the ${surface}`;\n    default:\n      return pageContext\n        ? `control in ${pageContext}`\n        : `control on the ${surface}`;\n  }\n}\n\nfunction getFallbackReplayInstruction(\n  event: MidsceneRecorderEvent,\n  elementDescription: string,\n) {\n  switch (event.type) {\n    case 'navigation':\n      return event.url\n        ? `Navigate to \\`${event.url}\\`.`\n        : `Navigate using ${elementDescription}.`;\n    case 'scroll':\n      return `Scroll the page/region with description \"${elementDescription}\" by value \"${event.value || 'down'}\".`;\n    case 'drag':\n      return (\n        getActionReplayInstruction(event, elementDescription) ||\n        `Drag through the area described as \"${elementDescription}\".`\n      );\n    case 'input':\n      return `Input \"${event.value || ''}\" into the element described as \"${elementDescription}\".`;\n    case 'keydown':\n      return `Press \"${event.value || 'the recorded key'}\" on the element described as \"${elementDescription}\".`;\n    default:\n      return (\n        getActionReplayInstruction(event, elementDescription) ||\n        `Click on the element described as \"${elementDescription}\".`\n      );\n  }\n}\n\nfunction getFallbackActionSummary(\n  event: MidsceneRecorderEvent,\n  elementDescription: string,\n) {\n  switch (event.type) {\n    case 'navigation':\n      return event.url ? `Navigate to ${event.url}` : 'Navigate';\n    case 'scroll':\n      return `Scroll ${elementDescription}`;\n    case 'drag':\n      return `${getActionSummaryVerb(event) || 'Drag'} ${elementDescription}`;\n    case 'input':\n      return `Input into ${elementDescription}`;\n    case 'keydown':\n      return `Press ${event.value || 'key'} on ${elementDescription}`;\n    default:\n      return `${getActionSummaryVerb(event) || 'Click'} ${elementDescription}`;\n  }\n}\n\nfunction getActionGuidance(\n  event: MidsceneRecorderEvent,\n  target?: MidsceneRecorderTarget,\n) {\n  const platformGuidance = getPlatformGuidance(target);\n\n  switch (event.type) {\n    case 'click':\n      return `${platformGuidance} Identify the ${getPointerActionVerb(event).toLowerCase()} target by exact visible text first, then label/placeholder, then role plus stable surrounding context, then icon purpose, then visual position. Never describe it by coordinates, marker location, or as a nearby element.`;\n    case 'input':\n      return `${platformGuidance} Identify the exact input field at the marker before text entry. Use visible label, placeholder, field name, or stable surrounding section. Preserve the recorded input value only in replayInstruction; never describe the typed value as the field.`;\n    case 'scroll':\n      return `${platformGuidance} Identify the scrollable page/region and concrete destination content revealed after scrolling. Use newly visible headings, section titles, list/table names, list items, or stable region labels; never say only \"more content\" or \"current page\".`;\n    case 'drag':\n      return `${platformGuidance} Identify the ${getDragActionVerb(event).toLowerCase()} start/end regions or the dragged UI control. Do not describe only the gesture path or coordinates.`;\n    case 'keydown':\n      return `${platformGuidance} Identify the focused element or keyboard target if visible, and preserve the recorded key in the replay instruction.`;\n    default:\n      return `${platformGuidance} Identify the UI target involved in this event using the most stable visible text or surrounding context.`;\n  }\n}\n\nfunction getEventRawCoordinates(event: MidsceneRecorderEvent) {\n  const x = event.elementRect?.x;\n  const y = event.elementRect?.y;\n  if (isFiniteNumber(x) && isFiniteNumber(y)) {\n    return { x, y };\n  }\n  return undefined;\n}\n\nfunction getPageSemanticContext(event: MidsceneRecorderEvent) {\n  const candidates = [event.title, event.url]\n    .map((item) => item?.trim())\n    .filter(Boolean) as string[];\n  return candidates[0];\n}\n\nfunction isWeakDescription(value?: string) {\n  if (!value) {\n    return true;\n  }\n  if (isPendingDescription(value)) {\n    return true;\n  }\n  const normalized = value.trim().toLowerCase();\n  const compact = normalized.replace(/\\s+/g, '');\n  return (\n    normalized.length === 0 ||\n    /^\\(?\\d+(?:\\.\\d+)?,\\s*\\d+(?:\\.\\d+)?\\)?$/.test(normalized) ||\n    normalized === 'target' ||\n    normalized === 'element' ||\n    normalized === 'target element' ||\n    normalized === 'the element' ||\n    normalized === 'page element' ||\n    normalized === 'input field' ||\n    normalized === 'text input' ||\n    normalized === 'text field' ||\n    normalized === 'search box' ||\n    normalized === 'more content' ||\n    normalized === 'the page' ||\n    normalized === 'current page' ||\n    normalized === 'current screen' ||\n    normalized === 'the screen' ||\n    normalized === 'current ui' ||\n    normalized === 'current visible ui' ||\n    normalized === 'current visible page' ||\n    normalized === 'current visible screen' ||\n    normalized === 'main area' ||\n    normalized === 'main scrollable area' ||\n    normalized === 'scrollable area' ||\n    normalized === 'highlighted element' ||\n    normalized === 'highlighted item' ||\n    normalized === 'marked element' ||\n    normalized === 'marked item' ||\n    normalized.includes('ai is analyzing element') ||\n    compact.includes('坐标') ||\n    compact.includes('附近') ||\n    compact.includes('附近的元素') ||\n    normalized.includes('coordinate') ||\n    normalized.includes('near the coordinate') ||\n    normalized.includes('near coordinates') ||\n    normalized.includes('nearby element') ||\n    normalized.includes('nearby item') ||\n    normalized.includes('near the marker') ||\n    normalized.includes('near marker') ||\n    normalized.includes('near the point') ||\n    normalized.includes('near point') ||\n    normalized.includes('at the point') ||\n    normalized.includes('button near point') ||\n    normalized.includes('shown in the screenshot') ||\n    normalized.includes('red rectangle') ||\n    normalized.includes('red marker') ||\n    normalized.includes('red box') ||\n    normalized.includes('highlighted element') ||\n    normalized.includes('highlighted item') ||\n    normalized.includes('highlighted screenshot')\n  );\n}\n\nfunction isWeakReplayInstruction(value?: string) {\n  if (!value) {\n    return true;\n  }\n  if (isPendingDescription(value)) {\n    return true;\n  }\n  const normalized = value.trim().toLowerCase();\n  const compact = normalized.replace(/\\s+/g, '');\n  return (\n    compact.includes('坐标') ||\n    compact.includes('附近') ||\n    normalized.includes('coordinate') ||\n    normalized.includes('near the coordinate') ||\n    normalized.includes('nearby element') ||\n    normalized.includes('nearby item') ||\n    normalized.includes('near the marker') ||\n    normalized.includes('near marker') ||\n    normalized.includes('near the point') ||\n    normalized.includes('near point') ||\n    normalized.includes('at the point') ||\n    normalized.includes('ai is analyzing element') ||\n    normalized.includes('more content') ||\n    normalized.includes('current page') ||\n    normalized.includes('current screen') ||\n    normalized.includes('highlighted element') ||\n    normalized.includes('highlighted item') ||\n    normalized.includes('red marker') ||\n    normalized.includes('red box') ||\n    normalized.includes('shown in the screenshot') ||\n    normalized.includes('highlighted screenshot')\n  );\n}\n\nfunction normalizeForComparison(value: string) {\n  return value.trim().toLowerCase().replace(/[\"'`]/g, '').replace(/\\s+/g, ' ');\n}\n\nfunction isInputValueUsedAsFieldDescription(\n  event: MidsceneRecorderEvent,\n  elementDescription?: string,\n) {\n  if (event.type !== 'input' || !event.value || !elementDescription) {\n    return false;\n  }\n\n  const typedValue = normalizeForComparison(event.value);\n  if (!typedValue) {\n    return false;\n  }\n  const description = normalizeForComparison(elementDescription);\n\n  return (\n    description === typedValue ||\n    description === `${typedValue} input` ||\n    description === `${typedValue} field` ||\n    description === `${typedValue} text field` ||\n    description === `input ${typedValue}` ||\n    description === `field ${typedValue}` ||\n    description.includes(`typed value ${typedValue}`) ||\n    description.includes(`value ${typedValue}`)\n  );\n}\n\nfunction hasScrollDestination(\n  replayInstruction: string,\n  scrollDestinationDescription?: string,\n) {\n  if (\n    scrollDestinationDescription &&\n    !isWeakDescription(scrollDestinationDescription)\n  ) {\n    return true;\n  }\n  const normalized = replayInstruction.toLowerCase();\n  return (\n    normalized.includes(' until ') ||\n    normalized.includes(' visible') ||\n    normalized.includes(' reveal') ||\n    normalized.includes(' to the ') ||\n    normalized.includes(' toward ')\n  );\n}\n\nasync function describeWithRetry(\n  event: MidsceneRecorderEvent,\n  target: MidsceneRecorderTarget | undefined,\n  highlightedScreenshot: string,\n  modelConfig: IModelConfig,\n  options: Required<\n    Pick<DescribeRecorderUIEventOptions, 'maxRetries' | 'retryDelayMs'>\n  >,\n) {\n  let lastError: unknown;\n  for (let attempt = 1; attempt <= options.maxRetries; attempt += 1) {\n    try {\n      const afterScreenshot = getRecorderEventAfterScreenshot(event);\n      const pageContext = getPageSemanticContext(event);\n      const platformGuidance = getPlatformGuidance(target);\n      const userContent: any[] = [\n        {\n          type: 'text',\n          text: `Recorder event:\n${JSON.stringify(\n  {\n    type: event.type,\n    actionType: event.actionType,\n    value: event.value,\n    rawCoordinates: getEventRawCoordinates(event),\n    url: event.url,\n    title: event.title,\n    pageContext,\n    pageInfo: event.pageInfo,\n    target,\n    platformGuidance,\n    guidance: getActionGuidance(event, target),\n  },\n  null,\n  2,\n)}\n\nThe target or region is highlighted in the screenshot below. Convert this event into semantic replay fields.`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: highlightedScreenshot,\n            detail: 'high',\n          },\n        },\n      ];\n      if (afterScreenshot) {\n        userContent.push(\n          {\n            type: 'text',\n            text: 'Screenshot after the recorded action, for context only:',\n          },\n          {\n            type: 'image_url',\n            image_url: {\n              url: afterScreenshot,\n              detail: 'high',\n            },\n          },\n        );\n      }\n      const response =\n        await callAIWithObjectResponse<RecorderUIEventAIResponse>(\n          [\n            {\n              role: 'system',\n              content: RECORDER_UI_DESCRIBER_SYSTEM_PROMPT,\n            },\n            {\n              role: 'user',\n              content: userContent,\n            },\n          ],\n          modelConfig,\n        );\n\n      const content = response.content;\n      if (content.error) {\n        throw new Error(content.error);\n      }\n      if (isWeakDescription(content.elementDescription)) {\n        throw new Error('AI returned a weak recorder event description.');\n      }\n      if (\n        isInputValueUsedAsFieldDescription(event, content.elementDescription)\n      ) {\n        throw new Error(\n          'AI used the recorded input value as the field description.',\n        );\n      }\n      const elementDescription = content.elementDescription!.trim();\n      const scrollDestinationDescription =\n        event.type === 'scroll'\n          ? content.scrollDestinationDescription?.trim()\n          : undefined;\n      if (\n        event.type === 'scroll' &&\n        !hasScrollDestination('', scrollDestinationDescription)\n      ) {\n        throw new Error(\n          'AI returned a scroll description without a destination.',\n        );\n      }\n      const aiReplayInstruction = content.replayInstruction?.trim();\n      if (aiReplayInstruction && isWeakReplayInstruction(aiReplayInstruction)) {\n        throw new Error('AI returned a weak recorder replay instruction.');\n      }\n      const replayInstruction =\n        event.type === 'scroll' && scrollDestinationDescription\n          ? `Scroll the page/region with description \"${elementDescription}\" by value \"${event.value || 'down'}\" until \"${scrollDestinationDescription}\" is visible.`\n          : getCanonicalReplayInstruction(event, elementDescription) ||\n            aiReplayInstruction ||\n            getFallbackReplayInstruction(event, elementDescription);\n      if (isWeakReplayInstruction(replayInstruction)) {\n        throw new Error('AI returned a weak recorder replay instruction.');\n      }\n      const actionSummary =\n        event.type === 'scroll' && scrollDestinationDescription\n          ? `Scroll ${elementDescription} toward ${scrollDestinationDescription}`\n          : getCanonicalActionSummary(event, elementDescription) ||\n            content.actionSummary?.trim() ||\n            getFallbackActionSummary(event, elementDescription);\n\n      return {\n        elementDescription,\n        replayInstruction,\n        actionSummary,\n        semanticConfidence: content.confidence || 'medium',\n      };\n    } catch (error) {\n      lastError = error;\n      if (attempt < options.maxRetries) {\n        await delay(options.retryDelayMs);\n      }\n    }\n  }\n  throw lastError;\n}\n\nasync function createScreenshotWithBox(\n  event: MidsceneRecorderEvent,\n  rect: Rect,\n) {\n  if (event.screenshotWithBox) {\n    return event.screenshotWithBox;\n  }\n  const screenshot = getRecorderEventScreenshot(event);\n  if (!screenshot) {\n    return undefined;\n  }\n  return compositeElementInfoImg({\n    inputImgBase64: screenshot,\n    size: event.pageInfo,\n    elementsPositionInfo: [{ rect }],\n    borderThickness: 3,\n    annotationPadding: 2,\n  });\n}\n\nfunction createFallbackEvent(\n  event: MidsceneRecorderEvent,\n  error: string,\n  screenshotWithBox?: string,\n  target?: MidsceneRecorderTarget,\n): MidsceneRecorderEvent {\n  const elementDescription =\n    event.elementDescription && !isWeakDescription(event.elementDescription)\n      ? event.elementDescription\n      : getFallbackDescription(event, target);\n  return {\n    ...event,\n    elementDescription,\n    replayInstruction:\n      event.replayInstruction ||\n      getFallbackReplayInstruction(event, elementDescription),\n    actionSummary:\n      event.actionSummary ||\n      getFallbackActionSummary(event, elementDescription),\n    semanticConfidence: 'low',\n    descriptionLoading: false,\n    descriptionSource: 'fallback',\n    descriptionError: error,\n    screenshotWithBox: screenshotWithBox || event.screenshotWithBox,\n  };\n}\n\nexport async function describeRecorderUIEvent(\n  input: DescribeRecorderUIEventInput,\n  modelConfig: IModelConfig,\n  options: DescribeRecorderUIEventOptions = {},\n): Promise<DescribeRecorderUIEventResult> {\n  const event = input.event;\n  const rect = getRecorderUIEventTargetRect(event);\n  const screenshot = getRecorderEventScreenshot(event);\n\n  if (!rect || !screenshot) {\n    const error = !rect\n      ? 'Recorder event has no target rectangle.'\n      : 'Recorder event has no screenshot.';\n    return {\n      usedFallback: true,\n      event: createFallbackEvent(event, error, undefined, input.target),\n    };\n  }\n\n  let screenshotWithBox: string | undefined;\n  try {\n    screenshotWithBox = await createScreenshotWithBox(event, rect);\n    const semanticFields = await describeWithRetry(\n      event,\n      input.target,\n      screenshotWithBox || screenshot,\n      modelConfig,\n      {\n        maxRetries: options.maxRetries ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRIES,\n        retryDelayMs:\n          options.retryDelayMs ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS,\n      },\n    );\n    return {\n      usedFallback: false,\n      event: {\n        ...event,\n        ...semanticFields,\n        descriptionLoading: false,\n        descriptionSource: 'ai',\n        descriptionError: undefined,\n        screenshotWithBox: screenshotWithBox || event.screenshotWithBox,\n      },\n    };\n  } catch (error) {\n    const message = error instanceof Error ? error.message : String(error);\n    return {\n      usedFallback: true,\n      error: message,\n      event: createFallbackEvent(\n        event,\n        message,\n        screenshotWithBox,\n        input.target,\n      ),\n    };\n  }\n}\n\nexport async function describeRecorderUIEvents(\n  inputs: DescribeRecorderUIEventInput[],\n  modelConfig: IModelConfig,\n  options: DescribeRecorderUIEventOptions = {},\n): Promise<DescribeRecorderUIEventResult[]> {\n  const concurrency = Math.max(\n    1,\n    options.concurrency ?? RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY,\n  );\n  const results: DescribeRecorderUIEventResult[] = new Array(inputs.length);\n  let cursor = 0;\n\n  async function worker() {\n    while (cursor < inputs.length) {\n      const index = cursor;\n      cursor += 1;\n      results[index] = await describeRecorderUIEvent(\n        inputs[index],\n        modelConfig,\n        options,\n      );\n    }\n  }\n\n  await Promise.all(\n    Array.from({ length: Math.min(concurrency, inputs.length) }, () =>\n      worker(),\n    ),\n  );\n  return results;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","RECORDER_UI_DESCRIBER_DEFAULT_RETRIES","RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS","RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY","delay","ms","Promise","resolve","setTimeout","clamp","value","min","max","Math","isFiniteNumber","Number","isPendingDescription","getRecorderEventScreenshot","event","getRecorderEventAfterScreenshot","normalizeActionType","getPlatformId","target","getPlatformSurface","getPlatformGuidance","getPointerActionVerb","getDragActionVerb","getActionReplayInstruction","elementDescription","verb","getActionSummaryVerb","getCanonicalReplayInstruction","getFallbackReplayInstruction","getCanonicalActionSummary","getFallbackActionSummary","pointToRect","x","y","size","pageInfo","width","height","left","top","getPointRectSize","getRecorderUIEventTargetRect","rect","getFallbackDescription","pageContext","getPageSemanticContext","surface","getActionGuidance","platformGuidance","getEventRawCoordinates","candidates","item","Boolean","isWeakDescription","normalized","compact","isWeakReplayInstruction","normalizeForComparison","isInputValueUsedAsFieldDescription","typedValue","description","hasScrollDestination","replayInstruction","scrollDestinationDescription","describeWithRetry","highlightedScreenshot","modelConfig","options","lastError","attempt","afterScreenshot","userContent","JSON","response","callAIWithObjectResponse","RECORDER_UI_DESCRIBER_SYSTEM_PROMPT","content","Error","undefined","aiReplayInstruction","actionSummary","error","createScreenshotWithBox","screenshot","compositeElementInfoImg","createFallbackEvent","screenshotWithBox","describeRecorderUIEvent","input","semanticFields","message","String","describeRecorderUIEvents","inputs","concurrency","results","Array","cursor","worker","index"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;AC+BA,MAAMI,wCAAwC;AAC9C,MAAMC,+CAA+C;AACrD,MAAMC,4CAA4C;AAElD,SAASC,MAAMC,EAAU;IACvB,OAAO,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;AACtD;AAEA,SAASI,MAAMC,KAAa,EAAEC,GAAW,EAAEC,GAAW;IACpD,OAAOC,KAAK,GAAG,CAACA,KAAK,GAAG,CAACH,OAAOC,MAAMC;AACxC;AAEA,SAASE,eAAeJ,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBK,OAAO,QAAQ,CAACL;AACtD;AAEA,SAASM,qBAAqBN,KAAc;IAC1C,OAAOA,OAAO,WAAW;AAC3B;AAEA,SAASO,2BAA2BC,KAA4B;IAC9D,OACEA,MAAM,iBAAiB,IAAIA,MAAM,gBAAgB,IAAIA,MAAM,eAAe;AAE9E;AAEA,SAASC,gCAAgCD,KAA4B;IACnE,OAAOA,MAAM,eAAe,IAAIA,MAAM,iBAAiB;AACzD;AAEA,SAASE,oBAAoBF,KAA4B;IACvD,OAAOA,MAAM,UAAU,EAAE;AAC3B;AAEA,SAASG,cAAcC,MAA+B;IACpD,OAAOA,QAAQ,YAAY;AAC7B;AAEA,SAASC,mBAAmBD,MAA+B;IACzD,OAAQD,cAAcC;QACpB,KAAK;YACH,OAAO;QACT,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASE,oBAAoBF,MAA+B;IAC1D,OAAQD,cAAcC;QACpB,KAAK;YACH,OAAO;QACT,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASG,qBAAqBP,KAA4B;IACxD,OAAQE,oBAAoBF;QAC1B,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASQ,kBAAkBR,KAA4B;IACrD,OAAQE,oBAAoBF;QAC1B,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASS,2BACPT,KAA4B,EAC5BU,kBAA0B;IAE1B,IAAIV,AAAe,YAAfA,MAAM,IAAI,EAAc;QAC1B,MAAMW,OAAOJ,qBAAqBP;QAClC,IAAIW,AAAS,iBAATA,MACF,OAAO,GAAGA,KAAK,2BAA2B,EAAED,mBAAmB,EAAE,CAAC;QAEpE,OAAO,GAAGC,KAAK,8BAA8B,EAAED,mBAAmB,EAAE,CAAC;IACvE;IAEA,IAAIV,AAAe,WAAfA,MAAM,IAAI,EAAa;QACzB,MAAMW,OAAOH,kBAAkBR;QAC/B,OAAO,GAAGW,KAAK,gCAAgC,EAAED,mBAAmB,EAAE,CAAC;IACzE;AAGF;AAEA,SAASE,qBAAqBZ,KAA4B;IACxD,IAAIA,AAAe,YAAfA,MAAM,IAAI,EACZ,OAAOO,qBAAqBP;IAE9B,IAAIA,AAAe,WAAfA,MAAM,IAAI,EACZ,OAAOQ,kBAAkBR;IAE3B,IAAIA,AAAe,cAAfA,MAAM,IAAI,EACZ,OAAO;AAGX;AAEA,SAASa,8BACPb,KAA4B,EAC5BU,kBAA0B;IAE1B,OAAQV,MAAM,IAAI;QAChB,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOc,6BAA6Bd,OAAOU;QAC7C;YACE;IACJ;AACF;AAEA,SAASK,0BACPf,KAA4B,EAC5BU,kBAA0B;IAE1B,OAAQV,MAAM,IAAI;QAChB,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOgB,yBAAyBhB,OAAOU;QACzC;YACE;IACJ;AACF;AAEA,SAASO,YACPC,CAAS,EACTC,CAAS,EACTC,IAAY,EACZC,QAAkC;IAElC,MAAMC,QAAQD,SAAS,KAAK,IAAID;IAChC,MAAMG,SAASF,SAAS,MAAM,IAAID;IAClC,MAAMI,OAAOjC,MAAMI,KAAK,KAAK,CAACuB,IAAIE,OAAO,IAAI,GAAGzB,KAAK,GAAG,CAAC2B,QAAQ,GAAG;IACpE,MAAMG,MAAMlC,MAAMI,KAAK,KAAK,CAACwB,IAAIC,OAAO,IAAI,GAAGzB,KAAK,GAAG,CAAC4B,SAAS,GAAG;IACpE,OAAO;QACLC;QACAC;QACA,OAAO9B,KAAK,GAAG,CAACyB,MAAMzB,KAAK,GAAG,CAAC2B,QAAQE,MAAM;QAC7C,QAAQ7B,KAAK,GAAG,CAACyB,MAAMzB,KAAK,GAAG,CAAC4B,SAASE,KAAK;IAChD;AACF;AAEA,SAASC,iBAAiB1B,KAA4B;IACpD,OAAQA,MAAM,IAAI;QAChB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEO,SAAS2B,6BACd3B,KAA4B;IAE5B,MAAM4B,OAAO5B,MAAM,WAAW;IAC9B,IAAI,CAAC4B,MACH,OAAO;IAGT,IACEhC,eAAegC,KAAK,KAAK,KACzBA,KAAK,KAAK,GAAG,KACbhC,eAAegC,KAAK,MAAM,KAC1BA,KAAK,MAAM,GAAG,KACbhC,CAAAA,eAAegC,KAAK,IAAI,KAAKhC,eAAegC,KAAK,GAAG,IAErD,OAAO;QACL,MAAMA,KAAK,IAAI,IAAI;QACnB,KAAKA,KAAK,GAAG,IAAI;QACjB,OAAOA,KAAK,KAAK;QACjB,QAAQA,KAAK,MAAM;IACrB;IAGF,IAAIhC,eAAegC,KAAK,CAAC,KAAKhC,eAAegC,KAAK,CAAC,GACjD,OAAOX,YAAYW,KAAK,CAAC,EAAEA,KAAK,CAAC,EAAEF,iBAAiB1B,QAAQA,MAAM,QAAQ;IAG5E,OAAO;AACT;AAEA,SAAS6B,uBACP7B,KAA4B,EAC5BI,MAA+B;IAE/B,MAAM0B,cAAcC,uBAAuB/B;IAC3C,MAAMgC,UAAU3B,mBAAmBD;IAEnC,OAAQJ,MAAM,IAAI;QAChB,KAAK;YACH,OAAOA,MAAM,GAAG,IAAIA,MAAM,KAAK,IAAIA,MAAM,UAAU,IAAI;QACzD,KAAK;YACH,OAAO8B,cACH,GAAGA,YAAY,mBAAmB,CAAC,GACnC,CAAC,0BAA0B,EAAEE,SAAS;QAC5C,KAAK;YACH,OAAOF,cACH,CAAC,gBAAgB,EAAEA,aAAa,GAChC,CAAC,oBAAoB,EAAEE,SAAS;QACtC,KAAK;YACH,OAAOF,cACH,CAAC,eAAe,EAAEA,aAAa,GAC/B,CAAC,mBAAmB,EAAEE,SAAS;QACrC,KAAK;YACH,OAAOF,cACH,CAAC,mBAAmB,EAAEA,aAAa,GACnC,CAAC,uBAAuB,EAAEE,SAAS;QACzC;YACE,OAAOF,cACH,CAAC,WAAW,EAAEA,aAAa,GAC3B,CAAC,eAAe,EAAEE,SAAS;IACnC;AACF;AAEA,SAASlB,6BACPd,KAA4B,EAC5BU,kBAA0B;IAE1B,OAAQV,MAAM,IAAI;QAChB,KAAK;YACH,OAAOA,MAAM,GAAG,GACZ,CAAC,cAAc,EAAEA,MAAM,GAAG,CAAC,GAAG,CAAC,GAC/B,CAAC,eAAe,EAAEU,mBAAmB,CAAC,CAAC;QAC7C,KAAK;YACH,OAAO,CAAC,yCAAyC,EAAEA,mBAAmB,YAAY,EAAEV,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC/G,KAAK;YACH,OACES,2BAA2BT,OAAOU,uBAClC,CAAC,oCAAoC,EAAEA,mBAAmB,EAAE,CAAC;QAEjE,KAAK;YACH,OAAO,CAAC,OAAO,EAAEV,MAAM,KAAK,IAAI,GAAG,iCAAiC,EAAEU,mBAAmB,EAAE,CAAC;QAC9F,KAAK;YACH,OAAO,CAAC,OAAO,EAAEV,MAAM,KAAK,IAAI,mBAAmB,+BAA+B,EAAEU,mBAAmB,EAAE,CAAC;QAC5G;YACE,OACED,2BAA2BT,OAAOU,uBAClC,CAAC,mCAAmC,EAAEA,mBAAmB,EAAE,CAAC;IAElE;AACF;AAEA,SAASM,yBACPhB,KAA4B,EAC5BU,kBAA0B;IAE1B,OAAQV,MAAM,IAAI;QAChB,KAAK;YACH,OAAOA,MAAM,GAAG,GAAG,CAAC,YAAY,EAAEA,MAAM,GAAG,EAAE,GAAG;QAClD,KAAK;YACH,OAAO,CAAC,OAAO,EAAEU,oBAAoB;QACvC,KAAK;YACH,OAAO,GAAGE,qBAAqBZ,UAAU,OAAO,CAAC,EAAEU,oBAAoB;QACzE,KAAK;YACH,OAAO,CAAC,WAAW,EAAEA,oBAAoB;QAC3C,KAAK;YACH,OAAO,CAAC,MAAM,EAAEV,MAAM,KAAK,IAAI,MAAM,IAAI,EAAEU,oBAAoB;QACjE;YACE,OAAO,GAAGE,qBAAqBZ,UAAU,QAAQ,CAAC,EAAEU,oBAAoB;IAC5E;AACF;AAEA,SAASuB,kBACPjC,KAA4B,EAC5BI,MAA+B;IAE/B,MAAM8B,mBAAmB5B,oBAAoBF;IAE7C,OAAQJ,MAAM,IAAI;QAChB,KAAK;YACH,OAAO,GAAGkC,iBAAiB,cAAc,EAAE3B,qBAAqBP,OAAO,WAAW,GAAG,2NAA2N,CAAC;QACnT,KAAK;YACH,OAAO,GAAGkC,iBAAiB,qPAAqP,CAAC;QACnR,KAAK;YACH,OAAO,GAAGA,iBAAiB,mPAAmP,CAAC;QACjR,KAAK;YACH,OAAO,GAAGA,iBAAiB,cAAc,EAAE1B,kBAAkBR,OAAO,WAAW,GAAG,mGAAmG,CAAC;QACxL,KAAK;YACH,OAAO,GAAGkC,iBAAiB,qHAAqH,CAAC;QACnJ;YACE,OAAO,GAAGA,iBAAiB,yGAAyG,CAAC;IACzI;AACF;AAEA,SAASC,uBAAuBnC,KAA4B;IAC1D,MAAMkB,IAAIlB,MAAM,WAAW,EAAE;IAC7B,MAAMmB,IAAInB,MAAM,WAAW,EAAE;IAC7B,IAAIJ,eAAesB,MAAMtB,eAAeuB,IACtC,OAAO;QAAED;QAAGC;IAAE;AAGlB;AAEA,SAASY,uBAAuB/B,KAA4B;IAC1D,MAAMoC,aAAa;QAACpC,MAAM,KAAK;QAAEA,MAAM,GAAG;KAAC,CACxC,GAAG,CAAC,CAACqC,OAASA,MAAM,QACpB,MAAM,CAACC;IACV,OAAOF,UAAU,CAAC,EAAE;AACtB;AAEA,SAASG,kBAAkB/C,KAAc;IACvC,IAAI,CAACA,OACH,OAAO;IAET,IAAIM,qBAAqBN,QACvB,OAAO;IAET,MAAMgD,aAAahD,MAAM,IAAI,GAAG,WAAW;IAC3C,MAAMiD,UAAUD,WAAW,OAAO,CAAC,QAAQ;IAC3C,OACEA,AAAsB,MAAtBA,WAAW,MAAM,IACjB,yCAAyC,IAAI,CAACA,eAC9CA,AAAe,aAAfA,cACAA,AAAe,cAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,kBAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,kBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,eAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,yBAAfA,cACAA,AAAe,2BAAfA,cACAA,AAAe,6BAAfA,cACAA,AAAe,gBAAfA,cACAA,AAAe,2BAAfA,cACAA,AAAe,sBAAfA,cACAA,AAAe,0BAAfA,cACAA,AAAe,uBAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,kBAAfA,cACAA,WAAW,QAAQ,CAAC,8BACpBC,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,YACjBD,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,sBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,wBACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC,oBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,SAASE,wBAAwBlD,KAAc;IAC7C,IAAI,CAACA,OACH,OAAO;IAET,IAAIM,qBAAqBN,QACvB,OAAO;IAET,MAAMgD,aAAahD,MAAM,IAAI,GAAG,WAAW;IAC3C,MAAMiD,UAAUD,WAAW,OAAO,CAAC,QAAQ;IAC3C,OACEC,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,SACjBD,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,sBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,SAASG,uBAAuBnD,KAAa;IAC3C,OAAOA,MAAM,IAAI,GAAG,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,QAAQ;AAC1E;AAEA,SAASoD,mCACP5C,KAA4B,EAC5BU,kBAA2B;IAE3B,IAAIV,AAAe,YAAfA,MAAM,IAAI,IAAgB,CAACA,MAAM,KAAK,IAAI,CAACU,oBAC7C,OAAO;IAGT,MAAMmC,aAAaF,uBAAuB3C,MAAM,KAAK;IACrD,IAAI,CAAC6C,YACH,OAAO;IAET,MAAMC,cAAcH,uBAAuBjC;IAE3C,OACEoC,gBAAgBD,cAChBC,gBAAgB,GAAGD,WAAW,MAAM,CAAC,IACrCC,gBAAgB,GAAGD,WAAW,MAAM,CAAC,IACrCC,gBAAgB,GAAGD,WAAW,WAAW,CAAC,IAC1CC,gBAAgB,CAAC,MAAM,EAAED,YAAY,IACrCC,gBAAgB,CAAC,MAAM,EAAED,YAAY,IACrCC,YAAY,QAAQ,CAAC,CAAC,YAAY,EAAED,YAAY,KAChDC,YAAY,QAAQ,CAAC,CAAC,MAAM,EAAED,YAAY;AAE9C;AAEA,SAASE,qBACPC,iBAAyB,EACzBC,4BAAqC;IAErC,IACEA,gCACA,CAACV,kBAAkBU,+BAEnB,OAAO;IAET,MAAMT,aAAaQ,kBAAkB,WAAW;IAChD,OACER,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,eACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,eACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,eAAeU,kBACblD,KAA4B,EAC5BI,MAA0C,EAC1C+C,qBAA6B,EAC7BC,WAAyB,EACzBC,OAEC;IAED,IAAIC;IACJ,IAAK,IAAIC,UAAU,GAAGA,WAAWF,QAAQ,UAAU,EAAEE,WAAW,EAC9D,IAAI;QACF,MAAMC,kBAAkBvD,gCAAgCD;QACxD,MAAM8B,cAAcC,uBAAuB/B;QAC3C,MAAMkC,mBAAmB5B,oBAAoBF;QAC7C,MAAMqD,cAAqB;YACzB;gBACE,MAAM;gBACN,MAAM,CAAC;AACjB,EAAEC,KAAK,SAAS,CACd;oBACE,MAAM1D,MAAM,IAAI;oBAChB,YAAYA,MAAM,UAAU;oBAC5B,OAAOA,MAAM,KAAK;oBAClB,gBAAgBmC,uBAAuBnC;oBACvC,KAAKA,MAAM,GAAG;oBACd,OAAOA,MAAM,KAAK;oBAClB8B;oBACA,UAAU9B,MAAM,QAAQ;oBACxBI;oBACA8B;oBACA,UAAUD,kBAAkBjC,OAAOI;gBACrC,GACA,MACA,GACA;;4GAE0G,CAAC;YACrG;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAK+C;oBACL,QAAQ;gBACV;YACF;SACD;QACD,IAAIK,iBACFC,YAAY,IAAI,CACd;YACE,MAAM;YACN,MAAM;QACR,GACA;YACE,MAAM;YACN,WAAW;gBACT,KAAKD;gBACL,QAAQ;YACV;QACF;QAGJ,MAAMG,WACJ,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACJ;YACE;gBACE,MAAM;gBACN,SAASC,yCAAAA,mCAAmCA;YAC9C;YACA;gBACE,MAAM;gBACN,SAASJ;YACX;SACD,EACDL;QAGJ,MAAMU,UAAUH,SAAS,OAAO;QAChC,IAAIG,QAAQ,KAAK,EACf,MAAM,IAAIC,MAAMD,QAAQ,KAAK;QAE/B,IAAIvB,kBAAkBuB,QAAQ,kBAAkB,GAC9C,MAAM,IAAIC,MAAM;QAElB,IACEnB,mCAAmC5C,OAAO8D,QAAQ,kBAAkB,GAEpE,MAAM,IAAIC,MACR;QAGJ,MAAMrD,qBAAqBoD,QAAQ,kBAAkB,CAAE,IAAI;QAC3D,MAAMb,+BACJjD,AAAe,aAAfA,MAAM,IAAI,GACN8D,QAAQ,4BAA4B,EAAE,SACtCE;QACN,IACEhE,AAAe,aAAfA,MAAM,IAAI,IACV,CAAC+C,qBAAqB,IAAIE,+BAE1B,MAAM,IAAIc,MACR;QAGJ,MAAME,sBAAsBH,QAAQ,iBAAiB,EAAE;QACvD,IAAIG,uBAAuBvB,wBAAwBuB,sBACjD,MAAM,IAAIF,MAAM;QAElB,MAAMf,oBACJhD,AAAe,aAAfA,MAAM,IAAI,IAAiBiD,+BACvB,CAAC,yCAAyC,EAAEvC,mBAAmB,YAAY,EAAEV,MAAM,KAAK,IAAI,OAAO,SAAS,EAAEiD,6BAA6B,aAAa,CAAC,GACzJpC,8BAA8Bb,OAAOU,uBACrCuD,uBACAnD,6BAA6Bd,OAAOU;QAC1C,IAAIgC,wBAAwBM,oBAC1B,MAAM,IAAIe,MAAM;QAElB,MAAMG,gBACJlE,AAAe,aAAfA,MAAM,IAAI,IAAiBiD,+BACvB,CAAC,OAAO,EAAEvC,mBAAmB,QAAQ,EAAEuC,8BAA8B,GACrElC,0BAA0Bf,OAAOU,uBACjCoD,QAAQ,aAAa,EAAE,UACvB9C,yBAAyBhB,OAAOU;QAEtC,OAAO;YACLA;YACAsC;YACAkB;YACA,oBAAoBJ,QAAQ,UAAU,IAAI;QAC5C;IACF,EAAE,OAAOK,OAAO;QACdb,YAAYa;QACZ,IAAIZ,UAAUF,QAAQ,UAAU,EAC9B,MAAMnE,MAAMmE,QAAQ,YAAY;IAEpC;IAEF,MAAMC;AACR;AAEA,eAAec,wBACbpE,KAA4B,EAC5B4B,IAAU;IAEV,IAAI5B,MAAM,iBAAiB,EACzB,OAAOA,MAAM,iBAAiB;IAEhC,MAAMqE,aAAatE,2BAA2BC;IAC9C,IAAI,CAACqE,YACH;IAEF,OAAOC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,gBAAgBD;QAChB,MAAMrE,MAAM,QAAQ;QACpB,sBAAsB;YAAC;gBAAE4B;YAAK;SAAE;QAChC,iBAAiB;QACjB,mBAAmB;IACrB;AACF;AAEA,SAAS2C,oBACPvE,KAA4B,EAC5BmE,KAAa,EACbK,iBAA0B,EAC1BpE,MAA+B;IAE/B,MAAMM,qBACJV,MAAM,kBAAkB,IAAI,CAACuC,kBAAkBvC,MAAM,kBAAkB,IACnEA,MAAM,kBAAkB,GACxB6B,uBAAuB7B,OAAOI;IACpC,OAAO;QACL,GAAGJ,KAAK;QACRU;QACA,mBACEV,MAAM,iBAAiB,IACvBc,6BAA6Bd,OAAOU;QACtC,eACEV,MAAM,aAAa,IACnBgB,yBAAyBhB,OAAOU;QAClC,oBAAoB;QACpB,oBAAoB;QACpB,mBAAmB;QACnB,kBAAkByD;QAClB,mBAAmBK,qBAAqBxE,MAAM,iBAAiB;IACjE;AACF;AAEO,eAAeyE,wBACpBC,KAAmC,EACnCtB,WAAyB,EACzBC,UAA0C,CAAC,CAAC;IAE5C,MAAMrD,QAAQ0E,MAAM,KAAK;IACzB,MAAM9C,OAAOD,6BAA6B3B;IAC1C,MAAMqE,aAAatE,2BAA2BC;IAE9C,IAAI,CAAC4B,QAAQ,CAACyC,YAAY;QACxB,MAAMF,QAAQ,AAACvC,OAEX,sCADA;QAEJ,OAAO;YACL,cAAc;YACd,OAAO2C,oBAAoBvE,OAAOmE,OAAOH,QAAWU,MAAM,MAAM;QAClE;IACF;IAEA,IAAIF;IACJ,IAAI;QACFA,oBAAoB,MAAMJ,wBAAwBpE,OAAO4B;QACzD,MAAM+C,iBAAiB,MAAMzB,kBAC3BlD,OACA0E,MAAM,MAAM,EACZF,qBAAqBH,YACrBjB,aACA;YACE,YAAYC,QAAQ,UAAU,IAAItE;YAClC,cACEsE,QAAQ,YAAY,IAAIrE;QAC5B;QAEF,OAAO;YACL,cAAc;YACd,OAAO;gBACL,GAAGgB,KAAK;gBACR,GAAG2E,cAAc;gBACjB,oBAAoB;gBACpB,mBAAmB;gBACnB,kBAAkBX;gBAClB,mBAAmBQ,qBAAqBxE,MAAM,iBAAiB;YACjE;QACF;IACF,EAAE,OAAOmE,OAAO;QACd,MAAMS,UAAUT,iBAAiBJ,QAAQI,MAAM,OAAO,GAAGU,OAAOV;QAChE,OAAO;YACL,cAAc;YACd,OAAOS;YACP,OAAOL,oBACLvE,OACA4E,SACAJ,mBACAE,MAAM,MAAM;QAEhB;IACF;AACF;AAEO,eAAeI,yBACpBC,MAAsC,EACtC3B,WAAyB,EACzBC,UAA0C,CAAC,CAAC;IAE5C,MAAM2B,cAAcrF,KAAK,GAAG,CAC1B,GACA0D,QAAQ,WAAW,IAAIpE;IAEzB,MAAMgG,UAA2C,IAAIC,MAAMH,OAAO,MAAM;IACxE,IAAII,SAAS;IAEb,eAAeC;QACb,MAAOD,SAASJ,OAAO,MAAM,CAAE;YAC7B,MAAMM,QAAQF;YACdA,UAAU;YACVF,OAAO,CAACI,MAAM,GAAG,MAAMZ,wBACrBM,MAAM,CAACM,MAAM,EACbjC,aACAC;QAEJ;IACF;IAEA,MAAMjE,QAAQ,GAAG,CACf8F,MAAM,IAAI,CAAC;QAAE,QAAQvF,KAAK,GAAG,CAACqF,aAAaD,OAAO,MAAM;IAAE,GAAG,IAC3DK;IAGJ,OAAOH;AACT"}
+{"version":3,"file":"recorder-ui-describer.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/recorder-ui-describer.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { IModelConfig } from '@midscene/shared/env';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport type {\n  MidsceneRecorderEvent,\n  MidsceneRecorderPageInfo,\n  MidsceneRecorderSemanticAction,\n  MidsceneRecorderTarget,\n} from '@midscene/shared/recorder';\nimport {\n  buildMidsceneRecorderActionSummary,\n  buildMidsceneRecorderReplayInstruction,\n  getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\nimport { RECORDER_UI_DESCRIBER_SYSTEM_PROMPT } from './ai-model/prompt/recorder-ui-describer';\nimport { callAIWithObjectResponse } from './ai-model/service-caller';\nimport type { Rect } from './types';\n\nexport interface DescribeRecorderUIEventInput {\n  event: MidsceneRecorderEvent;\n  target?: MidsceneRecorderTarget;\n}\n\nexport interface DescribeRecorderUIEventOptions {\n  maxRetries?: number;\n  retryDelayMs?: number;\n  concurrency?: number;\n}\n\nexport interface DescribeRecorderUIEventResult {\n  event: MidsceneRecorderEvent;\n  usedFallback: boolean;\n  error?: string;\n}\n\ninterface RecorderUIEventAIResponse {\n  elementDescription?: string;\n  replayInstruction?: string;\n  actionSummary?: string;\n  scrollDestinationDescription?: string;\n  confidence?: 'high' | 'medium' | 'low';\n  error?: string;\n}\n\nconst RECORDER_UI_DESCRIBER_DEFAULT_RETRIES = 2;\nconst RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS = 200;\nconst RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY = 2;\n\nfunction delay(ms: number) {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nfunction clamp(value: number, min: number, max: number) {\n  return Math.min(Math.max(value, min), max);\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction isPendingDescription(value?: string) {\n  return value?.trim() === 'AI is analyzing element...';\n}\n\nfunction getRecorderEventScreenshot(event: MidsceneRecorderEvent) {\n  return (\n    event.screenshotWithBox || event.screenshotBefore || event.screenshotAfter\n  );\n}\n\nfunction getRecorderEventAfterScreenshot(event: MidsceneRecorderEvent) {\n  return event.screenshotAfter || event.screenshotWithBox;\n}\n\nfunction normalizeActionType(event: MidsceneRecorderEvent) {\n  return event.actionType?.trim();\n}\n\nfunction getPlatformId(target?: MidsceneRecorderTarget) {\n  return target?.platformId?.toLowerCase();\n}\n\nfunction getPlatformSurface(target?: MidsceneRecorderTarget) {\n  switch (getPlatformId(target)) {\n    case 'web':\n      return 'current web page';\n    case 'android':\n    case 'ios':\n    case 'harmony':\n      return 'current mobile screen';\n    case 'computer':\n      return 'current desktop screen';\n    default:\n      return 'current UI';\n  }\n}\n\nfunction getPlatformGuidance(target?: MidsceneRecorderTarget) {\n  switch (getPlatformId(target)) {\n    case 'web':\n      return 'For web targets, use web UI terms such as button, input, link, menu item, tab, dialog, aria-label, placeholder, and form section when visible or inferable.';\n    case 'android':\n    case 'ios':\n    case 'harmony':\n      return 'For mobile targets, use mobile UI terms such as tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, and screen section.';\n    case 'computer':\n      return 'For desktop/computer targets, use desktop UI terms such as menu item, toolbar button, dialog field, sidebar item, window control, file row, and application region.';\n    default:\n      return 'Use platform-neutral UI terms such as control, field, item, icon button, list item, region, panel, and page section.';\n  }\n}\n\nfunction getPointerActionVerb(event: MidsceneRecorderEvent) {\n  switch (normalizeActionType(event)) {\n    case 'Tap':\n      return 'Tap';\n    case 'DoubleClick':\n      return 'Double click';\n    case 'LongPress':\n      return 'Long press';\n    case 'RightClick':\n      return 'Right click';\n    default:\n      return 'Click';\n  }\n}\n\nfunction getDragActionVerb(event: MidsceneRecorderEvent) {\n  switch (normalizeActionType(event)) {\n    case 'Swipe':\n      return 'Swipe';\n    case 'DragAndDrop':\n      return 'Drag';\n    default:\n      return 'Drag';\n  }\n}\n\nfunction pointToRect(\n  x: number,\n  y: number,\n  size: number,\n  pageInfo: MidsceneRecorderPageInfo,\n): Rect {\n  const width = pageInfo.width || size;\n  const height = pageInfo.height || size;\n  const left = clamp(Math.floor(x - size / 2), 0, Math.max(width - 1, 0));\n  const top = clamp(Math.floor(y - size / 2), 0, Math.max(height - 1, 0));\n  return {\n    left,\n    top,\n    width: Math.min(size, Math.max(width - left, 1)),\n    height: Math.min(size, Math.max(height - top, 1)),\n  };\n}\n\nfunction getPointRectSize(event: MidsceneRecorderEvent) {\n  switch (event.type) {\n    case 'scroll':\n      return 96;\n    case 'drag':\n      return 64;\n    default:\n      return 36;\n  }\n}\n\nexport function getRecorderUIEventTargetRect(\n  event: MidsceneRecorderEvent,\n): Rect | null {\n  const rect = event.elementRect;\n  if (!rect) {\n    return null;\n  }\n\n  if (\n    isFiniteNumber(rect.width) &&\n    rect.width > 0 &&\n    isFiniteNumber(rect.height) &&\n    rect.height > 0 &&\n    (isFiniteNumber(rect.left) || isFiniteNumber(rect.top))\n  ) {\n    return {\n      left: rect.left || 0,\n      top: rect.top || 0,\n      width: rect.width,\n      height: rect.height,\n    };\n  }\n\n  if (isFiniteNumber(rect.x) && isFiniteNumber(rect.y)) {\n    return pointToRect(rect.x, rect.y, getPointRectSize(event), event.pageInfo);\n  }\n\n  return null;\n}\n\nfunction getFallbackDescription(\n  event: MidsceneRecorderEvent,\n  target?: MidsceneRecorderTarget,\n) {\n  const pageContext = getPageSemanticContext(event);\n  const surface = getPlatformSurface(target);\n\n  switch (event.type) {\n    case 'navigation':\n      return event.url || event.value || event.actionType || 'navigation';\n    case 'scroll':\n      return pageContext\n        ? `${pageContext} scrollable content`\n        : `scrollable content on the ${surface}`;\n    case 'drag':\n      return pageContext\n        ? `gesture area in ${pageContext}`\n        : `gesture area on the ${surface}`;\n    case 'input':\n      return `unresolved input field on the ${surface}`;\n    case 'keydown':\n      return pageContext\n        ? `focused control in ${pageContext}`\n        : `focused control on the ${surface}`;\n    default:\n      return pageContext\n        ? `control in ${pageContext}`\n        : `control on the ${surface}`;\n  }\n}\n\nfunction buildSemanticAction(\n  event: MidsceneRecorderEvent,\n  scrollDestinationDescription?: string,\n): MidsceneRecorderSemanticAction {\n  return {\n    type: event.type,\n    actionType: event.actionType,\n    value: event.value,\n    url: event.url,\n    scrollDestinationDescription,\n  };\n}\n\nfunction getFallbackReplayInstruction(\n  event: MidsceneRecorderEvent,\n  elementDescription: string,\n) {\n  return buildMidsceneRecorderReplayInstruction(\n    buildSemanticAction(event),\n    elementDescription,\n  );\n}\n\nfunction getFallbackActionSummary(\n  event: MidsceneRecorderEvent,\n  elementDescription: string,\n) {\n  return buildMidsceneRecorderActionSummary(\n    buildSemanticAction(event),\n    elementDescription,\n  );\n}\n\nfunction getActionGuidance(\n  event: MidsceneRecorderEvent,\n  target?: MidsceneRecorderTarget,\n) {\n  const platformGuidance = getPlatformGuidance(target);\n\n  switch (event.type) {\n    case 'click':\n      return `${platformGuidance} Identify the ${getPointerActionVerb(event).toLowerCase()} target by exact visible text first, then label/placeholder, then role plus stable surrounding context, then icon purpose, then visual position. Never describe it by coordinates, marker location, or as a nearby element.`;\n    case 'input':\n      return `${platformGuidance} Identify the exact input field at the marker before text entry. Use stable visible label, field role, field name, surrounding section, or sequence intent. Treat hint text that can change by user, time, data, or context as secondary evidence. Preserve the recorded input value only in replayInstruction; never describe the typed value or page title alone as the field.`;\n    case 'scroll':\n      return `${platformGuidance} Identify the scrollable page/region at the highlighted scroll point and concrete destination content revealed after scrolling. If multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu; do not generalize a panel/list scroll into the whole page. Use newly visible headings, section titles, list/table names, list items, or stable region labels; never say only \"more content\" or \"current page\".`;\n    case 'drag':\n      return `${platformGuidance} Identify the ${getDragActionVerb(event).toLowerCase()} start/end regions or the dragged UI control. Do not describe only the gesture path or coordinates.`;\n    case 'keydown':\n      return `${platformGuidance} Identify the focused element or keyboard target if visible, and preserve the recorded key in the replay instruction.`;\n    default:\n      return `${platformGuidance} Identify the UI target involved in this event using the most stable visible text or surrounding context.`;\n  }\n}\n\nfunction getEventRawCoordinates(event: MidsceneRecorderEvent) {\n  const x = event.elementRect?.x;\n  const y = event.elementRect?.y;\n  if (isFiniteNumber(x) && isFiniteNumber(y)) {\n    return { x, y };\n  }\n  return undefined;\n}\n\nfunction getPageSemanticContext(event: MidsceneRecorderEvent) {\n  const candidates = [event.title, event.url]\n    .map((item) => item?.trim())\n    .filter(Boolean) as string[];\n  return candidates[0];\n}\n\nfunction isWeakDescription(value?: string) {\n  if (!value) {\n    return true;\n  }\n  if (isPendingDescription(value)) {\n    return true;\n  }\n  const normalized = value.trim().toLowerCase();\n  const compact = normalized.replace(/\\s+/g, '');\n  return (\n    normalized.length === 0 ||\n    /^\\(?\\d+(?:\\.\\d+)?,\\s*\\d+(?:\\.\\d+)?\\)?$/.test(normalized) ||\n    normalized === 'target' ||\n    normalized === 'element' ||\n    normalized === 'target element' ||\n    normalized === 'the element' ||\n    normalized === 'page element' ||\n    normalized === 'input field' ||\n    normalized === 'text input' ||\n    normalized === 'text field' ||\n    normalized === 'search box' ||\n    normalized === 'more content' ||\n    normalized === 'the page' ||\n    normalized === 'current page' ||\n    normalized === 'current screen' ||\n    normalized === 'the screen' ||\n    normalized === 'current ui' ||\n    normalized === 'current visible ui' ||\n    normalized === 'current visible page' ||\n    normalized === 'current visible screen' ||\n    normalized === 'main area' ||\n    normalized === 'main scrollable area' ||\n    normalized === 'scrollable area' ||\n    normalized === 'highlighted element' ||\n    normalized === 'highlighted item' ||\n    normalized === 'marked element' ||\n    normalized === 'marked item' ||\n    normalized.includes('ai is analyzing element') ||\n    compact.includes('坐标') ||\n    compact.includes('附近') ||\n    compact.includes('附近的元素') ||\n    normalized.includes('coordinate') ||\n    normalized.includes('near the coordinate') ||\n    normalized.includes('near coordinates') ||\n    normalized.includes('nearby element') ||\n    normalized.includes('nearby item') ||\n    normalized.includes('near the marker') ||\n    normalized.includes('near marker') ||\n    normalized.includes('near the point') ||\n    normalized.includes('near point') ||\n    normalized.includes('at the point') ||\n    normalized.includes('button near point') ||\n    normalized.includes('shown in the screenshot') ||\n    normalized.includes('red rectangle') ||\n    normalized.includes('red marker') ||\n    normalized.includes('red box') ||\n    normalized.includes('highlighted element') ||\n    normalized.includes('highlighted item') ||\n    normalized.includes('highlighted screenshot')\n  );\n}\n\nfunction isWeakReplayInstruction(value?: string) {\n  if (!value) {\n    return true;\n  }\n  if (isPendingDescription(value)) {\n    return true;\n  }\n  const normalized = value.trim().toLowerCase();\n  const compact = normalized.replace(/\\s+/g, '');\n  return (\n    compact.includes('坐标') ||\n    compact.includes('附近') ||\n    normalized.includes('coordinate') ||\n    normalized.includes('near the coordinate') ||\n    normalized.includes('nearby element') ||\n    normalized.includes('nearby item') ||\n    normalized.includes('near the marker') ||\n    normalized.includes('near marker') ||\n    normalized.includes('near the point') ||\n    normalized.includes('near point') ||\n    normalized.includes('at the point') ||\n    normalized.includes('ai is analyzing element') ||\n    normalized.includes('more content') ||\n    normalized.includes('current page') ||\n    normalized.includes('current screen') ||\n    normalized.includes('highlighted element') ||\n    normalized.includes('highlighted item') ||\n    normalized.includes('red marker') ||\n    normalized.includes('red box') ||\n    normalized.includes('shown in the screenshot') ||\n    normalized.includes('highlighted screenshot')\n  );\n}\n\nfunction normalizeForComparison(value: string) {\n  return value.trim().toLowerCase().replace(/[\"'`]/g, '').replace(/\\s+/g, ' ');\n}\n\nfunction isInputValueUsedAsFieldDescription(\n  event: MidsceneRecorderEvent,\n  elementDescription?: string,\n) {\n  if (event.type !== 'input' || !event.value || !elementDescription) {\n    return false;\n  }\n\n  const typedValue = normalizeForComparison(event.value);\n  if (!typedValue) {\n    return false;\n  }\n  const description = normalizeForComparison(elementDescription);\n\n  return (\n    description === typedValue ||\n    description === `${typedValue} input` ||\n    description === `${typedValue} field` ||\n    description === `${typedValue} text field` ||\n    description === `input ${typedValue}` ||\n    description === `field ${typedValue}` ||\n    description.includes(`typed value ${typedValue}`) ||\n    description.includes(`value ${typedValue}`)\n  );\n}\n\nfunction hasScrollDestination(\n  replayInstruction: string,\n  scrollDestinationDescription?: string,\n) {\n  if (\n    scrollDestinationDescription &&\n    !isWeakDescription(scrollDestinationDescription)\n  ) {\n    return true;\n  }\n  const normalized = replayInstruction.toLowerCase();\n  return (\n    normalized.includes(' until ') ||\n    normalized.includes(' visible') ||\n    normalized.includes(' reveal') ||\n    normalized.includes(' to the ') ||\n    normalized.includes(' toward ')\n  );\n}\n\nasync function describeWithRetry(\n  event: MidsceneRecorderEvent,\n  target: MidsceneRecorderTarget | undefined,\n  highlightedScreenshot: string,\n  modelConfig: IModelConfig,\n  options: Required<\n    Pick<DescribeRecorderUIEventOptions, 'maxRetries' | 'retryDelayMs'>\n  >,\n) {\n  let lastError: unknown;\n  for (let attempt = 1; attempt <= options.maxRetries; attempt += 1) {\n    try {\n      const afterScreenshot = getRecorderEventAfterScreenshot(event);\n      const pageContext = getPageSemanticContext(event);\n      const platformGuidance = getPlatformGuidance(target);\n      const userContent: any[] = [\n        {\n          type: 'text',\n          text: `Recorder event:\n${JSON.stringify(\n  {\n    type: event.type,\n    actionType: event.actionType,\n    value: event.value,\n    rawCoordinates: getEventRawCoordinates(event),\n    url: event.url,\n    title: event.title,\n    pageContext,\n    pageInfo: event.pageInfo,\n    target,\n    platformGuidance,\n    guidance: getActionGuidance(event, target),\n  },\n  null,\n  2,\n)}\n\nThe target or region is highlighted in the screenshot below. Convert this event into semantic replay fields.`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: highlightedScreenshot,\n            detail: 'high',\n          },\n        },\n      ];\n      if (afterScreenshot) {\n        userContent.push(\n          {\n            type: 'text',\n            text: 'Screenshot after the recorded action, for context only:',\n          },\n          {\n            type: 'image_url',\n            image_url: {\n              url: afterScreenshot,\n              detail: 'high',\n            },\n          },\n        );\n      }\n      const response =\n        await callAIWithObjectResponse<RecorderUIEventAIResponse>(\n          [\n            {\n              role: 'system',\n              content: RECORDER_UI_DESCRIBER_SYSTEM_PROMPT,\n            },\n            {\n              role: 'user',\n              content: userContent,\n            },\n          ],\n          modelConfig,\n        );\n\n      const content = response.content;\n      if (content.error) {\n        throw new Error(content.error);\n      }\n      if (isWeakDescription(content.elementDescription)) {\n        throw new Error('AI returned a weak recorder event description.');\n      }\n      if (\n        isInputValueUsedAsFieldDescription(event, content.elementDescription)\n      ) {\n        throw new Error(\n          'AI used the recorded input value as the field description.',\n        );\n      }\n      const elementDescription = content.elementDescription!.trim();\n      const scrollDestinationDescription =\n        event.type === 'scroll'\n          ? content.scrollDestinationDescription?.trim()\n          : undefined;\n      if (\n        event.type === 'scroll' &&\n        !hasScrollDestination('', scrollDestinationDescription)\n      ) {\n        throw new Error(\n          'AI returned a scroll description without a destination.',\n        );\n      }\n      const aiReplayInstruction = content.replayInstruction?.trim();\n      if (aiReplayInstruction && isWeakReplayInstruction(aiReplayInstruction)) {\n        throw new Error('AI returned a weak recorder replay instruction.');\n      }\n      const semanticAction = buildSemanticAction(\n        event,\n        scrollDestinationDescription,\n      );\n      const replayInstruction = buildMidsceneRecorderReplayInstruction(\n        semanticAction,\n        elementDescription,\n      );\n      if (isWeakReplayInstruction(replayInstruction)) {\n        throw new Error('AI returned a weak recorder replay instruction.');\n      }\n      const actionSummary = buildMidsceneRecorderActionSummary(\n        semanticAction,\n        elementDescription,\n      );\n\n      return {\n        source: 'recorderAI' as const,\n        status: 'ready' as const,\n        elementDescription,\n        replayInstruction,\n        actionSummary,\n        confidence: content.confidence || 'medium',\n      };\n    } catch (error) {\n      lastError = error;\n      if (attempt < options.maxRetries) {\n        await delay(options.retryDelayMs);\n      }\n    }\n  }\n  throw lastError;\n}\n\nasync function createScreenshotWithBox(\n  event: MidsceneRecorderEvent,\n  rect: Rect,\n) {\n  if (event.screenshotWithBox) {\n    return event.screenshotWithBox;\n  }\n  const screenshot = getRecorderEventScreenshot(event);\n  if (!screenshot) {\n    return undefined;\n  }\n  return compositeElementInfoImg({\n    inputImgBase64: screenshot,\n    size: event.pageInfo,\n    elementsPositionInfo: [{ rect }],\n    borderThickness: 3,\n    annotationPadding: 2,\n  });\n}\n\nfunction createFallbackEvent(\n  event: MidsceneRecorderEvent,\n  error: string,\n  screenshotWithBox?: string,\n  target?: MidsceneRecorderTarget,\n): MidsceneRecorderEvent {\n  const semantic = getMidsceneRecorderSemantic(event);\n  const elementDescription =\n    semantic?.elementDescription &&\n    !isWeakDescription(semantic.elementDescription)\n      ? semantic.elementDescription\n      : getFallbackDescription(event, target);\n  return {\n    ...event,\n    semantic: {\n      source: 'heuristic',\n      status: 'ready',\n      elementDescription,\n      replayInstruction: getFallbackReplayInstruction(\n        event,\n        elementDescription,\n      ),\n      actionSummary: getFallbackActionSummary(event, elementDescription),\n      confidence: 'low',\n      error,\n    },\n    screenshotWithBox: screenshotWithBox || event.screenshotWithBox,\n  };\n}\n\nexport async function describeRecorderUIEvent(\n  input: DescribeRecorderUIEventInput,\n  modelConfig: IModelConfig,\n  options: DescribeRecorderUIEventOptions = {},\n): Promise<DescribeRecorderUIEventResult> {\n  const event = input.event;\n  const rect = getRecorderUIEventTargetRect(event);\n  const screenshot = getRecorderEventScreenshot(event);\n\n  if (!rect || !screenshot) {\n    const error = !rect\n      ? 'Recorder event has no target rectangle.'\n      : 'Recorder event has no screenshot.';\n    return {\n      usedFallback: true,\n      event: createFallbackEvent(event, error, undefined, input.target),\n    };\n  }\n\n  let screenshotWithBox: string | undefined;\n  try {\n    screenshotWithBox = await createScreenshotWithBox(event, rect);\n    const semanticFields = await describeWithRetry(\n      event,\n      input.target,\n      screenshotWithBox || screenshot,\n      modelConfig,\n      {\n        maxRetries: options.maxRetries ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRIES,\n        retryDelayMs:\n          options.retryDelayMs ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS,\n      },\n    );\n    return {\n      usedFallback: false,\n      event: {\n        ...event,\n        semantic: semanticFields,\n        screenshotWithBox: screenshotWithBox || event.screenshotWithBox,\n      },\n    };\n  } catch (error) {\n    const message = error instanceof Error ? error.message : String(error);\n    return {\n      usedFallback: true,\n      error: message,\n      event: createFallbackEvent(\n        event,\n        message,\n        screenshotWithBox,\n        input.target,\n      ),\n    };\n  }\n}\n\nexport async function describeRecorderUIEvents(\n  inputs: DescribeRecorderUIEventInput[],\n  modelConfig: IModelConfig,\n  options: DescribeRecorderUIEventOptions = {},\n): Promise<DescribeRecorderUIEventResult[]> {\n  const concurrency = Math.max(\n    1,\n    options.concurrency ?? RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY,\n  );\n  const results: DescribeRecorderUIEventResult[] = new Array(inputs.length);\n  let cursor = 0;\n\n  async function worker() {\n    while (cursor < inputs.length) {\n      const index = cursor;\n      cursor += 1;\n      results[index] = await describeRecorderUIEvent(\n        inputs[index],\n        modelConfig,\n        options,\n      );\n    }\n  }\n\n  await Promise.all(\n    Array.from({ length: Math.min(concurrency, inputs.length) }, () =>\n      worker(),\n    ),\n  );\n  return results;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","RECORDER_UI_DESCRIBER_DEFAULT_RETRIES","RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS","RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY","delay","ms","Promise","resolve","setTimeout","clamp","value","min","max","Math","isFiniteNumber","Number","isPendingDescription","getRecorderEventScreenshot","event","getRecorderEventAfterScreenshot","normalizeActionType","getPlatformId","target","getPlatformSurface","getPlatformGuidance","getPointerActionVerb","getDragActionVerb","pointToRect","x","y","size","pageInfo","width","height","left","top","getPointRectSize","getRecorderUIEventTargetRect","rect","getFallbackDescription","pageContext","getPageSemanticContext","surface","buildSemanticAction","scrollDestinationDescription","getFallbackReplayInstruction","elementDescription","buildMidsceneRecorderReplayInstruction","getFallbackActionSummary","buildMidsceneRecorderActionSummary","getActionGuidance","platformGuidance","getEventRawCoordinates","candidates","item","Boolean","isWeakDescription","normalized","compact","isWeakReplayInstruction","normalizeForComparison","isInputValueUsedAsFieldDescription","typedValue","description","hasScrollDestination","replayInstruction","describeWithRetry","highlightedScreenshot","modelConfig","options","lastError","attempt","afterScreenshot","userContent","JSON","response","callAIWithObjectResponse","RECORDER_UI_DESCRIBER_SYSTEM_PROMPT","content","Error","undefined","aiReplayInstruction","semanticAction","actionSummary","error","createScreenshotWithBox","screenshot","compositeElementInfoImg","createFallbackEvent","screenshotWithBox","semantic","getMidsceneRecorderSemantic","describeRecorderUIEvent","input","semanticFields","message","String","describeRecorderUIEvents","inputs","concurrency","results","Array","cursor","worker","index"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACqCA,MAAMI,wCAAwC;AAC9C,MAAMC,+CAA+C;AACrD,MAAMC,4CAA4C;AAElD,SAASC,MAAMC,EAAU;IACvB,OAAO,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;AACtD;AAEA,SAASI,MAAMC,KAAa,EAAEC,GAAW,EAAEC,GAAW;IACpD,OAAOC,KAAK,GAAG,CAACA,KAAK,GAAG,CAACH,OAAOC,MAAMC;AACxC;AAEA,SAASE,eAAeJ,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBK,OAAO,QAAQ,CAACL;AACtD;AAEA,SAASM,qBAAqBN,KAAc;IAC1C,OAAOA,OAAO,WAAW;AAC3B;AAEA,SAASO,2BAA2BC,KAA4B;IAC9D,OACEA,MAAM,iBAAiB,IAAIA,MAAM,gBAAgB,IAAIA,MAAM,eAAe;AAE9E;AAEA,SAASC,gCAAgCD,KAA4B;IACnE,OAAOA,MAAM,eAAe,IAAIA,MAAM,iBAAiB;AACzD;AAEA,SAASE,oBAAoBF,KAA4B;IACvD,OAAOA,MAAM,UAAU,EAAE;AAC3B;AAEA,SAASG,cAAcC,MAA+B;IACpD,OAAOA,QAAQ,YAAY;AAC7B;AAEA,SAASC,mBAAmBD,MAA+B;IACzD,OAAQD,cAAcC;QACpB,KAAK;YACH,OAAO;QACT,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASE,oBAAoBF,MAA+B;IAC1D,OAAQD,cAAcC;QACpB,KAAK;YACH,OAAO;QACT,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASG,qBAAqBP,KAA4B;IACxD,OAAQE,oBAAoBF;QAC1B,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASQ,kBAAkBR,KAA4B;IACrD,OAAQE,oBAAoBF;QAC1B,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASS,YACPC,CAAS,EACTC,CAAS,EACTC,IAAY,EACZC,QAAkC;IAElC,MAAMC,QAAQD,SAAS,KAAK,IAAID;IAChC,MAAMG,SAASF,SAAS,MAAM,IAAID;IAClC,MAAMI,OAAOzB,MAAMI,KAAK,KAAK,CAACe,IAAIE,OAAO,IAAI,GAAGjB,KAAK,GAAG,CAACmB,QAAQ,GAAG;IACpE,MAAMG,MAAM1B,MAAMI,KAAK,KAAK,CAACgB,IAAIC,OAAO,IAAI,GAAGjB,KAAK,GAAG,CAACoB,SAAS,GAAG;IACpE,OAAO;QACLC;QACAC;QACA,OAAOtB,KAAK,GAAG,CAACiB,MAAMjB,KAAK,GAAG,CAACmB,QAAQE,MAAM;QAC7C,QAAQrB,KAAK,GAAG,CAACiB,MAAMjB,KAAK,GAAG,CAACoB,SAASE,KAAK;IAChD;AACF;AAEA,SAASC,iBAAiBlB,KAA4B;IACpD,OAAQA,MAAM,IAAI;QAChB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEO,SAASmB,6BACdnB,KAA4B;IAE5B,MAAMoB,OAAOpB,MAAM,WAAW;IAC9B,IAAI,CAACoB,MACH,OAAO;IAGT,IACExB,eAAewB,KAAK,KAAK,KACzBA,KAAK,KAAK,GAAG,KACbxB,eAAewB,KAAK,MAAM,KAC1BA,KAAK,MAAM,GAAG,KACbxB,CAAAA,eAAewB,KAAK,IAAI,KAAKxB,eAAewB,KAAK,GAAG,IAErD,OAAO;QACL,MAAMA,KAAK,IAAI,IAAI;QACnB,KAAKA,KAAK,GAAG,IAAI;QACjB,OAAOA,KAAK,KAAK;QACjB,QAAQA,KAAK,MAAM;IACrB;IAGF,IAAIxB,eAAewB,KAAK,CAAC,KAAKxB,eAAewB,KAAK,CAAC,GACjD,OAAOX,YAAYW,KAAK,CAAC,EAAEA,KAAK,CAAC,EAAEF,iBAAiBlB,QAAQA,MAAM,QAAQ;IAG5E,OAAO;AACT;AAEA,SAASqB,uBACPrB,KAA4B,EAC5BI,MAA+B;IAE/B,MAAMkB,cAAcC,uBAAuBvB;IAC3C,MAAMwB,UAAUnB,mBAAmBD;IAEnC,OAAQJ,MAAM,IAAI;QAChB,KAAK;YACH,OAAOA,MAAM,GAAG,IAAIA,MAAM,KAAK,IAAIA,MAAM,UAAU,IAAI;QACzD,KAAK;YACH,OAAOsB,cACH,GAAGA,YAAY,mBAAmB,CAAC,GACnC,CAAC,0BAA0B,EAAEE,SAAS;QAC5C,KAAK;YACH,OAAOF,cACH,CAAC,gBAAgB,EAAEA,aAAa,GAChC,CAAC,oBAAoB,EAAEE,SAAS;QACtC,KAAK;YACH,OAAO,CAAC,8BAA8B,EAAEA,SAAS;QACnD,KAAK;YACH,OAAOF,cACH,CAAC,mBAAmB,EAAEA,aAAa,GACnC,CAAC,uBAAuB,EAAEE,SAAS;QACzC;YACE,OAAOF,cACH,CAAC,WAAW,EAAEA,aAAa,GAC3B,CAAC,eAAe,EAAEE,SAAS;IACnC;AACF;AAEA,SAASC,oBACPzB,KAA4B,EAC5B0B,4BAAqC;IAErC,OAAO;QACL,MAAM1B,MAAM,IAAI;QAChB,YAAYA,MAAM,UAAU;QAC5B,OAAOA,MAAM,KAAK;QAClB,KAAKA,MAAM,GAAG;QACd0B;IACF;AACF;AAEA,SAASC,6BACP3B,KAA4B,EAC5B4B,kBAA0B;IAE1B,OAAOC,AAAAA,IAAAA,yBAAAA,sCAAAA,AAAAA,EACLJ,oBAAoBzB,QACpB4B;AAEJ;AAEA,SAASE,yBACP9B,KAA4B,EAC5B4B,kBAA0B;IAE1B,OAAOG,AAAAA,IAAAA,yBAAAA,kCAAAA,AAAAA,EACLN,oBAAoBzB,QACpB4B;AAEJ;AAEA,SAASI,kBACPhC,KAA4B,EAC5BI,MAA+B;IAE/B,MAAM6B,mBAAmB3B,oBAAoBF;IAE7C,OAAQJ,MAAM,IAAI;QAChB,KAAK;YACH,OAAO,GAAGiC,iBAAiB,cAAc,EAAE1B,qBAAqBP,OAAO,WAAW,GAAG,2NAA2N,CAAC;QACnT,KAAK;YACH,OAAO,GAAGiC,iBAAiB,gXAAgX,CAAC;QAC9Y,KAAK;YACH,OAAO,GAAGA,iBAAiB,yhBAAyhB,CAAC;QACvjB,KAAK;YACH,OAAO,GAAGA,iBAAiB,cAAc,EAAEzB,kBAAkBR,OAAO,WAAW,GAAG,mGAAmG,CAAC;QACxL,KAAK;YACH,OAAO,GAAGiC,iBAAiB,qHAAqH,CAAC;QACnJ;YACE,OAAO,GAAGA,iBAAiB,yGAAyG,CAAC;IACzI;AACF;AAEA,SAASC,uBAAuBlC,KAA4B;IAC1D,MAAMU,IAAIV,MAAM,WAAW,EAAE;IAC7B,MAAMW,IAAIX,MAAM,WAAW,EAAE;IAC7B,IAAIJ,eAAec,MAAMd,eAAee,IACtC,OAAO;QAAED;QAAGC;IAAE;AAGlB;AAEA,SAASY,uBAAuBvB,KAA4B;IAC1D,MAAMmC,aAAa;QAACnC,MAAM,KAAK;QAAEA,MAAM,GAAG;KAAC,CACxC,GAAG,CAAC,CAACoC,OAASA,MAAM,QACpB,MAAM,CAACC;IACV,OAAOF,UAAU,CAAC,EAAE;AACtB;AAEA,SAASG,kBAAkB9C,KAAc;IACvC,IAAI,CAACA,OACH,OAAO;IAET,IAAIM,qBAAqBN,QACvB,OAAO;IAET,MAAM+C,aAAa/C,MAAM,IAAI,GAAG,WAAW;IAC3C,MAAMgD,UAAUD,WAAW,OAAO,CAAC,QAAQ;IAC3C,OACEA,AAAsB,MAAtBA,WAAW,MAAM,IACjB,yCAAyC,IAAI,CAACA,eAC9CA,AAAe,aAAfA,cACAA,AAAe,cAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,kBAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,kBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,eAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,yBAAfA,cACAA,AAAe,2BAAfA,cACAA,AAAe,6BAAfA,cACAA,AAAe,gBAAfA,cACAA,AAAe,2BAAfA,cACAA,AAAe,sBAAfA,cACAA,AAAe,0BAAfA,cACAA,AAAe,uBAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,kBAAfA,cACAA,WAAW,QAAQ,CAAC,8BACpBC,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,YACjBD,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,sBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,wBACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC,oBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,SAASE,wBAAwBjD,KAAc;IAC7C,IAAI,CAACA,OACH,OAAO;IAET,IAAIM,qBAAqBN,QACvB,OAAO;IAET,MAAM+C,aAAa/C,MAAM,IAAI,GAAG,WAAW;IAC3C,MAAMgD,UAAUD,WAAW,OAAO,CAAC,QAAQ;IAC3C,OACEC,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,SACjBD,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,sBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,SAASG,uBAAuBlD,KAAa;IAC3C,OAAOA,MAAM,IAAI,GAAG,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,QAAQ;AAC1E;AAEA,SAASmD,mCACP3C,KAA4B,EAC5B4B,kBAA2B;IAE3B,IAAI5B,AAAe,YAAfA,MAAM,IAAI,IAAgB,CAACA,MAAM,KAAK,IAAI,CAAC4B,oBAC7C,OAAO;IAGT,MAAMgB,aAAaF,uBAAuB1C,MAAM,KAAK;IACrD,IAAI,CAAC4C,YACH,OAAO;IAET,MAAMC,cAAcH,uBAAuBd;IAE3C,OACEiB,gBAAgBD,cAChBC,gBAAgB,GAAGD,WAAW,MAAM,CAAC,IACrCC,gBAAgB,GAAGD,WAAW,MAAM,CAAC,IACrCC,gBAAgB,GAAGD,WAAW,WAAW,CAAC,IAC1CC,gBAAgB,CAAC,MAAM,EAAED,YAAY,IACrCC,gBAAgB,CAAC,MAAM,EAAED,YAAY,IACrCC,YAAY,QAAQ,CAAC,CAAC,YAAY,EAAED,YAAY,KAChDC,YAAY,QAAQ,CAAC,CAAC,MAAM,EAAED,YAAY;AAE9C;AAEA,SAASE,qBACPC,iBAAyB,EACzBrB,4BAAqC;IAErC,IACEA,gCACA,CAACY,kBAAkBZ,+BAEnB,OAAO;IAET,MAAMa,aAAaQ,kBAAkB,WAAW;IAChD,OACER,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,eACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,eACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,eAAeS,kBACbhD,KAA4B,EAC5BI,MAA0C,EAC1C6C,qBAA6B,EAC7BC,WAAyB,EACzBC,OAEC;IAED,IAAIC;IACJ,IAAK,IAAIC,UAAU,GAAGA,WAAWF,QAAQ,UAAU,EAAEE,WAAW,EAC9D,IAAI;QACF,MAAMC,kBAAkBrD,gCAAgCD;QACxD,MAAMsB,cAAcC,uBAAuBvB;QAC3C,MAAMiC,mBAAmB3B,oBAAoBF;QAC7C,MAAMmD,cAAqB;YACzB;gBACE,MAAM;gBACN,MAAM,CAAC;AACjB,EAAEC,KAAK,SAAS,CACd;oBACE,MAAMxD,MAAM,IAAI;oBAChB,YAAYA,MAAM,UAAU;oBAC5B,OAAOA,MAAM,KAAK;oBAClB,gBAAgBkC,uBAAuBlC;oBACvC,KAAKA,MAAM,GAAG;oBACd,OAAOA,MAAM,KAAK;oBAClBsB;oBACA,UAAUtB,MAAM,QAAQ;oBACxBI;oBACA6B;oBACA,UAAUD,kBAAkBhC,OAAOI;gBACrC,GACA,MACA,GACA;;4GAE0G,CAAC;YACrG;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAK6C;oBACL,QAAQ;gBACV;YACF;SACD;QACD,IAAIK,iBACFC,YAAY,IAAI,CACd;YACE,MAAM;YACN,MAAM;QACR,GACA;YACE,MAAM;YACN,WAAW;gBACT,KAAKD;gBACL,QAAQ;YACV;QACF;QAGJ,MAAMG,WACJ,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACJ;YACE;gBACE,MAAM;gBACN,SAASC,yCAAAA,mCAAmCA;YAC9C;YACA;gBACE,MAAM;gBACN,SAASJ;YACX;SACD,EACDL;QAGJ,MAAMU,UAAUH,SAAS,OAAO;QAChC,IAAIG,QAAQ,KAAK,EACf,MAAM,IAAIC,MAAMD,QAAQ,KAAK;QAE/B,IAAItB,kBAAkBsB,QAAQ,kBAAkB,GAC9C,MAAM,IAAIC,MAAM;QAElB,IACElB,mCAAmC3C,OAAO4D,QAAQ,kBAAkB,GAEpE,MAAM,IAAIC,MACR;QAGJ,MAAMjC,qBAAqBgC,QAAQ,kBAAkB,CAAE,IAAI;QAC3D,MAAMlC,+BACJ1B,AAAe,aAAfA,MAAM,IAAI,GACN4D,QAAQ,4BAA4B,EAAE,SACtCE;QACN,IACE9D,AAAe,aAAfA,MAAM,IAAI,IACV,CAAC8C,qBAAqB,IAAIpB,+BAE1B,MAAM,IAAImC,MACR;QAGJ,MAAME,sBAAsBH,QAAQ,iBAAiB,EAAE;QACvD,IAAIG,uBAAuBtB,wBAAwBsB,sBACjD,MAAM,IAAIF,MAAM;QAElB,MAAMG,iBAAiBvC,oBACrBzB,OACA0B;QAEF,MAAMqB,oBAAoBlB,AAAAA,IAAAA,yBAAAA,sCAAAA,AAAAA,EACxBmC,gBACApC;QAEF,IAAIa,wBAAwBM,oBAC1B,MAAM,IAAIc,MAAM;QAElB,MAAMI,gBAAgBlC,AAAAA,IAAAA,yBAAAA,kCAAAA,AAAAA,EACpBiC,gBACApC;QAGF,OAAO;YACL,QAAQ;YACR,QAAQ;YACRA;YACAmB;YACAkB;YACA,YAAYL,QAAQ,UAAU,IAAI;QACpC;IACF,EAAE,OAAOM,OAAO;QACdd,YAAYc;QACZ,IAAIb,UAAUF,QAAQ,UAAU,EAC9B,MAAMjE,MAAMiE,QAAQ,YAAY;IAEpC;IAEF,MAAMC;AACR;AAEA,eAAee,wBACbnE,KAA4B,EAC5BoB,IAAU;IAEV,IAAIpB,MAAM,iBAAiB,EACzB,OAAOA,MAAM,iBAAiB;IAEhC,MAAMoE,aAAarE,2BAA2BC;IAC9C,IAAI,CAACoE,YACH;IAEF,OAAOC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,gBAAgBD;QAChB,MAAMpE,MAAM,QAAQ;QACpB,sBAAsB;YAAC;gBAAEoB;YAAK;SAAE;QAChC,iBAAiB;QACjB,mBAAmB;IACrB;AACF;AAEA,SAASkD,oBACPtE,KAA4B,EAC5BkE,KAAa,EACbK,iBAA0B,EAC1BnE,MAA+B;IAE/B,MAAMoE,WAAWC,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BzE;IAC7C,MAAM4B,qBACJ4C,UAAU,sBACV,CAAClC,kBAAkBkC,SAAS,kBAAkB,IAC1CA,SAAS,kBAAkB,GAC3BnD,uBAAuBrB,OAAOI;IACpC,OAAO;QACL,GAAGJ,KAAK;QACR,UAAU;YACR,QAAQ;YACR,QAAQ;YACR4B;YACA,mBAAmBD,6BACjB3B,OACA4B;YAEF,eAAeE,yBAAyB9B,OAAO4B;YAC/C,YAAY;YACZsC;QACF;QACA,mBAAmBK,qBAAqBvE,MAAM,iBAAiB;IACjE;AACF;AAEO,eAAe0E,wBACpBC,KAAmC,EACnCzB,WAAyB,EACzBC,UAA0C,CAAC,CAAC;IAE5C,MAAMnD,QAAQ2E,MAAM,KAAK;IACzB,MAAMvD,OAAOD,6BAA6BnB;IAC1C,MAAMoE,aAAarE,2BAA2BC;IAE9C,IAAI,CAACoB,QAAQ,CAACgD,YAAY;QACxB,MAAMF,QAAQ,AAAC9C,OAEX,sCADA;QAEJ,OAAO;YACL,cAAc;YACd,OAAOkD,oBAAoBtE,OAAOkE,OAAOJ,QAAWa,MAAM,MAAM;QAClE;IACF;IAEA,IAAIJ;IACJ,IAAI;QACFA,oBAAoB,MAAMJ,wBAAwBnE,OAAOoB;QACzD,MAAMwD,iBAAiB,MAAM5B,kBAC3BhD,OACA2E,MAAM,MAAM,EACZJ,qBAAqBH,YACrBlB,aACA;YACE,YAAYC,QAAQ,UAAU,IAAIpE;YAClC,cACEoE,QAAQ,YAAY,IAAInE;QAC5B;QAEF,OAAO;YACL,cAAc;YACd,OAAO;gBACL,GAAGgB,KAAK;gBACR,UAAU4E;gBACV,mBAAmBL,qBAAqBvE,MAAM,iBAAiB;YACjE;QACF;IACF,EAAE,OAAOkE,OAAO;QACd,MAAMW,UAAUX,iBAAiBL,QAAQK,MAAM,OAAO,GAAGY,OAAOZ;QAChE,OAAO;YACL,cAAc;YACd,OAAOW;YACP,OAAOP,oBACLtE,OACA6E,SACAN,mBACAI,MAAM,MAAM;QAEhB;IACF;AACF;AAEO,eAAeI,yBACpBC,MAAsC,EACtC9B,WAAyB,EACzBC,UAA0C,CAAC,CAAC;IAE5C,MAAM8B,cAActF,KAAK,GAAG,CAC1B,GACAwD,QAAQ,WAAW,IAAIlE;IAEzB,MAAMiG,UAA2C,IAAIC,MAAMH,OAAO,MAAM;IACxE,IAAII,SAAS;IAEb,eAAeC;QACb,MAAOD,SAASJ,OAAO,MAAM,CAAE;YAC7B,MAAMM,QAAQF;YACdA,UAAU;YACVF,OAAO,CAACI,MAAM,GAAG,MAAMZ,wBACrBM,MAAM,CAACM,MAAM,EACbpC,aACAC;QAEJ;IACF;IAEA,MAAM/D,QAAQ,GAAG,CACf+F,MAAM,IAAI,CAAC;QAAE,QAAQxF,KAAK,GAAG,CAACsF,aAAaD,OAAO,MAAM;IAAE,GAAG,IAC3DK;IAGJ,OAAOH;AACT"}

package/dist/lib/service/index.js

@@ -271,7 +271,7 @@ class Service {
     }
     async describe(target, modelRuntime, opt) {
         (0, utils_namespaceObject.assert)(target, 'target is required for service.describe');
-        const context = await this.contextRetrieverFn();
+        const context = opt?.context || await this.contextRetrieverFn();
         const { shotSize } = context;
         const screenshotBase64 = context.screenshot.base64;
         (0, utils_namespaceObject.assert)(screenshotBase64, 'screenshot is required for service.describe');
@@ -317,7 +317,15 @@ class Service {
                 ]
             }
         ];
-        const res = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, modelRuntime);
+        let res;
+        try {
+            res = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, modelRuntime);
+        } catch (error) {
+            const recoveredResponse = (0, external_utils_js_namespaceObject.recoverDescribeResponseFromParseError)(error);
+            if (!recoveredResponse) throw error;
+            debug("describe: recovered malformed description JSON response");
+            return recoveredResponse;
+        }
         const { content } = res;
         (0, utils_namespaceObject.assert)(!content.error, `describe failed: ${content.error}`);
         (0, utils_namespaceObject.assert)(content.description, 'failed to describe the element');

package/dist/lib/service/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"service/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { defaultModelFamilyRequiredForLocateMessage } from '@/ai-model/errors';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  AiLocateSection,\n  buildSearchAreaConfig,\n} from '@/ai-model/inspect';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport {\n  AIResponseParseError,\n  callAIWithObjectResponse,\n} from '@/ai-model/service-caller';\nimport type { AIArgs } from '@/ai-model/types';\nimport type { SearchAreaConfig } from '@/ai-model/workflows/inspect/types';\nimport { expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  PlanningLocateParam,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\ninterface LocateSearchAreaResult {\n  config?: SearchAreaConfig;\n  trace: {\n    sourceRect?: Rect;\n    rawResponse?: string;\n    rawChoiceMessage?: unknown;\n    usage?: AIUsageInfo;\n  };\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: PlanningLocateParam,\n    opt: LocateOpts,\n    modelRuntime: ModelRuntime,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const { config: modelConfig } = modelRuntime;\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    if (!modelConfig.modelFamily) {\n      throw new Error(defaultModelFamilyRequiredForLocateMessage);\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    const searchArea = await this.resolveLocateSearchArea({\n      query,\n      queryPrompt,\n      opt,\n      context,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content,\n    } = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchArea.config,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      rawChoiceMessage,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea: searchArea.trace.sourceRect,\n      searchAreaRawResponse: searchArea.trace.rawResponse,\n      searchAreaRawChoiceMessage: searchArea.trace.rawChoiceMessage,\n      searchAreaUsage: searchArea.trace.usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea.trace.sourceRect,\n      error: errorLog,\n    };\n\n    const element = parseResult.element;\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: element ? [element] : [],\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (element) {\n      return {\n        element: {\n          center: element.center,\n          rect: element.rect,\n          description: element.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  private async resolveLocateSearchArea(options: {\n    query: PlanningLocateParam;\n    queryPrompt: TUserPrompt;\n    opt: LocateOpts;\n    context: UIContext;\n    modelRuntime: ModelRuntime;\n    abortSignal?: AbortSignal;\n  }): Promise<LocateSearchAreaResult> {\n    const { query, queryPrompt, opt, context, modelRuntime, abortSignal } =\n      options;\n    const { adapter } = modelRuntime;\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n    if (!query.deepLocate) {\n      return { trace: {} };\n    }\n\n    if (hasPlanLocatedElement) {\n      const config = await buildSearchAreaConfig({\n        context,\n        baseRect: opt.planLocatedElement!.rect,\n      });\n\n      return {\n        config,\n        trace: {\n          sourceRect: config.sourceRect,\n          rawResponse: JSON.stringify({\n            source: 'plan-located-element',\n            rect: opt.planLocatedElement!.rect,\n          }),\n        },\n      };\n    }\n\n    if (adapter.locate.supportsSearchArea) {\n      const searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: queryPrompt,\n        modelRuntime,\n        abortSignal,\n      });\n      const { searchAreaConfig } = searchAreaResponse;\n      assert(\n        searchAreaConfig,\n        `cannot find search area for \"${queryPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n\n      return {\n        config: searchAreaConfig,\n        trace: {\n          sourceRect: searchAreaConfig.sourceRect,\n          rawResponse: searchAreaResponse.rawResponse,\n          rawChoiceMessage: searchAreaResponse.rawChoiceMessage,\n          usage: searchAreaResponse.usage,\n        },\n      };\n    }\n\n    const firstPassLocateResult = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      modelRuntime,\n      abortSignal,\n    });\n    assert(\n      firstPassLocateResult.rect,\n      `cannot find search area for \"${queryPrompt}\"${\n        firstPassLocateResult.parseResult.errors?.length\n          ? `: ${firstPassLocateResult.parseResult.errors.join('\\n')}`\n          : ''\n      }`,\n    );\n\n    const config = await buildSearchAreaConfig({\n      context,\n      baseRect: firstPassLocateResult.rect,\n    });\n\n    return {\n      config,\n      trace: {\n        sourceRect: config.sourceRect,\n        rawResponse: JSON.stringify({\n          source: 'deep-locate-first-pass',\n          rect: firstPassLocateResult.rect,\n          rawResponse: firstPassLocateResult.rawResponse,\n        }),\n        rawChoiceMessage: firstPassLocateResult.rawChoiceMessage,\n        usage: firstPassLocateResult.usage,\n      },\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let rawChoiceMessage: unknown;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelRuntime,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      rawChoiceMessage = result.rawChoiceMessage;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          rawChoiceMessage: error.rawChoiceMessage,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawChoiceMessage,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelRuntime: ModelRuntime,\n    opt?: {\n      deepLocate?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      // Describe is not a coordinate-parsing flow, so it does not need image\n      // padding for bbox normalization.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(imagePayload, searchArea);\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n      msgs,\n      modelRuntime,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Service","query","opt","modelRuntime","abortSignal","modelConfig","queryPrompt","assert","Error","defaultModelFamilyRequiredForLocateMessage","context","searchArea","startTime","Date","parseResult","rect","rawResponse","rawChoiceMessage","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","element","dump","createServiceDump","ServiceError","options","adapter","hasPlanLocatedElement","config","buildSearchAreaConfig","searchAreaResponse","AiLocateSection","searchAreaConfig","firstPassLocateResult","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;ACuDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,YAA0B,EAC1BC,WAAyB,EACM;QAC/B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGF;QAChC,MAAMG,cAAc,AAAiB,YAAjB,OAAOL,QAAqBA,QAAQA,MAAM,MAAM;QACpEM,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QAEpBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAON,OAAoB;QAElC,IAAI,CAACI,YAAY,WAAW,EAC1B,MAAM,IAAIG,MAAMC,0BAAAA,0CAA0CA;QAG5D,MAAMC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,MAAMS,aAAa,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACpDV;YACAK;YACAJ;YACAQ;YACAP;YACAC;QACF;QAEA,MAAMQ,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YACxBV;YACA,0BAA0BJ;YAC1B,cAAcK,WAAW,MAAM;YAC/BR;YACAC;QACF;QAEA,MAAMiB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACP;YAC5BC;YACA,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACA,YAAYP,WAAW,KAAK,CAAC,UAAU;YACvC,uBAAuBA,WAAW,KAAK,CAAC,WAAW;YACnD,4BAA4BA,WAAW,KAAK,CAAC,gBAAgB;YAC7D,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACvCQ;QACF;QAEA,IAAIK;QACJ,IAAIV,YAAY,MAAM,EAAE,QACtBU,WAAW,CAAC,4BAA4B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASnB;YACX;YACA,aAAaS;YACb,MAAM;YACNO;YACA,YAAY,CAAC,CAACX,WAAW,KAAK,CAAC,UAAU;YACzC,OAAOa;QACT;QAEA,MAAME,UAAUZ,YAAY,OAAO;QAEnC,MAAMa,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC,UAAU;gBAACA;aAAQ,GAAG,EAAE;QAC1C;QAEA,IAAIF,UACF,MAAM,IAAIK,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,IAAID,SACF,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,MAAM;gBACtB,MAAMA,QAAQ,IAAI;gBAClB,aAAaA,QAAQ,WAAW;YAClC;YACAX;YACAY;QACF;QAGF,OAAO;YACL,SAAS;YACTZ;YACAY;QACF;IACF;IAEA,MAAc,wBAAwBG,OAOrC,EAAmC;QAClC,MAAM,EAAE7B,KAAK,EAAEK,WAAW,EAAEJ,GAAG,EAAEQ,OAAO,EAAEP,YAAY,EAAEC,WAAW,EAAE,GACnE0B;QACF,MAAM,EAAEC,OAAO,EAAE,GAAG5B;QACpB,MAAM6B,wBAAwB,CAAC,CAAC9B,KAAK,oBAAoB;QAEzD,IAAI,CAACD,MAAM,UAAU,EACnB,OAAO;YAAE,OAAO,CAAC;QAAE;QAGrB,IAAI+B,uBAAuB;YACzB,MAAMC,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;gBACzCxB;gBACA,UAAUR,IAAI,kBAAkB,CAAE,IAAI;YACxC;YAEA,OAAO;gBACL+B;gBACA,OAAO;oBACL,YAAYA,OAAO,UAAU;oBAC7B,aAAaV,KAAK,SAAS,CAAC;wBAC1B,QAAQ;wBACR,MAAMrB,IAAI,kBAAkB,CAAE,IAAI;oBACpC;gBACF;YACF;QACF;QAEA,IAAI6B,QAAQ,MAAM,CAAC,kBAAkB,EAAE;YACrC,MAAMI,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBAC/C1B;gBACA,oBAAoBJ;gBACpBH;gBACAC;YACF;YACA,MAAM,EAAEiC,gBAAgB,EAAE,GAAGF;YAC7B5B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8B,kBACA,CAAC,6BAA6B,EAAE/B,YAAY,CAAC,EAC3C6B,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAGJ,OAAO;gBACL,QAAQE;gBACR,OAAO;oBACL,YAAYA,iBAAiB,UAAU;oBACvC,aAAaF,mBAAmB,WAAW;oBAC3C,kBAAkBA,mBAAmB,gBAAgB;oBACrD,OAAOA,mBAAmB,KAAK;gBACjC;YACF;QACF;QAEA,MAAMG,wBAAwB,MAAMlB,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YAClDV;YACA,0BAA0BJ;YAC1BH;YACAC;QACF;QACAG,IAAAA,sBAAAA,MAAAA,AAAAA,EACE+B,sBAAsB,IAAI,EAC1B,CAAC,6BAA6B,EAAEhC,YAAY,CAAC,EAC3CgC,sBAAsB,WAAW,CAAC,MAAM,EAAE,SACtC,CAAC,EAAE,EAAEA,sBAAsB,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,GAC1D,IACJ;QAGJ,MAAML,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;YACzCxB;YACA,UAAU4B,sBAAsB,IAAI;QACtC;QAEA,OAAO;YACLL;YACA,OAAO;gBACL,YAAYA,OAAO,UAAU;gBAC7B,aAAaV,KAAK,SAAS,CAAC;oBAC1B,QAAQ;oBACR,MAAMe,sBAAsB,IAAI;oBAChC,aAAaA,sBAAsB,WAAW;gBAChD;gBACA,kBAAkBA,sBAAsB,gBAAgB;gBACxD,OAAOA,sBAAsB,KAAK;YACpC;QACF;IACF;IAEA,MAAM,QACJC,UAA+B,EAC/BpC,YAA0B,EAC1BD,GAA0B,EAC1BsC,eAAwB,EACxBC,gBAAoC,EACpC/B,OAAmB,EACe;QAClCH,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,SAAS;QAChBH,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAOgC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAM3B,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMuB,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,oBAAAA,AAAAA,EAAwB;gBAC3CjC;gBACA,WAAW6B;gBACXE;gBACA,eAAevC;gBACfC;gBACAqC;YACF;YACA1B,cAAc4B,OAAO,WAAW;YAChC1B,cAAc0B,OAAO,WAAW;YAChCzB,mBAAmByB,OAAO,gBAAgB;YAC1CxB,QAAQwB,OAAO,KAAK;YACpBvB,oBAAoBuB,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,yBAAAA,oBAAoBA,EAAE;gBAEzC,MAAMxB,WAAWR,KAAK,GAAG,KAAKD;gBAC9B,MAAMU,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAauB,MAAM,WAAW;oBAC9B,kBAAkBA,MAAM,gBAAgB;oBACxC,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMjB,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEW;oBAAW;oBACxB,MAAM;oBACNjB;oBACA,OAAOsB,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIf,kCAAAA,YAAYA,CAACe,MAAM,OAAO,EAAEjB;YACxC;YACA,MAAMiB;QACR;QAEA,MAAMvB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZL;YACAC;YACA,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIV,YAAY,MAAM,EAAE,QACtBU,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTc;YACF;YACA,MAAM;YACNjB;YACA,OAAOE;QACT;QAEA,MAAM,EAAEsB,IAAI,EAAEC,OAAO,EAAE,GAAGjC,eAAe,CAAC;QAE1C,MAAMa,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACXqB;QACF;QAEA,IAAItB,YAAY,CAACsB,MACf,MAAM,IAAIjB,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,OAAO;YACLmB;YACAC;YACA7B;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJqB,MAA+B,EAC/B7C,YAA0B,EAC1BD,GAEC,EACwD;QACzDK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOyC,QAAQ;QACf,MAAMtC,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEuC,QAAQ,EAAE,GAAGvC;QACrB,MAAMwC,mBAAmBxC,QAAQ,UAAU,CAAC,MAAM;QAClDH,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO2C,kBAAkB;QACzB,MAAMC,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIpD,KAAK,YAAY;YACnB,MAAMS,aAAagD,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;YAQhDnD,MAAM,oCAAoCa;YAC1C,MAAMiD,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAAWJ,cAAc9C;YACrD8C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChBF,MACA3D;QAGF,MAAM,EAAE8D,OAAO,EAAE,GAAGF;QACpBxD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC0D,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1D1D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO0D,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IA/YA,YACEvD,OAA2D,EAC3DR,GAAoB,CACpB;QAPF;QAEA;QAMEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMwD,QAAQ,OAAO,CAACxD;QAGlD,IAAI,AAAyB,WAAlBR,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAkYF"}
+{"version":3,"file":"service/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { defaultModelFamilyRequiredForLocateMessage } from '@/ai-model/errors';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  AiLocateSection,\n  buildSearchAreaConfig,\n} from '@/ai-model/inspect';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport {\n  AIResponseParseError,\n  callAIWithObjectResponse,\n} from '@/ai-model/service-caller';\nimport type { AIArgs } from '@/ai-model/types';\nimport type { SearchAreaConfig } from '@/ai-model/workflows/inspect/types';\nimport { expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  PlanningLocateParam,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  createServiceDump,\n  recoverDescribeResponseFromParseError,\n} from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\ninterface LocateSearchAreaResult {\n  config?: SearchAreaConfig;\n  trace: {\n    sourceRect?: Rect;\n    rawResponse?: string;\n    rawChoiceMessage?: unknown;\n    usage?: AIUsageInfo;\n  };\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: PlanningLocateParam,\n    opt: LocateOpts,\n    modelRuntime: ModelRuntime,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const { config: modelConfig } = modelRuntime;\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    if (!modelConfig.modelFamily) {\n      throw new Error(defaultModelFamilyRequiredForLocateMessage);\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    const searchArea = await this.resolveLocateSearchArea({\n      query,\n      queryPrompt,\n      opt,\n      context,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content,\n    } = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchArea.config,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      rawChoiceMessage,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea: searchArea.trace.sourceRect,\n      searchAreaRawResponse: searchArea.trace.rawResponse,\n      searchAreaRawChoiceMessage: searchArea.trace.rawChoiceMessage,\n      searchAreaUsage: searchArea.trace.usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea.trace.sourceRect,\n      error: errorLog,\n    };\n\n    const element = parseResult.element;\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: element ? [element] : [],\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (element) {\n      return {\n        element: {\n          center: element.center,\n          rect: element.rect,\n          description: element.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  private async resolveLocateSearchArea(options: {\n    query: PlanningLocateParam;\n    queryPrompt: TUserPrompt;\n    opt: LocateOpts;\n    context: UIContext;\n    modelRuntime: ModelRuntime;\n    abortSignal?: AbortSignal;\n  }): Promise<LocateSearchAreaResult> {\n    const { query, queryPrompt, opt, context, modelRuntime, abortSignal } =\n      options;\n    const { adapter } = modelRuntime;\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n    if (!query.deepLocate) {\n      return { trace: {} };\n    }\n\n    if (hasPlanLocatedElement) {\n      const config = await buildSearchAreaConfig({\n        context,\n        baseRect: opt.planLocatedElement!.rect,\n      });\n\n      return {\n        config,\n        trace: {\n          sourceRect: config.sourceRect,\n          rawResponse: JSON.stringify({\n            source: 'plan-located-element',\n            rect: opt.planLocatedElement!.rect,\n          }),\n        },\n      };\n    }\n\n    if (adapter.locate.supportsSearchArea) {\n      const searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: queryPrompt,\n        modelRuntime,\n        abortSignal,\n      });\n      const { searchAreaConfig } = searchAreaResponse;\n      assert(\n        searchAreaConfig,\n        `cannot find search area for \"${queryPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n\n      return {\n        config: searchAreaConfig,\n        trace: {\n          sourceRect: searchAreaConfig.sourceRect,\n          rawResponse: searchAreaResponse.rawResponse,\n          rawChoiceMessage: searchAreaResponse.rawChoiceMessage,\n          usage: searchAreaResponse.usage,\n        },\n      };\n    }\n\n    const firstPassLocateResult = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      modelRuntime,\n      abortSignal,\n    });\n    assert(\n      firstPassLocateResult.rect,\n      `cannot find search area for \"${queryPrompt}\"${\n        firstPassLocateResult.parseResult.errors?.length\n          ? `: ${firstPassLocateResult.parseResult.errors.join('\\n')}`\n          : ''\n      }`,\n    );\n\n    const config = await buildSearchAreaConfig({\n      context,\n      baseRect: firstPassLocateResult.rect,\n    });\n\n    return {\n      config,\n      trace: {\n        sourceRect: config.sourceRect,\n        rawResponse: JSON.stringify({\n          source: 'deep-locate-first-pass',\n          rect: firstPassLocateResult.rect,\n          rawResponse: firstPassLocateResult.rawResponse,\n        }),\n        rawChoiceMessage: firstPassLocateResult.rawChoiceMessage,\n        usage: firstPassLocateResult.usage,\n      },\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let rawChoiceMessage: unknown;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelRuntime,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      rawChoiceMessage = result.rawChoiceMessage;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          rawChoiceMessage: error.rawChoiceMessage,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawChoiceMessage,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelRuntime: ModelRuntime,\n    opt?: {\n      deepLocate?: boolean;\n      context?: UIContext;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = opt?.context || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      // Describe is not a coordinate-parsing flow, so it does not need image\n      // padding for bbox normalization.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(imagePayload, searchArea);\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    let res: Awaited<\n      ReturnType<typeof callAIWithObjectResponse<AIDescribeElementResponse>>\n    >;\n    try {\n      res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n        msgs,\n        modelRuntime,\n      );\n    } catch (error) {\n      const recoveredResponse = recoverDescribeResponseFromParseError(error);\n      if (!recoveredResponse) {\n        throw error;\n      }\n      debug('describe: recovered malformed description JSON response');\n      return recoveredResponse;\n    }\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Service","query","opt","modelRuntime","abortSignal","modelConfig","queryPrompt","assert","Error","defaultModelFamilyRequiredForLocateMessage","context","searchArea","startTime","Date","parseResult","rect","rawResponse","rawChoiceMessage","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","element","dump","createServiceDump","ServiceError","options","adapter","hasPlanLocatedElement","config","buildSearchAreaConfig","searchAreaResponse","AiLocateSection","searchAreaConfig","firstPassLocateResult","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","recoveredResponse","recoverDescribeResponseFromParseError","content","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;AC0DA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,YAA0B,EAC1BC,WAAyB,EACM;QAC/B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGF;QAChC,MAAMG,cAAc,AAAiB,YAAjB,OAAOL,QAAqBA,QAAQA,MAAM,MAAM;QACpEM,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QAEpBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAON,OAAoB;QAElC,IAAI,CAACI,YAAY,WAAW,EAC1B,MAAM,IAAIG,MAAMC,0BAAAA,0CAA0CA;QAG5D,MAAMC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,MAAMS,aAAa,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACpDV;YACAK;YACAJ;YACAQ;YACAP;YACAC;QACF;QAEA,MAAMQ,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YACxBV;YACA,0BAA0BJ;YAC1B,cAAcK,WAAW,MAAM;YAC/BR;YACAC;QACF;QAEA,MAAMiB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACP;YAC5BC;YACA,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACA,YAAYP,WAAW,KAAK,CAAC,UAAU;YACvC,uBAAuBA,WAAW,KAAK,CAAC,WAAW;YACnD,4BAA4BA,WAAW,KAAK,CAAC,gBAAgB;YAC7D,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACvCQ;QACF;QAEA,IAAIK;QACJ,IAAIV,YAAY,MAAM,EAAE,QACtBU,WAAW,CAAC,4BAA4B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASnB;YACX;YACA,aAAaS;YACb,MAAM;YACNO;YACA,YAAY,CAAC,CAACX,WAAW,KAAK,CAAC,UAAU;YACzC,OAAOa;QACT;QAEA,MAAME,UAAUZ,YAAY,OAAO;QAEnC,MAAMa,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC,UAAU;gBAACA;aAAQ,GAAG,EAAE;QAC1C;QAEA,IAAIF,UACF,MAAM,IAAIK,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,IAAID,SACF,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,MAAM;gBACtB,MAAMA,QAAQ,IAAI;gBAClB,aAAaA,QAAQ,WAAW;YAClC;YACAX;YACAY;QACF;QAGF,OAAO;YACL,SAAS;YACTZ;YACAY;QACF;IACF;IAEA,MAAc,wBAAwBG,OAOrC,EAAmC;QAClC,MAAM,EAAE7B,KAAK,EAAEK,WAAW,EAAEJ,GAAG,EAAEQ,OAAO,EAAEP,YAAY,EAAEC,WAAW,EAAE,GACnE0B;QACF,MAAM,EAAEC,OAAO,EAAE,GAAG5B;QACpB,MAAM6B,wBAAwB,CAAC,CAAC9B,KAAK,oBAAoB;QAEzD,IAAI,CAACD,MAAM,UAAU,EACnB,OAAO;YAAE,OAAO,CAAC;QAAE;QAGrB,IAAI+B,uBAAuB;YACzB,MAAMC,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;gBACzCxB;gBACA,UAAUR,IAAI,kBAAkB,CAAE,IAAI;YACxC;YAEA,OAAO;gBACL+B;gBACA,OAAO;oBACL,YAAYA,OAAO,UAAU;oBAC7B,aAAaV,KAAK,SAAS,CAAC;wBAC1B,QAAQ;wBACR,MAAMrB,IAAI,kBAAkB,CAAE,IAAI;oBACpC;gBACF;YACF;QACF;QAEA,IAAI6B,QAAQ,MAAM,CAAC,kBAAkB,EAAE;YACrC,MAAMI,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBAC/C1B;gBACA,oBAAoBJ;gBACpBH;gBACAC;YACF;YACA,MAAM,EAAEiC,gBAAgB,EAAE,GAAGF;YAC7B5B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8B,kBACA,CAAC,6BAA6B,EAAE/B,YAAY,CAAC,EAC3C6B,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAGJ,OAAO;gBACL,QAAQE;gBACR,OAAO;oBACL,YAAYA,iBAAiB,UAAU;oBACvC,aAAaF,mBAAmB,WAAW;oBAC3C,kBAAkBA,mBAAmB,gBAAgB;oBACrD,OAAOA,mBAAmB,KAAK;gBACjC;YACF;QACF;QAEA,MAAMG,wBAAwB,MAAMlB,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;YAClDV;YACA,0BAA0BJ;YAC1BH;YACAC;QACF;QACAG,IAAAA,sBAAAA,MAAAA,AAAAA,EACE+B,sBAAsB,IAAI,EAC1B,CAAC,6BAA6B,EAAEhC,YAAY,CAAC,EAC3CgC,sBAAsB,WAAW,CAAC,MAAM,EAAE,SACtC,CAAC,EAAE,EAAEA,sBAAsB,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,GAC1D,IACJ;QAGJ,MAAML,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,qBAAAA,AAAAA,EAAsB;YACzCxB;YACA,UAAU4B,sBAAsB,IAAI;QACtC;QAEA,OAAO;YACLL;YACA,OAAO;gBACL,YAAYA,OAAO,UAAU;gBAC7B,aAAaV,KAAK,SAAS,CAAC;oBAC1B,QAAQ;oBACR,MAAMe,sBAAsB,IAAI;oBAChC,aAAaA,sBAAsB,WAAW;gBAChD;gBACA,kBAAkBA,sBAAsB,gBAAgB;gBACxD,OAAOA,sBAAsB,KAAK;YACpC;QACF;IACF;IAEA,MAAM,QACJC,UAA+B,EAC/BpC,YAA0B,EAC1BD,GAA0B,EAC1BsC,eAAwB,EACxBC,gBAAoC,EACpC/B,OAAmB,EACe;QAClCH,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,SAAS;QAChBH,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAOgC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAM3B,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMuB,SAAS,MAAMC,AAAAA,IAAAA,2BAAAA,oBAAAA,AAAAA,EAAwB;gBAC3CjC;gBACA,WAAW6B;gBACXE;gBACA,eAAevC;gBACfC;gBACAqC;YACF;YACA1B,cAAc4B,OAAO,WAAW;YAChC1B,cAAc0B,OAAO,WAAW;YAChCzB,mBAAmByB,OAAO,gBAAgB;YAC1CxB,QAAQwB,OAAO,KAAK;YACpBvB,oBAAoBuB,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,yBAAAA,oBAAoBA,EAAE;gBAEzC,MAAMxB,WAAWR,KAAK,GAAG,KAAKD;gBAC9B,MAAMU,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAauB,MAAM,WAAW;oBAC9B,kBAAkBA,MAAM,gBAAgB;oBACxC,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMjB,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEW;oBAAW;oBACxB,MAAM;oBACNjB;oBACA,OAAOsB,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIf,kCAAAA,YAAYA,CAACe,MAAM,OAAO,EAAEjB;YACxC;YACA,MAAMiB;QACR;QAEA,MAAMvB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZL;YACAC;YACA,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIV,YAAY,MAAM,EAAE,QACtBU,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTc;YACF;YACA,MAAM;YACNjB;YACA,OAAOE;QACT;QAEA,MAAM,EAAEsB,IAAI,EAAEC,OAAO,EAAE,GAAGjC,eAAe,CAAC;QAE1C,MAAMa,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACXqB;QACF;QAEA,IAAItB,YAAY,CAACsB,MACf,MAAM,IAAIjB,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,OAAO;YACLmB;YACAC;YACA7B;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJqB,MAA+B,EAC/B7C,YAA0B,EAC1BD,GAGC,EACwD;QACzDK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOyC,QAAQ;QACf,MAAMtC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAC9D,MAAM,EAAE+C,QAAQ,EAAE,GAAGvC;QACrB,MAAMwC,mBAAmBxC,QAAQ,UAAU,CAAC,MAAM;QAClDH,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO2C,kBAAkB;QACzB,MAAMC,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIpD,KAAK,YAAY;YACnB,MAAMS,aAAagD,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;YAQhDnD,MAAM,oCAAoCa;YAC1C,MAAMiD,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAAWJ,cAAc9C;YACrD8C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,IAAIM;QAGJ,IAAI;YACFA,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACVF,MACA3D;QAEJ,EAAE,OAAOyC,OAAO;YACd,MAAMqB,oBAAoBC,AAAAA,IAAAA,kCAAAA,qCAAAA,AAAAA,EAAsCtB;YAChE,IAAI,CAACqB,mBACH,MAAMrB;YAER9C,MAAM;YACN,OAAOmE;QACT;QAEA,MAAM,EAAEE,OAAO,EAAE,GAAGJ;QACpBxD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC4D,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1D5D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO4D,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IA5ZA,YACEzD,OAA2D,EAC3DR,GAAoB,CACpB;QAPF;QAEA;QAMEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAM0D,QAAQ,OAAO,CAAC1D;QAGlD,IAAI,AAAyB,WAAlBR,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AA+YF"}

package/dist/lib/service/utils.js

@@ -24,8 +24,10 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
+    recoverDescribeResponseFromParseError: ()=>recoverDescribeResponseFromParseError,
     createServiceDump: ()=>createServiceDump
 });
+const index_js_namespaceObject = require("../ai-model/service-caller/index.js");
 const utils_namespaceObject = require("@midscene/shared/utils");
 function createServiceDump(data) {
     const baseData = {
@@ -38,9 +40,59 @@ function createServiceDump(data) {
     };
     return finalData;
 }
+function readNextSignificantChar(input, startIndex) {
+    let index = startIndex;
+    while(index < input.length && /\s/.test(input[index]))index += 1;
+    return input[index];
+}
+function extractPossiblyMalformedStringField(input, fieldName) {
+    const escapedFieldName = fieldName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const fieldStart = new RegExp(`"${escapedFieldName}"\\s*:\\s*"`).exec(input);
+    if (!fieldStart) return;
+    let index = fieldStart.index + fieldStart[0].length;
+    let escaped = false;
+    let valueForJsonParse = '';
+    for(; index < input.length; index += 1){
+        const char = input[index];
+        if (escaped) {
+            valueForJsonParse += char;
+            escaped = false;
+            continue;
+        }
+        if ('\\' === char) {
+            valueForJsonParse += char;
+            escaped = true;
+            continue;
+        }
+        if ('"' !== char) {
+            valueForJsonParse += char;
+            continue;
+        }
+        const nextSignificantChar = readNextSignificantChar(input, index + 1);
+        if (',' === nextSignificantChar || '}' === nextSignificantChar || ']' === nextSignificantChar || void 0 === nextSignificantChar) try {
+            return JSON.parse(`"${valueForJsonParse}"`);
+        } catch  {
+            return valueForJsonParse;
+        }
+        valueForJsonParse += '\\"';
+    }
+}
+function recoverDescribeResponseFromParseError(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const rawResponse = error instanceof index_js_namespaceObject.AIResponseParseError ? error.rawResponse : message.match(/Response -\s*\n\s*([\s\S]*)$/)?.[1];
+    if (!rawResponse || !message.includes('failed to parse LLM response into JSON') && !(error instanceof index_js_namespaceObject.AIResponseParseError)) return;
+    const jsonLikeResponse = (0, index_js_namespaceObject.extractJSONFromCodeBlock)(rawResponse);
+    const description = extractPossiblyMalformedStringField(jsonLikeResponse, "description")?.trim();
+    if (!description) return;
+    return {
+        description
+    };
+}
 exports.createServiceDump = __webpack_exports__.createServiceDump;
+exports.recoverDescribeResponseFromParseError = __webpack_exports__.recoverDescribeResponseFromParseError;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
-    "createServiceDump"
+    "createServiceDump",
+    "recoverDescribeResponseFromParseError"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true

package/dist/lib/service/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"service/utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DumpMeta, PartialServiceDumpFromSDK, ServiceDump } from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function createServiceDump(\n  data: PartialServiceDumpFromSDK,\n): ServiceDump {\n  const baseData: DumpMeta = {\n    logTime: Date.now(),\n  };\n  const finalData: ServiceDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  return finalData;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createServiceDump","data","baseData","Date","finalData","uuid"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACHO,SAASI,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT"}
+{"version":3,"file":"service/utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  AIResponseParseError,\n  extractJSONFromCodeBlock,\n} from '@/ai-model/service-caller';\nimport type { AIDescribeElementResponse } from '@/types';\nimport type { DumpMeta, PartialServiceDumpFromSDK, ServiceDump } from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function createServiceDump(\n  data: PartialServiceDumpFromSDK,\n): ServiceDump {\n  const baseData: DumpMeta = {\n    logTime: Date.now(),\n  };\n  const finalData: ServiceDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  return finalData;\n}\n\nfunction readNextSignificantChar(input: string, startIndex: number) {\n  let index = startIndex;\n  while (index < input.length && /\\s/.test(input[index])) {\n    index += 1;\n  }\n  return input[index];\n}\n\nfunction extractPossiblyMalformedStringField(input: string, fieldName: string) {\n  const escapedFieldName = fieldName.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n  const fieldStart = new RegExp(`\"${escapedFieldName}\"\\\\s*:\\\\s*\"`).exec(input);\n  if (!fieldStart) {\n    return undefined;\n  }\n\n  let index = fieldStart.index + fieldStart[0].length;\n  let escaped = false;\n  let valueForJsonParse = '';\n\n  for (; index < input.length; index += 1) {\n    const char = input[index];\n\n    if (escaped) {\n      valueForJsonParse += char;\n      escaped = false;\n      continue;\n    }\n\n    if (char === '\\\\') {\n      valueForJsonParse += char;\n      escaped = true;\n      continue;\n    }\n\n    if (char !== '\"') {\n      valueForJsonParse += char;\n      continue;\n    }\n\n    const nextSignificantChar = readNextSignificantChar(input, index + 1);\n    if (\n      nextSignificantChar === ',' ||\n      nextSignificantChar === '}' ||\n      nextSignificantChar === ']' ||\n      nextSignificantChar === undefined\n    ) {\n      try {\n        return JSON.parse(`\"${valueForJsonParse}\"`);\n      } catch {\n        return valueForJsonParse;\n      }\n    }\n\n    valueForJsonParse += '\\\\\"';\n  }\n\n  return undefined;\n}\n\nexport function recoverDescribeResponseFromParseError(\n  error: unknown,\n): Pick<AIDescribeElementResponse, 'description'> | undefined {\n  const message = error instanceof Error ? error.message : String(error);\n  const rawResponse =\n    error instanceof AIResponseParseError\n      ? error.rawResponse\n      : message.match(/Response -\\s*\\n\\s*([\\s\\S]*)$/)?.[1];\n\n  if (\n    !rawResponse ||\n    (!message.includes('failed to parse LLM response into JSON') &&\n      !(error instanceof AIResponseParseError))\n  ) {\n    return undefined;\n  }\n\n  const jsonLikeResponse = extractJSONFromCodeBlock(rawResponse);\n  const description = extractPossiblyMalformedStringField(\n    jsonLikeResponse,\n    'description',\n  )?.trim();\n\n  if (!description) {\n    return undefined;\n  }\n\n  return { description };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createServiceDump","data","baseData","Date","finalData","uuid","readNextSignificantChar","input","startIndex","index","extractPossiblyMalformedStringField","fieldName","escapedFieldName","fieldStart","RegExp","escaped","valueForJsonParse","char","nextSignificantChar","undefined","JSON","recoverDescribeResponseFromParseError","error","message","Error","String","rawResponse","AIResponseParseError","jsonLikeResponse","extractJSONFromCodeBlock","description"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACEO,SAASI,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT;AAEA,SAASE,wBAAwBC,KAAa,EAAEC,UAAkB;IAChE,IAAIC,QAAQD;IACZ,MAAOC,QAAQF,MAAM,MAAM,IAAI,KAAK,IAAI,CAACA,KAAK,CAACE,MAAM,EACnDA,SAAS;IAEX,OAAOF,KAAK,CAACE,MAAM;AACrB;AAEA,SAASC,oCAAoCH,KAAa,EAAEI,SAAiB;IAC3E,MAAMC,mBAAmBD,UAAU,OAAO,CAAC,uBAAuB;IAClE,MAAME,aAAa,IAAIC,OAAO,CAAC,CAAC,EAAEF,iBAAiB,WAAW,CAAC,EAAE,IAAI,CAACL;IACtE,IAAI,CAACM,YACH;IAGF,IAAIJ,QAAQI,WAAW,KAAK,GAAGA,UAAU,CAAC,EAAE,CAAC,MAAM;IACnD,IAAIE,UAAU;IACd,IAAIC,oBAAoB;IAExB,MAAOP,QAAQF,MAAM,MAAM,EAAEE,SAAS,EAAG;QACvC,MAAMQ,OAAOV,KAAK,CAACE,MAAM;QAEzB,IAAIM,SAAS;YACXC,qBAAqBC;YACrBF,UAAU;YACV;QACF;QAEA,IAAIE,AAAS,SAATA,MAAe;YACjBD,qBAAqBC;YACrBF,UAAU;YACV;QACF;QAEA,IAAIE,AAAS,QAATA,MAAc;YAChBD,qBAAqBC;YACrB;QACF;QAEA,MAAMC,sBAAsBZ,wBAAwBC,OAAOE,QAAQ;QACnE,IACES,AAAwB,QAAxBA,uBACAA,AAAwB,QAAxBA,uBACAA,AAAwB,QAAxBA,uBACAA,AAAwBC,WAAxBD,qBAEA,IAAI;YACF,OAAOE,KAAK,KAAK,CAAC,CAAC,CAAC,EAAEJ,kBAAkB,CAAC,CAAC;QAC5C,EAAE,OAAM;YACN,OAAOA;QACT;QAGFA,qBAAqB;IACvB;AAGF;AAEO,SAASK,sCACdC,KAAc;IAEd,MAAMC,UAAUD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;IAChE,MAAMI,cACJJ,iBAAiBK,yBAAAA,oBAAoBA,GACjCL,MAAM,WAAW,GACjBC,QAAQ,KAAK,CAAC,iCAAiC,CAAC,EAAE;IAExD,IACE,CAACG,eACA,CAACH,QAAQ,QAAQ,CAAC,6CACjB,CAAED,CAAAA,iBAAiBK,yBAAAA,oBAAmB,AAAnBA,GAErB;IAGF,MAAMC,mBAAmBC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyBH;IAClD,MAAMI,cAAcpB,oCAClBkB,kBACA,gBACC;IAEH,IAAI,CAACE,aACH;IAGF,OAAO;QAAEA;IAAY;AACvB"}

package/dist/lib/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/types.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport type { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport { ServiceError } from './errors';\nexport {\n  ExecutionDump,\n  ReportActionDump,\n  GroupedActionDump,\n} from './dump/report-action-dump';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  /**\n   * Raw top-level `.model` value returned by the model service response.\n   */\n  response_model_name: string | undefined;\n  /**\n   * Semantic intent of the model call, such as default, planning, or insight.\n   */\n  intent: string | undefined;\n  /**\n   * Config slot where the model config was resolved from. For example, a\n   * planning call may use the default slot when no planning model is configured.\n   */\n  slot: string | undefined;\n  request_id: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type LocateResultPoint = [number, number];\nexport type Bbox = [number, number, number, number];\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\n\nexport interface AIElementLocateResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  errors?: string[];\n}\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  references_bbox?: LocateResultBbox[];\n  references_point?: LocateResultPoint[];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepLocate: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  /**\n   * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n   */\n  abstract screenshot: ScreenshotItem;\n\n  /**\n   * screenshot size after shrinking\n   */\n  abstract shotSize: Size;\n\n  /**\n   * The ratio for converting shrunk screenshot coordinates to logical coordinates.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - User-defined screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n   * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n   */\n  abstract shrunkShotToLogicalRatio: number;\n\n  abstract _isFrozen?: boolean;\n\n  // @deprecated - backward compatibility for aiLocate\n  abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  /**\n   * Adapter-extracted content from the search-area model call. This is not the\n   * full provider response or choices[0].message.\n   */\n  searchAreaRawResponse?: string;\n  searchAreaRawChoiceMessage?: unknown;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport type ReportAttributes = Record<\n  string,\n  string | number | boolean | null | undefined\n>;\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: ReportAttributes;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement?: LocateResultElement[];\n  matchedRect?: Rect;\n  deepLocate?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n}\n\nexport type PlanningLocateParamWithLocatedPixelBbox = PlanningLocateParam & {\n  /** Pixel bbox of the located element in screenshot coordinates. */\n  locatedPixelBbox: PixelBbox;\n};\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n  param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' | 'running' | 'finished';\n\nexport interface SubGoal {\n  index: number;\n  status: SubGoalStatus;\n  description: string;\n  logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  thought?: string;\n  log: string;\n  memory?: string;\n  error?: string;\n  finalizeMessage?: string;\n  finalizeSuccess?: boolean;\n  updateSubGoals?: SubGoal[];\n  markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: string;\n  rawChoiceMessage?: unknown;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n  output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    taskId: string;\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    /**\n     * Optional feedback produced by a task for the next planning round.\n     * This is execution metadata, not part of the action return value.\n     */\n    planningFeedback?: string;\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      getUiContextStart?: number;\n      getUiContextEnd?: number;\n      callAiStart?: number;\n      callAiEnd?: number;\n      beforeInvokeActionHookStart?: number;\n      beforeInvokeActionHookEnd?: number;\n      callActionStart?: number;\n      callActionEnd?: number;\n      afterInvokeActionHookStart?: number;\n      afterInvokeActionHookEnd?: number;\n      captureAfterCallingSnapshotStart?: number;\n      captureAfterCallingSnapshotEnd?: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    /**\n     * Pixel rect of the deepLocate first-stage search area in screenshot\n     * coordinates. Used by reports to explain the crop/zoom area that the\n     * final locate ran against.\n     */\n    searchArea?: Rect;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  /** Stable unique identifier for this execution run */\n  id?: string;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n  domIncluded?: boolean | 'visible-only';\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport interface ExecutionTaskPlanningParam {\n  userInstruction: TUserPrompt;\n  userInstructionDisplay?: string;\n  aiActContext?: string;\n  imagesIncludeCount?: number;\n  deepThink?: DeepThinkOption;\n  subGoalStatus?: string;\n  memoriesStatus?: string;\n}\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningParam,\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nReport metadata - extracted from ReportActionDump for per-execution writes\n*/\nexport interface ReportMeta {\n  groupName: string;\n  groupDescription?: string;\n  sdkVersion: string;\n  modelBriefs: ModelBrief[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupMeta = ReportMeta;\n\n/*\nReport dump\n*/\nexport interface IReportActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: ModelBrief[];\n  executions: IExecutionDump[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type IGroupedActionDump = IReportActionDump;\n\nexport interface ModelBrief {\n  /**\n   * The intent/category of the model call, for example \"planning\" or \"insight\".\n   */\n  intent?: string;\n\n  /**\n   * The model name returned by usage metadata, for example \"gpt-4o\".\n   */\n  name?: string;\n\n  /**\n   * Optional human-readable model description, for example \"qwen2.5-vl mode\".\n   */\n  modelDescription?: string;\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (\n    param: TParam,\n    context?: ExecutorContext,\n  ) => Promise<TReturn> | TReturn;\n  delayBeforeRunner?: number;\n  delayAfterRunner?: number;\n  /**\n   * An example param object for this action.\n   * Locate fields with { prompt } may be resolved to internal pixel bboxes when needed.\n   */\n  sample?: { [K in keyof TParam]?: any };\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n  /**\n   * Optional cache directory path.\n   * When set, cache files are written to this directory instead of\n   * `<MIDSCENE_RUN_DIR>/cache`.\n   */\n  cacheDir?: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  // @deprecated Use `reportFileName` and `cache.id` instead.\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if persist per-execution dump files next to the report, default false */\n  persistExecutionDump?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n\n  /**\n   * Use directory-based report format with separate image files.\n   *\n   * When enabled:\n   * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n   * - Report is generated as `index.html` with relative image paths\n   * - Reduces memory usage and report file size\n   *\n   * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n   * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n   * work due to browser CORS restrictions.\n   *\n   * @default 'single-html'\n   */\n  outputFormat?: 'single-html' | 'html-and-external-assets';\n\n  onTaskStartTip?: OnTaskStartTip;\n  aiActContext?: string;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  reportAttributes?: ReportAttributes;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults are resolved by the active model adapter: 20 for standard planning,\n   * 40 for UI-TARS, and 100 for Auto-GLM.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Wait time in milliseconds after each action execution.\n   * This allows the UI to settle and stabilize before the next action.\n   * Defaults to 300ms when not provided.\n   */\n  waitAfterAction?: number;\n\n  /**\n   * When set to true, Midscene will use the target device's formatted local\n   * time instead of the runtime system time. The target interface must implement\n   * getDeviceLocalTimeString to provide device-local wall-clock time.\n   * Default: false\n   */\n  useDeviceTime?: boolean;\n\n  /**\n   * Custom screenshot shrink factor to reduce AI token usage.\n   * When set, the screenshot will be scaled down by this factor from the physical resolution.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - AI analyzes the 1500px screenshot\n   * - Coordinates are transformed back to logical (500px) before actions execute\n   *\n   * Benefits:\n   * - Reduces token usage for high-resolution screenshots\n   * - Maintains accuracy by scaling coordinates appropriately\n   *\n   * Must be >= 1 (shrinking only, enlarging is not supported).\n   *\n   * @default 1 (no shrinking, uses original physical screenshot)\n   */\n  screenshotShrinkFactor?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileAttributes {\n  testDuration: number;\n  testStatus: TestStatus;\n  testTitle: string;\n  testId: string;\n  testDescription: string;\n}\n\nexport type ReportFileWithAttributes =\n  | {\n      reportFilePath: string;\n      reportAttributes: ReportFileAttributes;\n    }\n  | {\n      reportFilePath?: string;\n      reportAttributes: ReportFileAttributes & { testStatus: 'skipped' };\n    };\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","UIContext"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;ICyHO,MAAeI;IA4BtB"}
+{"version":3,"file":"types.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/types.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport type { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport { ServiceError } from './errors';\nexport {\n  ExecutionDump,\n  ReportActionDump,\n  GroupedActionDump,\n} from './dump/report-action-dump';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  /**\n   * Raw top-level `.model` value returned by the model service response.\n   */\n  response_model_name: string | undefined;\n  /**\n   * Semantic intent of the model call, such as default, planning, or insight.\n   */\n  intent: string | undefined;\n  /**\n   * Config slot where the model config was resolved from. For example, a\n   * planning call may use the default slot when no planning model is configured.\n   */\n  slot: string | undefined;\n  request_id: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type LocateResultPoint = [number, number];\nexport type Bbox = [number, number, number, number];\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\n\nexport interface AIElementLocateResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  errors?: string[];\n}\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  references_bbox?: LocateResultBbox[];\n  references_point?: LocateResultPoint[];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n  includedInRect?: boolean;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepLocate: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  /**\n   * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n   */\n  abstract screenshot: ScreenshotItem;\n\n  /**\n   * screenshot size after shrinking\n   */\n  abstract shotSize: Size;\n\n  /**\n   * The ratio for converting shrunk screenshot coordinates to logical coordinates.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - User-defined screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n   * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n   */\n  abstract shrunkShotToLogicalRatio: number;\n\n  abstract _isFrozen?: boolean;\n\n  // @deprecated - backward compatibility for aiLocate\n  abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  /**\n   * Adapter-extracted content from the search-area model call. This is not the\n   * full provider response or choices[0].message.\n   */\n  searchAreaRawResponse?: string;\n  searchAreaRawChoiceMessage?: unknown;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport type ReportAttributes = Record<\n  string,\n  string | number | boolean | null | undefined\n>;\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: ReportAttributes;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement?: LocateResultElement[];\n  matchedRect?: Rect;\n  deepLocate?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n}\n\nexport type PlanningLocateParamWithLocatedPixelBbox = PlanningLocateParam & {\n  /** Pixel bbox of the located element in screenshot coordinates. */\n  locatedPixelBbox: PixelBbox;\n};\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n  param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' | 'running' | 'finished';\n\nexport interface SubGoal {\n  index: number;\n  status: SubGoalStatus;\n  description: string;\n  logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  thought?: string;\n  log: string;\n  memory?: string;\n  error?: string;\n  finalizeMessage?: string;\n  finalizeSuccess?: boolean;\n  updateSubGoals?: SubGoal[];\n  markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: string;\n  rawChoiceMessage?: unknown;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n  output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    taskId: string;\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    /**\n     * Optional feedback produced by a task for the next planning round.\n     * This is execution metadata, not part of the action return value.\n     */\n    planningFeedback?: string;\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      getUiContextStart?: number;\n      getUiContextEnd?: number;\n      callAiStart?: number;\n      callAiEnd?: number;\n      beforeInvokeActionHookStart?: number;\n      beforeInvokeActionHookEnd?: number;\n      callActionStart?: number;\n      callActionEnd?: number;\n      afterInvokeActionHookStart?: number;\n      afterInvokeActionHookEnd?: number;\n      captureAfterCallingSnapshotStart?: number;\n      captureAfterCallingSnapshotEnd?: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    /**\n     * Pixel rect of the deepLocate first-stage search area in screenshot\n     * coordinates. Used by reports to explain the crop/zoom area that the\n     * final locate ran against.\n     */\n    searchArea?: Rect;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  /** Stable unique identifier for this execution run */\n  id?: string;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n  domIncluded?: boolean | 'visible-only';\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport interface ExecutionTaskPlanningParam {\n  userInstruction: TUserPrompt;\n  userInstructionDisplay?: string;\n  aiActContext?: string;\n  imagesIncludeCount?: number;\n  deepThink?: DeepThinkOption;\n  subGoalStatus?: string;\n  memoriesStatus?: string;\n}\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningParam,\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nReport metadata - extracted from ReportActionDump for per-execution writes\n*/\nexport interface ReportMeta {\n  groupName: string;\n  groupDescription?: string;\n  sdkVersion: string;\n  modelBriefs: ModelBrief[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupMeta = ReportMeta;\n\n/*\nReport dump\n*/\nexport interface IReportActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: ModelBrief[];\n  executions: IExecutionDump[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type IGroupedActionDump = IReportActionDump;\n\nexport interface ModelBrief {\n  /**\n   * The intent/category of the model call, for example \"planning\" or \"insight\".\n   */\n  intent?: string;\n\n  /**\n   * The model name returned by usage metadata, for example \"gpt-4o\".\n   */\n  name?: string;\n\n  /**\n   * Optional human-readable model description, for example \"qwen2.5-vl mode\".\n   */\n  modelDescription?: string;\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (\n    param: TParam,\n    context?: ExecutorContext,\n  ) => Promise<TReturn> | TReturn;\n  delayBeforeRunner?: number;\n  delayAfterRunner?: number;\n  /**\n   * An example param object for this action.\n   * Locate fields with { prompt } may be resolved to internal pixel bboxes when needed.\n   */\n  sample?: { [K in keyof TParam]?: any };\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n  /**\n   * Optional cache directory path.\n   * When set, cache files are written to this directory instead of\n   * `<MIDSCENE_RUN_DIR>/cache`.\n   */\n  cacheDir?: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  // @deprecated Use `reportFileName` and `cache.id` instead.\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if persist per-execution dump files next to the report, default false */\n  persistExecutionDump?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n\n  /**\n   * Use directory-based report format with separate image files.\n   *\n   * When enabled:\n   * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n   * - Report is generated as `index.html` with relative image paths\n   * - Reduces memory usage and report file size\n   *\n   * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n   * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n   * work due to browser CORS restrictions.\n   *\n   * @default 'single-html'\n   */\n  outputFormat?: 'single-html' | 'html-and-external-assets';\n\n  onTaskStartTip?: OnTaskStartTip;\n  aiActContext?: string;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  reportAttributes?: ReportAttributes;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults are resolved by the active model adapter: 20 for standard planning,\n   * 40 for UI-TARS, and 100 for Auto-GLM.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Wait time in milliseconds after each action execution.\n   * This allows the UI to settle and stabilize before the next action.\n   * Defaults to 300ms when not provided.\n   */\n  waitAfterAction?: number;\n\n  /**\n   * When set to true, Midscene will use the target device's formatted local\n   * time instead of the runtime system time. The target interface must implement\n   * getDeviceLocalTimeString to provide device-local wall-clock time.\n   * Default: false\n   */\n  useDeviceTime?: boolean;\n\n  /**\n   * Custom screenshot shrink factor to reduce AI token usage.\n   * When set, the screenshot will be scaled down by this factor from the physical resolution.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - AI analyzes the 1500px screenshot\n   * - Coordinates are transformed back to logical (500px) before actions execute\n   *\n   * Benefits:\n   * - Reduces token usage for high-resolution screenshots\n   * - Maintains accuracy by scaling coordinates appropriately\n   *\n   * Must be >= 1 (shrinking only, enlarging is not supported).\n   *\n   * @default 1 (no shrinking, uses original physical screenshot)\n   */\n  screenshotShrinkFactor?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileAttributes {\n  testDuration: number;\n  testStatus: TestStatus;\n  testTitle: string;\n  testId: string;\n  testDescription: string;\n}\n\nexport type ReportFileWithAttributes =\n  | {\n      reportFilePath: string;\n      reportAttributes: ReportFileAttributes;\n    }\n  | {\n      reportFilePath?: string;\n      reportAttributes: ReportFileAttributes & { testStatus: 'skipped' };\n    };\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","UIContext"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;IC0HO,MAAeI;IA4BtB"}