@midscene/playground 1.9.8 → 1.10.0

package/dist/es/recorder-ui-describer.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"recorder-ui-describer.mjs","sources":["../../src/recorder-ui-describer.ts"],"sourcesContent":["import type { Rect } from '@midscene/core';\nimport { type IModelConfig, ModelConfigManager } from '@midscene/shared/env';\nimport type {\n  MidsceneRecorderEvent,\n  MidsceneRecorderPageInfo,\n  MidsceneRecorderSemanticAction,\n  MidsceneRecorderTarget,\n} from '@midscene/shared/recorder';\nimport {\n  buildMidsceneRecorderActionSummary,\n  buildMidsceneRecorderReplayInstruction,\n  getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\nimport { callAIWithObjectResponse } from './recorder-ai-service';\nimport { RECORDER_UI_DESCRIBER_SYSTEM_PROMPT } from './recorder-ui-describer-prompt';\n\nexport interface DescribeRecorderUIEventInput {\n  event: MidsceneRecorderEvent;\n  target?: MidsceneRecorderTarget;\n}\n\nexport interface DescribeRecorderUIEventOptions {\n  maxRetries?: number;\n  retryDelayMs?: number;\n  concurrency?: number;\n}\n\nexport interface DescribeRecorderUIEventResult {\n  event: MidsceneRecorderEvent;\n  usedFallback: boolean;\n  error?: string;\n}\n\ninterface RecorderUIEventAIResponse {\n  elementDescription?: string;\n  replayInstruction?: string;\n  actionSummary?: string;\n  scrollDestinationDescription?: string;\n  confidence?: 'high' | 'medium' | 'low';\n  error?: string;\n}\n\nconst RECORDER_UI_DESCRIBER_DEFAULT_RETRIES = 2;\nconst RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS = 200;\nconst RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY = 2;\n\nfunction delay(ms: number) {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nfunction isResolvedModelConfig(\n  modelConfig: IModelConfig | Record<string, unknown>,\n): modelConfig is IModelConfig {\n  return (\n    typeof modelConfig.modelName === 'string' &&\n    (typeof modelConfig.modelDescription === 'string' ||\n      typeof modelConfig.intent === 'string' ||\n      typeof modelConfig.slot === 'string' ||\n      typeof modelConfig.openaiApiKey === 'string')\n  );\n}\n\nfunction resolveRecorderModelConfig(\n  modelConfig: IModelConfig | Record<string, unknown>,\n): IModelConfig {\n  if (isResolvedModelConfig(modelConfig)) {\n    return modelConfig;\n  }\n  const rawModelConfig = Object.entries(modelConfig).reduce<\n    Record<string, string | number>\n  >((result, [key, value]) => {\n    if (typeof value === 'string' || typeof value === 'number') {\n      result[key] = value;\n    }\n    return result;\n  }, {});\n  return new ModelConfigManager(rawModelConfig).getModelConfig('default');\n}\n\nfunction clamp(value: number, min: number, max: number) {\n  return Math.min(Math.max(value, min), max);\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction isPendingDescription(value?: string) {\n  return value?.trim() === 'AI is analyzing element...';\n}\n\nfunction getRecorderEventScreenshot(event: MidsceneRecorderEvent) {\n  return (\n    event.screenshotWithBox || event.screenshotBefore || event.screenshotAfter\n  );\n}\n\nfunction getRecorderEventAfterScreenshot(event: MidsceneRecorderEvent) {\n  return event.screenshotAfter || event.screenshotWithBox;\n}\n\nfunction normalizeActionType(event: MidsceneRecorderEvent) {\n  return event.actionType?.trim();\n}\n\nfunction getPlatformId(target?: MidsceneRecorderTarget) {\n  return target?.platformId?.toLowerCase();\n}\n\nfunction getPlatformSurface(target?: MidsceneRecorderTarget) {\n  switch (getPlatformId(target)) {\n    case 'web':\n      return 'current web page';\n    case 'android':\n    case 'ios':\n    case 'harmony':\n      return 'current mobile screen';\n    case 'computer':\n      return 'current desktop screen';\n    default:\n      return 'current UI';\n  }\n}\n\nfunction getPlatformGuidance(target?: MidsceneRecorderTarget) {\n  switch (getPlatformId(target)) {\n    case 'web':\n      return 'For web targets, use web UI terms such as button, input, link, menu item, tab, dialog, aria-label, placeholder, and form section when visible or inferable.';\n    case 'android':\n    case 'ios':\n    case 'harmony':\n      return 'For mobile targets, use mobile UI terms such as tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, and screen section.';\n    case 'computer':\n      return 'For desktop/computer targets, use desktop UI terms such as menu item, toolbar button, dialog field, sidebar item, window control, file row, and application region.';\n    default:\n      return 'Use platform-neutral UI terms such as control, field, item, icon button, list item, region, panel, and page section.';\n  }\n}\n\nfunction getPointerActionVerb(event: MidsceneRecorderEvent) {\n  switch (normalizeActionType(event)) {\n    case 'Tap':\n      return 'Tap';\n    case 'DoubleClick':\n      return 'Double click';\n    case 'LongPress':\n      return 'Long press';\n    case 'RightClick':\n      return 'Right click';\n    default:\n      return 'Click';\n  }\n}\n\nfunction getDragActionVerb(event: MidsceneRecorderEvent) {\n  switch (normalizeActionType(event)) {\n    case 'Swipe':\n      return 'Swipe';\n    case 'DragAndDrop':\n      return 'Drag';\n    default:\n      return 'Drag';\n  }\n}\n\nfunction pointToRect(\n  x: number,\n  y: number,\n  size: number,\n  pageInfo: MidsceneRecorderPageInfo,\n): Rect {\n  const width = pageInfo.width || size;\n  const height = pageInfo.height || size;\n  const left = clamp(Math.floor(x - size / 2), 0, Math.max(width - 1, 0));\n  const top = clamp(Math.floor(y - size / 2), 0, Math.max(height - 1, 0));\n  return {\n    left,\n    top,\n    width: Math.min(size, Math.max(width - left, 1)),\n    height: Math.min(size, Math.max(height - top, 1)),\n  };\n}\n\nfunction getPointRectSize(event: MidsceneRecorderEvent) {\n  switch (event.type) {\n    case 'scroll':\n      return 96;\n    case 'drag':\n      return 64;\n    default:\n      return 36;\n  }\n}\n\nexport function getRecorderUIEventTargetRect(\n  event: MidsceneRecorderEvent,\n): Rect | null {\n  const rect = event.elementRect;\n  if (!rect) {\n    return null;\n  }\n\n  if (\n    isFiniteNumber(rect.width) &&\n    rect.width > 0 &&\n    isFiniteNumber(rect.height) &&\n    rect.height > 0 &&\n    (isFiniteNumber(rect.left) || isFiniteNumber(rect.top))\n  ) {\n    return {\n      left: rect.left || 0,\n      top: rect.top || 0,\n      width: rect.width,\n      height: rect.height,\n    };\n  }\n\n  if (isFiniteNumber(rect.x) && isFiniteNumber(rect.y)) {\n    return pointToRect(rect.x, rect.y, getPointRectSize(event), event.pageInfo);\n  }\n\n  return null;\n}\n\nfunction getFallbackDescription(\n  event: MidsceneRecorderEvent,\n  target?: MidsceneRecorderTarget,\n) {\n  const pageContext = getPageSemanticContext(event);\n  const surface = getPlatformSurface(target);\n\n  switch (event.type) {\n    case 'navigation':\n      return event.url || event.value || event.actionType || 'navigation';\n    case 'scroll':\n      return pageContext\n        ? `${pageContext} scrollable content`\n        : `scrollable content on the ${surface}`;\n    case 'drag':\n      return pageContext\n        ? `gesture area in ${pageContext}`\n        : `gesture area on the ${surface}`;\n    case 'input':\n      return `unresolved input field on the ${surface}`;\n    case 'keydown':\n      return pageContext\n        ? `focused control in ${pageContext}`\n        : `focused control on the ${surface}`;\n    default:\n      return pageContext\n        ? `control in ${pageContext}`\n        : `control on the ${surface}`;\n  }\n}\n\nfunction buildSemanticAction(\n  event: MidsceneRecorderEvent,\n  scrollDestinationDescription?: string,\n): MidsceneRecorderSemanticAction {\n  return {\n    type: event.type,\n    actionType: event.actionType,\n    value: event.value,\n    url: event.url,\n    scrollDestinationDescription,\n  };\n}\n\nfunction getFallbackReplayInstruction(\n  event: MidsceneRecorderEvent,\n  elementDescription: string,\n) {\n  return buildMidsceneRecorderReplayInstruction(\n    buildSemanticAction(event),\n    elementDescription,\n  );\n}\n\nfunction getFallbackActionSummary(\n  event: MidsceneRecorderEvent,\n  elementDescription: string,\n) {\n  return buildMidsceneRecorderActionSummary(\n    buildSemanticAction(event),\n    elementDescription,\n  );\n}\n\nfunction getActionGuidance(\n  event: MidsceneRecorderEvent,\n  target?: MidsceneRecorderTarget,\n) {\n  const platformGuidance = getPlatformGuidance(target);\n\n  switch (event.type) {\n    case 'click':\n      return `${platformGuidance} Identify the ${getPointerActionVerb(event).toLowerCase()} target by exact visible text first, then label/placeholder, then role plus stable surrounding context, then icon purpose, then visual position. Never describe it by coordinates, marker location, or as a nearby element.`;\n    case 'input':\n      return `${platformGuidance} Identify the exact input field at the marker before text entry. Use stable visible label, field role, field name, surrounding section, or sequence intent. Treat hint text that can change by user, time, data, or context as secondary evidence. Preserve the recorded input value only in replayInstruction; never describe the typed value or page title alone as the field.`;\n    case 'scroll':\n      return `${platformGuidance} Identify the scrollable page/region at the highlighted scroll point and concrete destination content revealed after scrolling. If multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu; do not generalize a panel/list scroll into the whole page. Use newly visible headings, section titles, list/table names, list items, or stable region labels; never say only \"more content\" or \"current page\".`;\n    case 'drag':\n      return `${platformGuidance} Identify the ${getDragActionVerb(event).toLowerCase()} start/end regions or the dragged UI control. Do not describe only the gesture path or coordinates.`;\n    case 'keydown':\n      return `${platformGuidance} Identify the focused element or keyboard target if visible, and preserve the recorded key in the replay instruction.`;\n    default:\n      return `${platformGuidance} Identify the UI target involved in this event using the most stable visible text or surrounding context.`;\n  }\n}\n\nfunction getEventRawCoordinates(event: MidsceneRecorderEvent) {\n  const x = event.elementRect?.x;\n  const y = event.elementRect?.y;\n  if (isFiniteNumber(x) && isFiniteNumber(y)) {\n    return { x, y };\n  }\n  return undefined;\n}\n\nfunction getPageSemanticContext(event: MidsceneRecorderEvent) {\n  const candidates = [event.title, event.url]\n    .map((item) => item?.trim())\n    .filter(Boolean) as string[];\n  return candidates[0];\n}\n\nfunction isWeakDescription(value?: string) {\n  if (!value) {\n    return true;\n  }\n  if (isPendingDescription(value)) {\n    return true;\n  }\n  const normalized = value.trim().toLowerCase();\n  const compact = normalized.replace(/\\s+/g, '');\n  return (\n    normalized.length === 0 ||\n    /^\\(?\\d+(?:\\.\\d+)?,\\s*\\d+(?:\\.\\d+)?\\)?$/.test(normalized) ||\n    normalized === 'target' ||\n    normalized === 'element' ||\n    normalized === 'target element' ||\n    normalized === 'the element' ||\n    normalized === 'page element' ||\n    normalized === 'input field' ||\n    normalized === 'text input' ||\n    normalized === 'text field' ||\n    normalized === 'search box' ||\n    normalized === 'more content' ||\n    normalized === 'the page' ||\n    normalized === 'current page' ||\n    normalized === 'current screen' ||\n    normalized === 'the screen' ||\n    normalized === 'current ui' ||\n    normalized === 'current visible ui' ||\n    normalized === 'current visible page' ||\n    normalized === 'current visible screen' ||\n    normalized === 'main area' ||\n    normalized === 'main scrollable area' ||\n    normalized === 'scrollable area' ||\n    normalized === 'highlighted element' ||\n    normalized === 'highlighted item' ||\n    normalized === 'marked element' ||\n    normalized === 'marked item' ||\n    normalized.includes('ai is analyzing element') ||\n    compact.includes('坐标') ||\n    compact.includes('附近') ||\n    compact.includes('附近的元素') ||\n    normalized.includes('coordinate') ||\n    normalized.includes('near the coordinate') ||\n    normalized.includes('near coordinates') ||\n    normalized.includes('nearby element') ||\n    normalized.includes('nearby item') ||\n    normalized.includes('near the marker') ||\n    normalized.includes('near marker') ||\n    normalized.includes('near the point') ||\n    normalized.includes('near point') ||\n    normalized.includes('at the point') ||\n    normalized.includes('button near point') ||\n    normalized.includes('shown in the screenshot') ||\n    normalized.includes('red rectangle') ||\n    normalized.includes('red marker') ||\n    normalized.includes('red box') ||\n    normalized.includes('highlighted element') ||\n    normalized.includes('highlighted item') ||\n    normalized.includes('highlighted screenshot')\n  );\n}\n\nfunction isWeakReplayInstruction(value?: string) {\n  if (!value) {\n    return true;\n  }\n  if (isPendingDescription(value)) {\n    return true;\n  }\n  const normalized = value.trim().toLowerCase();\n  const compact = normalized.replace(/\\s+/g, '');\n  return (\n    compact.includes('坐标') ||\n    compact.includes('附近') ||\n    normalized.includes('coordinate') ||\n    normalized.includes('near the coordinate') ||\n    normalized.includes('nearby element') ||\n    normalized.includes('nearby item') ||\n    normalized.includes('near the marker') ||\n    normalized.includes('near marker') ||\n    normalized.includes('near the point') ||\n    normalized.includes('near point') ||\n    normalized.includes('at the point') ||\n    normalized.includes('ai is analyzing element') ||\n    normalized.includes('more content') ||\n    normalized.includes('current page') ||\n    normalized.includes('current screen') ||\n    normalized.includes('highlighted element') ||\n    normalized.includes('highlighted item') ||\n    normalized.includes('red marker') ||\n    normalized.includes('red box') ||\n    normalized.includes('shown in the screenshot') ||\n    normalized.includes('highlighted screenshot')\n  );\n}\n\nfunction normalizeForComparison(value: string) {\n  return value.trim().toLowerCase().replace(/[\"'`]/g, '').replace(/\\s+/g, ' ');\n}\n\nfunction isInputValueUsedAsFieldDescription(\n  event: MidsceneRecorderEvent,\n  elementDescription?: string,\n) {\n  if (event.type !== 'input' || !event.value || !elementDescription) {\n    return false;\n  }\n\n  const typedValue = normalizeForComparison(event.value);\n  if (!typedValue) {\n    return false;\n  }\n  const description = normalizeForComparison(elementDescription);\n\n  return (\n    description === typedValue ||\n    description === `${typedValue} input` ||\n    description === `${typedValue} field` ||\n    description === `${typedValue} text field` ||\n    description === `input ${typedValue}` ||\n    description === `field ${typedValue}` ||\n    description.includes(`typed value ${typedValue}`) ||\n    description.includes(`value ${typedValue}`)\n  );\n}\n\nfunction hasScrollDestination(\n  replayInstruction: string,\n  scrollDestinationDescription?: string,\n) {\n  if (\n    scrollDestinationDescription &&\n    !isWeakDescription(scrollDestinationDescription)\n  ) {\n    return true;\n  }\n  const normalized = replayInstruction.toLowerCase();\n  return (\n    normalized.includes(' until ') ||\n    normalized.includes(' visible') ||\n    normalized.includes(' reveal') ||\n    normalized.includes(' to the ') ||\n    normalized.includes(' toward ')\n  );\n}\n\nasync function describeWithRetry(\n  event: MidsceneRecorderEvent,\n  target: MidsceneRecorderTarget | undefined,\n  highlightedScreenshot: string,\n  modelConfig: IModelConfig,\n  options: Required<\n    Pick<DescribeRecorderUIEventOptions, 'maxRetries' | 'retryDelayMs'>\n  >,\n) {\n  let lastError: unknown;\n  for (let attempt = 1; attempt <= options.maxRetries; attempt += 1) {\n    try {\n      const afterScreenshot = getRecorderEventAfterScreenshot(event);\n      const pageContext = getPageSemanticContext(event);\n      const platformGuidance = getPlatformGuidance(target);\n      const userContent: any[] = [\n        {\n          type: 'text',\n          text: `Recorder event:\n${JSON.stringify(\n  {\n    type: event.type,\n    actionType: event.actionType,\n    value: event.value,\n    rawCoordinates: getEventRawCoordinates(event),\n    url: event.url,\n    title: event.title,\n    pageContext,\n    pageInfo: event.pageInfo,\n    target,\n    platformGuidance,\n    guidance: getActionGuidance(event, target),\n  },\n  null,\n  2,\n)}\n\nThe target or region is highlighted in the screenshot below. Convert this event into semantic replay fields.`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: highlightedScreenshot,\n            detail: 'high',\n          },\n        },\n      ];\n      if (afterScreenshot) {\n        userContent.push(\n          {\n            type: 'text',\n            text: 'Screenshot after the recorded action, for context only:',\n          },\n          {\n            type: 'image_url',\n            image_url: {\n              url: afterScreenshot,\n              detail: 'high',\n            },\n          },\n        );\n      }\n      const response =\n        await callAIWithObjectResponse<RecorderUIEventAIResponse>(\n          [\n            {\n              role: 'system',\n              content: RECORDER_UI_DESCRIBER_SYSTEM_PROMPT,\n            },\n            {\n              role: 'user',\n              content: userContent,\n            },\n          ],\n          modelConfig,\n        );\n\n      const content = response.content;\n      if (content.error) {\n        throw new Error(content.error);\n      }\n      if (isWeakDescription(content.elementDescription)) {\n        throw new Error('AI returned a weak recorder event description.');\n      }\n      if (\n        isInputValueUsedAsFieldDescription(event, content.elementDescription)\n      ) {\n        throw new Error(\n          'AI used the recorded input value as the field description.',\n        );\n      }\n      const elementDescription = content.elementDescription!.trim();\n      const scrollDestinationDescription =\n        event.type === 'scroll'\n          ? content.scrollDestinationDescription?.trim()\n          : undefined;\n      if (\n        event.type === 'scroll' &&\n        !hasScrollDestination('', scrollDestinationDescription)\n      ) {\n        throw new Error(\n          'AI returned a scroll description without a destination.',\n        );\n      }\n      const aiReplayInstruction = content.replayInstruction?.trim();\n      if (aiReplayInstruction && isWeakReplayInstruction(aiReplayInstruction)) {\n        throw new Error('AI returned a weak recorder replay instruction.');\n      }\n      const semanticAction = buildSemanticAction(\n        event,\n        scrollDestinationDescription,\n      );\n      const replayInstruction = buildMidsceneRecorderReplayInstruction(\n        semanticAction,\n        elementDescription,\n      );\n      if (isWeakReplayInstruction(replayInstruction)) {\n        throw new Error('AI returned a weak recorder replay instruction.');\n      }\n      const actionSummary = buildMidsceneRecorderActionSummary(\n        semanticAction,\n        elementDescription,\n      );\n\n      return {\n        source: 'recorderAI' as const,\n        status: 'ready' as const,\n        elementDescription,\n        replayInstruction,\n        actionSummary,\n        confidence: content.confidence || 'medium',\n      };\n    } catch (error) {\n      lastError = error;\n      if (attempt < options.maxRetries) {\n        await delay(options.retryDelayMs);\n      }\n    }\n  }\n  throw lastError;\n}\n\nasync function createScreenshotWithBox(\n  event: MidsceneRecorderEvent,\n  rect: Rect,\n) {\n  if (event.screenshotWithBox) {\n    return event.screenshotWithBox;\n  }\n  const screenshot = getRecorderEventScreenshot(event);\n  if (!screenshot) {\n    return undefined;\n  }\n  const { compositeElementInfoImg } = await import(\n    /* @vite-ignore */ '@midscene/shared/img'\n  );\n  return compositeElementInfoImg({\n    inputImgBase64: screenshot,\n    size: event.pageInfo,\n    elementsPositionInfo: [{ rect }],\n    borderThickness: 3,\n    annotationPadding: 2,\n  });\n}\n\nfunction createFallbackEvent(\n  event: MidsceneRecorderEvent,\n  error: string,\n  screenshotWithBox?: string,\n  target?: MidsceneRecorderTarget,\n): MidsceneRecorderEvent {\n  const semantic = getMidsceneRecorderSemantic(event);\n  const elementDescription =\n    semantic?.elementDescription &&\n    !isWeakDescription(semantic.elementDescription)\n      ? semantic.elementDescription\n      : getFallbackDescription(event, target);\n  return {\n    ...event,\n    semantic: {\n      source: 'heuristic',\n      status: 'ready',\n      elementDescription,\n      replayInstruction: getFallbackReplayInstruction(\n        event,\n        elementDescription,\n      ),\n      actionSummary: getFallbackActionSummary(event, elementDescription),\n      confidence: 'low',\n      error,\n    },\n    screenshotWithBox: screenshotWithBox || event.screenshotWithBox,\n  };\n}\n\nexport async function describeRecorderUIEvent(\n  input: DescribeRecorderUIEventInput,\n  modelConfig: IModelConfig | Record<string, unknown>,\n  options: DescribeRecorderUIEventOptions = {},\n): Promise<DescribeRecorderUIEventResult> {\n  const event = input.event;\n  const rect = getRecorderUIEventTargetRect(event);\n  const screenshot = getRecorderEventScreenshot(event);\n\n  if (!rect || !screenshot) {\n    const error = !rect\n      ? 'Recorder event has no target rectangle.'\n      : 'Recorder event has no screenshot.';\n    return {\n      usedFallback: true,\n      event: createFallbackEvent(event, error, undefined, input.target),\n    };\n  }\n\n  let screenshotWithBox: string | undefined;\n  try {\n    const resolvedModelConfig = resolveRecorderModelConfig(modelConfig);\n    screenshotWithBox = await createScreenshotWithBox(event, rect);\n    const semanticFields = await describeWithRetry(\n      event,\n      input.target,\n      screenshotWithBox || screenshot,\n      resolvedModelConfig,\n      {\n        maxRetries: options.maxRetries ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRIES,\n        retryDelayMs:\n          options.retryDelayMs ?? RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS,\n      },\n    );\n    return {\n      usedFallback: false,\n      event: {\n        ...event,\n        semantic: semanticFields,\n        screenshotWithBox: screenshotWithBox || event.screenshotWithBox,\n      },\n    };\n  } catch (error) {\n    const message = error instanceof Error ? error.message : String(error);\n    return {\n      usedFallback: true,\n      error: message,\n      event: createFallbackEvent(\n        event,\n        message,\n        screenshotWithBox,\n        input.target,\n      ),\n    };\n  }\n}\n\nexport async function describeRecorderUIEvents(\n  inputs: DescribeRecorderUIEventInput[],\n  modelConfig: IModelConfig | Record<string, unknown>,\n  options: DescribeRecorderUIEventOptions = {},\n): Promise<DescribeRecorderUIEventResult[]> {\n  const concurrency = Math.max(\n    1,\n    options.concurrency ?? RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY,\n  );\n  const results: DescribeRecorderUIEventResult[] = new Array(inputs.length);\n  let cursor = 0;\n\n  async function worker() {\n    while (cursor < inputs.length) {\n      const index = cursor;\n      cursor += 1;\n      results[index] = await describeRecorderUIEvent(\n        inputs[index],\n        modelConfig,\n        options,\n      );\n    }\n  }\n\n  await Promise.all(\n    Array.from({ length: Math.min(concurrency, inputs.length) }, () =>\n      worker(),\n    ),\n  );\n  return results;\n}\n"],"names":["RECORDER_UI_DESCRIBER_DEFAULT_RETRIES","RECORDER_UI_DESCRIBER_DEFAULT_RETRY_DELAY_MS","RECORDER_UI_DESCRIBER_DEFAULT_CONCURRENCY","delay","ms","Promise","resolve","setTimeout","isResolvedModelConfig","modelConfig","resolveRecorderModelConfig","rawModelConfig","Object","result","key","value","ModelConfigManager","clamp","min","max","Math","isFiniteNumber","Number","isPendingDescription","getRecorderEventScreenshot","event","getRecorderEventAfterScreenshot","normalizeActionType","getPlatformId","target","getPlatformSurface","getPlatformGuidance","getPointerActionVerb","getDragActionVerb","pointToRect","x","y","size","pageInfo","width","height","left","top","getPointRectSize","getRecorderUIEventTargetRect","rect","getFallbackDescription","pageContext","getPageSemanticContext","surface","buildSemanticAction","scrollDestinationDescription","getFallbackReplayInstruction","elementDescription","buildMidsceneRecorderReplayInstruction","getFallbackActionSummary","buildMidsceneRecorderActionSummary","getActionGuidance","platformGuidance","getEventRawCoordinates","candidates","item","Boolean","isWeakDescription","normalized","compact","isWeakReplayInstruction","normalizeForComparison","isInputValueUsedAsFieldDescription","typedValue","description","hasScrollDestination","replayInstruction","describeWithRetry","highlightedScreenshot","options","lastError","attempt","afterScreenshot","userContent","JSON","response","callAIWithObjectResponse","RECORDER_UI_DESCRIBER_SYSTEM_PROMPT","content","Error","undefined","aiReplayInstruction","semanticAction","actionSummary","error","createScreenshotWithBox","screenshot","compositeElementInfoImg","createFallbackEvent","screenshotWithBox","semantic","getMidsceneRecorderSemantic","describeRecorderUIEvent","input","resolvedModelConfig","semanticFields","message","String","describeRecorderUIEvents","inputs","concurrency","results","Array","cursor","worker","index"],"mappings":";;;;AA0CA,MAAMA,wCAAwC;AAC9C,MAAMC,+CAA+C;AACrD,MAAMC,4CAA4C;AAElD,SAASC,MAAMC,EAAU;IACvB,OAAO,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;AACtD;AAEA,SAASI,sBACPC,WAAmD;IAEnD,OACE,AAAiC,YAAjC,OAAOA,YAAY,SAAS,IAC3B,CAAwC,YAAxC,OAAOA,YAAY,gBAAgB,IAClC,AAA8B,YAA9B,OAAOA,YAAY,MAAM,IACzB,AAA4B,YAA5B,OAAOA,YAAY,IAAI,IACvB,AAAoC,YAApC,OAAOA,YAAY,YAAY,AAAY;AAEjD;AAEA,SAASC,2BACPD,WAAmD;IAEnD,IAAID,sBAAsBC,cACxB,OAAOA;IAET,MAAME,iBAAiBC,OAAO,OAAO,CAACH,aAAa,MAAM,CAEvD,CAACI,QAAQ,CAACC,KAAKC,MAAM;QACrB,IAAI,AAAiB,YAAjB,OAAOA,SAAsB,AAAiB,YAAjB,OAAOA,OACtCF,MAAM,CAACC,IAAI,GAAGC;QAEhB,OAAOF;IACT,GAAG,CAAC;IACJ,OAAO,IAAIG,mBAAmBL,gBAAgB,cAAc,CAAC;AAC/D;AAEA,SAASM,MAAMF,KAAa,EAAEG,GAAW,EAAEC,GAAW;IACpD,OAAOC,KAAK,GAAG,CAACA,KAAK,GAAG,CAACL,OAAOG,MAAMC;AACxC;AAEA,SAASE,eAAeN,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBO,OAAO,QAAQ,CAACP;AACtD;AAEA,SAASQ,qBAAqBR,KAAc;IAC1C,OAAOA,OAAO,WAAW;AAC3B;AAEA,SAASS,2BAA2BC,KAA4B;IAC9D,OACEA,MAAM,iBAAiB,IAAIA,MAAM,gBAAgB,IAAIA,MAAM,eAAe;AAE9E;AAEA,SAASC,gCAAgCD,KAA4B;IACnE,OAAOA,MAAM,eAAe,IAAIA,MAAM,iBAAiB;AACzD;AAEA,SAASE,oBAAoBF,KAA4B;IACvD,OAAOA,MAAM,UAAU,EAAE;AAC3B;AAEA,SAASG,cAAcC,MAA+B;IACpD,OAAOA,QAAQ,YAAY;AAC7B;AAEA,SAASC,mBAAmBD,MAA+B;IACzD,OAAQD,cAAcC;QACpB,KAAK;YACH,OAAO;QACT,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASE,oBAAoBF,MAA+B;IAC1D,OAAQD,cAAcC;QACpB,KAAK;YACH,OAAO;QACT,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASG,qBAAqBP,KAA4B;IACxD,OAAQE,oBAAoBF;QAC1B,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASQ,kBAAkBR,KAA4B;IACrD,OAAQE,oBAAoBF;QAC1B,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEA,SAASS,YACPC,CAAS,EACTC,CAAS,EACTC,IAAY,EACZC,QAAkC;IAElC,MAAMC,QAAQD,SAAS,KAAK,IAAID;IAChC,MAAMG,SAASF,SAAS,MAAM,IAAID;IAClC,MAAMI,OAAOxB,MAAMG,KAAK,KAAK,CAACe,IAAIE,OAAO,IAAI,GAAGjB,KAAK,GAAG,CAACmB,QAAQ,GAAG;IACpE,MAAMG,MAAMzB,MAAMG,KAAK,KAAK,CAACgB,IAAIC,OAAO,IAAI,GAAGjB,KAAK,GAAG,CAACoB,SAAS,GAAG;IACpE,OAAO;QACLC;QACAC;QACA,OAAOtB,KAAK,GAAG,CAACiB,MAAMjB,KAAK,GAAG,CAACmB,QAAQE,MAAM;QAC7C,QAAQrB,KAAK,GAAG,CAACiB,MAAMjB,KAAK,GAAG,CAACoB,SAASE,KAAK;IAChD;AACF;AAEA,SAASC,iBAAiBlB,KAA4B;IACpD,OAAQA,MAAM,IAAI;QAChB,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT;YACE,OAAO;IACX;AACF;AAEO,SAASmB,6BACdnB,KAA4B;IAE5B,MAAMoB,OAAOpB,MAAM,WAAW;IAC9B,IAAI,CAACoB,MACH,OAAO;IAGT,IACExB,eAAewB,KAAK,KAAK,KACzBA,KAAK,KAAK,GAAG,KACbxB,eAAewB,KAAK,MAAM,KAC1BA,KAAK,MAAM,GAAG,KACbxB,CAAAA,eAAewB,KAAK,IAAI,KAAKxB,eAAewB,KAAK,GAAG,IAErD,OAAO;QACL,MAAMA,KAAK,IAAI,IAAI;QACnB,KAAKA,KAAK,GAAG,IAAI;QACjB,OAAOA,KAAK,KAAK;QACjB,QAAQA,KAAK,MAAM;IACrB;IAGF,IAAIxB,eAAewB,KAAK,CAAC,KAAKxB,eAAewB,KAAK,CAAC,GACjD,OAAOX,YAAYW,KAAK,CAAC,EAAEA,KAAK,CAAC,EAAEF,iBAAiBlB,QAAQA,MAAM,QAAQ;IAG5E,OAAO;AACT;AAEA,SAASqB,uBACPrB,KAA4B,EAC5BI,MAA+B;IAE/B,MAAMkB,cAAcC,uBAAuBvB;IAC3C,MAAMwB,UAAUnB,mBAAmBD;IAEnC,OAAQJ,MAAM,IAAI;QAChB,KAAK;YACH,OAAOA,MAAM,GAAG,IAAIA,MAAM,KAAK,IAAIA,MAAM,UAAU,IAAI;QACzD,KAAK;YACH,OAAOsB,cACH,GAAGA,YAAY,mBAAmB,CAAC,GACnC,CAAC,0BAA0B,EAAEE,SAAS;QAC5C,KAAK;YACH,OAAOF,cACH,CAAC,gBAAgB,EAAEA,aAAa,GAChC,CAAC,oBAAoB,EAAEE,SAAS;QACtC,KAAK;YACH,OAAO,CAAC,8BAA8B,EAAEA,SAAS;QACnD,KAAK;YACH,OAAOF,cACH,CAAC,mBAAmB,EAAEA,aAAa,GACnC,CAAC,uBAAuB,EAAEE,SAAS;QACzC;YACE,OAAOF,cACH,CAAC,WAAW,EAAEA,aAAa,GAC3B,CAAC,eAAe,EAAEE,SAAS;IACnC;AACF;AAEA,SAASC,oBACPzB,KAA4B,EAC5B0B,4BAAqC;IAErC,OAAO;QACL,MAAM1B,MAAM,IAAI;QAChB,YAAYA,MAAM,UAAU;QAC5B,OAAOA,MAAM,KAAK;QAClB,KAAKA,MAAM,GAAG;QACd0B;IACF;AACF;AAEA,SAASC,6BACP3B,KAA4B,EAC5B4B,kBAA0B;IAE1B,OAAOC,uCACLJ,oBAAoBzB,QACpB4B;AAEJ;AAEA,SAASE,yBACP9B,KAA4B,EAC5B4B,kBAA0B;IAE1B,OAAOG,mCACLN,oBAAoBzB,QACpB4B;AAEJ;AAEA,SAASI,kBACPhC,KAA4B,EAC5BI,MAA+B;IAE/B,MAAM6B,mBAAmB3B,oBAAoBF;IAE7C,OAAQJ,MAAM,IAAI;QAChB,KAAK;YACH,OAAO,GAAGiC,iBAAiB,cAAc,EAAE1B,qBAAqBP,OAAO,WAAW,GAAG,2NAA2N,CAAC;QACnT,KAAK;YACH,OAAO,GAAGiC,iBAAiB,gXAAgX,CAAC;QAC9Y,KAAK;YACH,OAAO,GAAGA,iBAAiB,yhBAAyhB,CAAC;QACvjB,KAAK;YACH,OAAO,GAAGA,iBAAiB,cAAc,EAAEzB,kBAAkBR,OAAO,WAAW,GAAG,mGAAmG,CAAC;QACxL,KAAK;YACH,OAAO,GAAGiC,iBAAiB,qHAAqH,CAAC;QACnJ;YACE,OAAO,GAAGA,iBAAiB,yGAAyG,CAAC;IACzI;AACF;AAEA,SAASC,uBAAuBlC,KAA4B;IAC1D,MAAMU,IAAIV,MAAM,WAAW,EAAE;IAC7B,MAAMW,IAAIX,MAAM,WAAW,EAAE;IAC7B,IAAIJ,eAAec,MAAMd,eAAee,IACtC,OAAO;QAAED;QAAGC;IAAE;AAGlB;AAEA,SAASY,uBAAuBvB,KAA4B;IAC1D,MAAMmC,aAAa;QAACnC,MAAM,KAAK;QAAEA,MAAM,GAAG;KAAC,CACxC,GAAG,CAAC,CAACoC,OAASA,MAAM,QACpB,MAAM,CAACC;IACV,OAAOF,UAAU,CAAC,EAAE;AACtB;AAEA,SAASG,kBAAkBhD,KAAc;IACvC,IAAI,CAACA,OACH,OAAO;IAET,IAAIQ,qBAAqBR,QACvB,OAAO;IAET,MAAMiD,aAAajD,MAAM,IAAI,GAAG,WAAW;IAC3C,MAAMkD,UAAUD,WAAW,OAAO,CAAC,QAAQ;IAC3C,OACEA,AAAsB,MAAtBA,WAAW,MAAM,IACjB,yCAAyC,IAAI,CAACA,eAC9CA,AAAe,aAAfA,cACAA,AAAe,cAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,kBAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,kBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,eAAfA,cACAA,AAAe,mBAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,iBAAfA,cACAA,AAAe,yBAAfA,cACAA,AAAe,2BAAfA,cACAA,AAAe,6BAAfA,cACAA,AAAe,gBAAfA,cACAA,AAAe,2BAAfA,cACAA,AAAe,sBAAfA,cACAA,AAAe,0BAAfA,cACAA,AAAe,uBAAfA,cACAA,AAAe,qBAAfA,cACAA,AAAe,kBAAfA,cACAA,WAAW,QAAQ,CAAC,8BACpBC,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,YACjBD,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,sBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,wBACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC,oBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,SAASE,wBAAwBnD,KAAc;IAC7C,IAAI,CAACA,OACH,OAAO;IAET,IAAIQ,qBAAqBR,QACvB,OAAO;IAET,MAAMiD,aAAajD,MAAM,IAAI,GAAG,WAAW;IAC3C,MAAMkD,UAAUD,WAAW,OAAO,CAAC,QAAQ;IAC3C,OACEC,QAAQ,QAAQ,CAAC,SACjBA,QAAQ,QAAQ,CAAC,SACjBD,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,sBACpBA,WAAW,QAAQ,CAAC,kBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,mBACpBA,WAAW,QAAQ,CAAC,qBACpBA,WAAW,QAAQ,CAAC,0BACpBA,WAAW,QAAQ,CAAC,uBACpBA,WAAW,QAAQ,CAAC,iBACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,8BACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,SAASG,uBAAuBpD,KAAa;IAC3C,OAAOA,MAAM,IAAI,GAAG,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,QAAQ;AAC1E;AAEA,SAASqD,mCACP3C,KAA4B,EAC5B4B,kBAA2B;IAE3B,IAAI5B,AAAe,YAAfA,MAAM,IAAI,IAAgB,CAACA,MAAM,KAAK,IAAI,CAAC4B,oBAC7C,OAAO;IAGT,MAAMgB,aAAaF,uBAAuB1C,MAAM,KAAK;IACrD,IAAI,CAAC4C,YACH,OAAO;IAET,MAAMC,cAAcH,uBAAuBd;IAE3C,OACEiB,gBAAgBD,cAChBC,gBAAgB,GAAGD,WAAW,MAAM,CAAC,IACrCC,gBAAgB,GAAGD,WAAW,MAAM,CAAC,IACrCC,gBAAgB,GAAGD,WAAW,WAAW,CAAC,IAC1CC,gBAAgB,CAAC,MAAM,EAAED,YAAY,IACrCC,gBAAgB,CAAC,MAAM,EAAED,YAAY,IACrCC,YAAY,QAAQ,CAAC,CAAC,YAAY,EAAED,YAAY,KAChDC,YAAY,QAAQ,CAAC,CAAC,MAAM,EAAED,YAAY;AAE9C;AAEA,SAASE,qBACPC,iBAAyB,EACzBrB,4BAAqC;IAErC,IACEA,gCACA,CAACY,kBAAkBZ,+BAEnB,OAAO;IAET,MAAMa,aAAaQ,kBAAkB,WAAW;IAChD,OACER,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,eACpBA,WAAW,QAAQ,CAAC,cACpBA,WAAW,QAAQ,CAAC,eACpBA,WAAW,QAAQ,CAAC;AAExB;AAEA,eAAeS,kBACbhD,KAA4B,EAC5BI,MAA0C,EAC1C6C,qBAA6B,EAC7BjE,WAAyB,EACzBkE,OAEC;IAED,IAAIC;IACJ,IAAK,IAAIC,UAAU,GAAGA,WAAWF,QAAQ,UAAU,EAAEE,WAAW,EAC9D,IAAI;QACF,MAAMC,kBAAkBpD,gCAAgCD;QACxD,MAAMsB,cAAcC,uBAAuBvB;QAC3C,MAAMiC,mBAAmB3B,oBAAoBF;QAC7C,MAAMkD,cAAqB;YACzB;gBACE,MAAM;gBACN,MAAM,CAAC;AACjB,EAAEC,KAAK,SAAS,CACd;oBACE,MAAMvD,MAAM,IAAI;oBAChB,YAAYA,MAAM,UAAU;oBAC5B,OAAOA,MAAM,KAAK;oBAClB,gBAAgBkC,uBAAuBlC;oBACvC,KAAKA,MAAM,GAAG;oBACd,OAAOA,MAAM,KAAK;oBAClBsB;oBACA,UAAUtB,MAAM,QAAQ;oBACxBI;oBACA6B;oBACA,UAAUD,kBAAkBhC,OAAOI;gBACrC,GACA,MACA,GACA;;4GAE0G,CAAC;YACrG;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAK6C;oBACL,QAAQ;gBACV;YACF;SACD;QACD,IAAII,iBACFC,YAAY,IAAI,CACd;YACE,MAAM;YACN,MAAM;QACR,GACA;YACE,MAAM;YACN,WAAW;gBACT,KAAKD;gBACL,QAAQ;YACV;QACF;QAGJ,MAAMG,WACJ,MAAMC,yBACJ;YACE;gBACE,MAAM;gBACN,SAASC;YACX;YACA;gBACE,MAAM;gBACN,SAASJ;YACX;SACD,EACDtE;QAGJ,MAAM2E,UAAUH,SAAS,OAAO;QAChC,IAAIG,QAAQ,KAAK,EACf,MAAM,IAAIC,MAAMD,QAAQ,KAAK;QAE/B,IAAIrB,kBAAkBqB,QAAQ,kBAAkB,GAC9C,MAAM,IAAIC,MAAM;QAElB,IACEjB,mCAAmC3C,OAAO2D,QAAQ,kBAAkB,GAEpE,MAAM,IAAIC,MACR;QAGJ,MAAMhC,qBAAqB+B,QAAQ,kBAAkB,CAAE,IAAI;QAC3D,MAAMjC,+BACJ1B,AAAe,aAAfA,MAAM,IAAI,GACN2D,QAAQ,4BAA4B,EAAE,SACtCE;QACN,IACE7D,AAAe,aAAfA,MAAM,IAAI,IACV,CAAC8C,qBAAqB,IAAIpB,+BAE1B,MAAM,IAAIkC,MACR;QAGJ,MAAME,sBAAsBH,QAAQ,iBAAiB,EAAE;QACvD,IAAIG,uBAAuBrB,wBAAwBqB,sBACjD,MAAM,IAAIF,MAAM;QAElB,MAAMG,iBAAiBtC,oBACrBzB,OACA0B;QAEF,MAAMqB,oBAAoBlB,uCACxBkC,gBACAnC;QAEF,IAAIa,wBAAwBM,oBAC1B,MAAM,IAAIa,MAAM;QAElB,MAAMI,gBAAgBjC,mCACpBgC,gBACAnC;QAGF,OAAO;YACL,QAAQ;YACR,QAAQ;YACRA;YACAmB;YACAiB;YACA,YAAYL,QAAQ,UAAU,IAAI;QACpC;IACF,EAAE,OAAOM,OAAO;QACdd,YAAYc;QACZ,IAAIb,UAAUF,QAAQ,UAAU,EAC9B,MAAMxE,MAAMwE,QAAQ,YAAY;IAEpC;IAEF,MAAMC;AACR;AAEA,eAAee,wBACblE,KAA4B,EAC5BoB,IAAU;IAEV,IAAIpB,MAAM,iBAAiB,EACzB,OAAOA,MAAM,iBAAiB;IAEhC,MAAMmE,aAAapE,2BAA2BC;IAC9C,IAAI,CAACmE,YACH;IAEF,MAAM,EAAEC,uBAAuB,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAP;IAG1C,OAAOA,wBAAwB;QAC7B,gBAAgBD;QAChB,MAAMnE,MAAM,QAAQ;QACpB,sBAAsB;YAAC;gBAAEoB;YAAK;SAAE;QAChC,iBAAiB;QACjB,mBAAmB;IACrB;AACF;AAEA,SAASiD,oBACPrE,KAA4B,EAC5BiE,KAAa,EACbK,iBAA0B,EAC1BlE,MAA+B;IAE/B,MAAMmE,WAAWC,4BAA4BxE;IAC7C,MAAM4B,qBACJ2C,UAAU,sBACV,CAACjC,kBAAkBiC,SAAS,kBAAkB,IAC1CA,SAAS,kBAAkB,GAC3BlD,uBAAuBrB,OAAOI;IACpC,OAAO;QACL,GAAGJ,KAAK;QACR,UAAU;YACR,QAAQ;YACR,QAAQ;YACR4B;YACA,mBAAmBD,6BACjB3B,OACA4B;YAEF,eAAeE,yBAAyB9B,OAAO4B;YAC/C,YAAY;YACZqC;QACF;QACA,mBAAmBK,qBAAqBtE,MAAM,iBAAiB;IACjE;AACF;AAEO,eAAeyE,wBACpBC,KAAmC,EACnC1F,WAAmD,EACnDkE,UAA0C,CAAC,CAAC;IAE5C,MAAMlD,QAAQ0E,MAAM,KAAK;IACzB,MAAMtD,OAAOD,6BAA6BnB;IAC1C,MAAMmE,aAAapE,2BAA2BC;IAE9C,IAAI,CAACoB,QAAQ,CAAC+C,YAAY;QACxB,MAAMF,QAAQ,AAAC7C,OAEX,sCADA;QAEJ,OAAO;YACL,cAAc;YACd,OAAOiD,oBAAoBrE,OAAOiE,OAAOJ,QAAWa,MAAM,MAAM;QAClE;IACF;IAEA,IAAIJ;IACJ,IAAI;QACF,MAAMK,sBAAsB1F,2BAA2BD;QACvDsF,oBAAoB,MAAMJ,wBAAwBlE,OAAOoB;QACzD,MAAMwD,iBAAiB,MAAM5B,kBAC3BhD,OACA0E,MAAM,MAAM,EACZJ,qBAAqBH,YACrBQ,qBACA;YACE,YAAYzB,QAAQ,UAAU,IAAI3E;YAClC,cACE2E,QAAQ,YAAY,IAAI1E;QAC5B;QAEF,OAAO;YACL,cAAc;YACd,OAAO;gBACL,GAAGwB,KAAK;gBACR,UAAU4E;gBACV,mBAAmBN,qBAAqBtE,MAAM,iBAAiB;YACjE;QACF;IACF,EAAE,OAAOiE,OAAO;QACd,MAAMY,UAAUZ,iBAAiBL,QAAQK,MAAM,OAAO,GAAGa,OAAOb;QAChE,OAAO;YACL,cAAc;YACd,OAAOY;YACP,OAAOR,oBACLrE,OACA6E,SACAP,mBACAI,MAAM,MAAM;QAEhB;IACF;AACF;AAEO,eAAeK,yBACpBC,MAAsC,EACtChG,WAAmD,EACnDkE,UAA0C,CAAC,CAAC;IAE5C,MAAM+B,cAActF,KAAK,GAAG,CAC1B,GACAuD,QAAQ,WAAW,IAAIzE;IAEzB,MAAMyG,UAA2C,IAAIC,MAAMH,OAAO,MAAM;IACxE,IAAII,SAAS;IAEb,eAAeC;QACb,MAAOD,SAASJ,OAAO,MAAM,CAAE;YAC7B,MAAMM,QAAQF;YACdA,UAAU;YACVF,OAAO,CAACI,MAAM,GAAG,MAAMb,wBACrBO,MAAM,CAACM,MAAM,EACbtG,aACAkE;QAEJ;IACF;IAEA,MAAMtE,QAAQ,GAAG,CACfuG,MAAM,IAAI,CAAC;QAAE,QAAQxF,KAAK,GAAG,CAACsF,aAAaD,OAAO,MAAM;IAAE,GAAG,IAC3DK;IAGJ,OAAOH;AACT"}

package/dist/es/server.mjs

@@ -2,7 +2,8 @@ import { createHash, randomUUID } from "node:crypto";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join, resolve as external_node_path_resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
-import { ReportActionDump, runConnectivityTest } from "@midscene/core";
+import { ReportActionDump, describeElementAtPoint, runConnectivityTest } from "@midscene/core";
+import { getModelRuntime } from "@midscene/core/ai-model";
 import { getTmpDir, sleep } from "@midscene/core/utils";
 import { getMidsceneRunSubDir } from "@midscene/shared/common";
 import { PLAYGROUND_SERVER_PORT } from "@midscene/shared/constants";
@@ -32,6 +33,15 @@ const defaultPort = PLAYGROUND_SERVER_PORT;
 const RECORDER_CAPTURE_AFTER_INTERACT_DELAY_MS = 250;
 const RECORDER_AI_DESCRIBE_AFTER_INTERACT_TIMEOUT_MS = 30000;
 const RECORDER_AI_DESCRIBE_SCREENSHOT_DUMP_DIR = 'recorder-ai-describe-screenshots';
+function createElementDescriberRuntime(agent) {
+    const modelConfigManager = agent.modelConfigManager;
+    const missingModelRuntime = void 0;
+    return {
+        service: agent.service,
+        describeModelRuntime: modelConfigManager ? getModelRuntime(modelConfigManager.getModelConfig('insight')) : missingModelRuntime,
+        locateModelRuntime: modelConfigManager ? getModelRuntime(modelConfigManager.getModelConfig('default')) : missingModelRuntime
+    };
+}
 function extractBase64Payload(value) {
     const dataUrlMatch = value.match(/^data:([^;,]+)?(?:;[^,]*)?,([\s\S]*)$/);
     if (dataUrlMatch) return {
@@ -240,6 +250,17 @@ function createRecorderAiDescribeTraceBase(event, eventScreenshot) {
         screenshotBytes: estimateBase64Bytes(eventScreenshot)
     };
 }
+function resolvePlaygroundListenHost() {
+    return process.env.MIDSCENE_PLAYGROUND_HOST?.trim() || '0.0.0.0';
+}
+function resolvePlaygroundBrowserHost() {
+    const listenHost = resolvePlaygroundListenHost();
+    return '0.0.0.0' === listenHost || '::' === listenHost ? '127.0.0.1' : listenHost;
+}
+function buildPlaygroundBrowserUrl(host, port) {
+    const normalizedHost = host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;
+    return `http://${normalizedHost}:${port}`;
+}
 function serializeAiConfigSignature(aiConfig) {
     return JSON.stringify(Object.entries(aiConfig).sort(([leftKey], [rightKey])=>leftKey.localeCompare(rightKey)));
 }
@@ -954,7 +975,7 @@ class PlaygroundServer {
             let screenshotAnnotation;
             let screenshotPersistError;
             let annotatedScreenshotPersistError;
-            if ('failed' === status) try {
+            if ('failed' === status || extra.verifyResult?.pass === false) try {
                 const screenshotDump = await persistRecorderAiDescribeScreenshot(event, eventScreenshot, extra.verifyResult);
                 screenshotRef = screenshotDump?.screenshotRef;
                 annotatedScreenshotRef = screenshotDump?.annotatedScreenshotRef;
@@ -990,7 +1011,7 @@ class PlaygroundServer {
                 trace
             };
         }
-        if ('function' != typeof agent?.describeElementAtPoint) {
+        if (!agent) {
             const error = 'Active agent does not support describeElementAtPoint.';
             const trace = await finishTrace('failed', {
                 error
@@ -1017,7 +1038,8 @@ class PlaygroundServer {
             if (!event.pageInfo?.width || !event.pageInfo?.height) throw new Error('Skipped aiDescribe because the recorder event has no pageInfo for coordinate mapping.');
             const verifyPrompt = shouldVerifyRecorderAiDescribeEvent(event);
             modelCallStartedAt = Date.now();
-            const describeResult = await withTimeout(agent.describeElementAtPoint([
+            const elementDescriber = createElementDescriberRuntime(agent);
+            const describeResult = await withTimeout(describeElementAtPoint(elementDescriber, [
                 x,
                 y
             ], {
@@ -1036,7 +1058,7 @@ class PlaygroundServer {
             deepLocate = describeResult.deepLocate;
             verifyResult = verifyPrompt ? describeResult.verifyResult : void 0;
             if (!elementDescription) throw new Error("aiDescribe returned an empty element description.");
-            if (verifyResult && false === verifyResult.pass) throw new Error('aiDescribe verification failed.');
+            if (false === describeResult.success) throw new Error(describeResult.error || `aiDescribe ${describeResult.failureStage || 'unknown'} failed.`);
             const semanticAction = buildRecorderSemanticAction(event.actionType || event.type, event.rawPayload || {}, event.url);
             const trace = await finishTrace('ready', {
                 modelCallDurationMs,
@@ -1059,7 +1081,8 @@ class PlaygroundServer {
                                 x,
                                 y
                             ],
-                            actualCenter: verifyResult?.center
+                            actualCenter: verifyResult?.center,
+                            annotatedScreenshotPath: trace.annotatedScreenshotRef?.path
                         }
                     })
                 },
@@ -2023,7 +2046,8 @@ class PlaygroundServer {
         this.port = port || defaultPort;
         return new Promise((resolve)=>{
             const serverPort = this.port ?? defaultPort;
-            this.server = this._app.listen(serverPort, '0.0.0.0', ()=>{
+            const listenHost = resolvePlaygroundListenHost();
+            this.server = this._app.listen(serverPort, listenHost, ()=>{
                 resolve(this);
             });
         });
@@ -2100,6 +2124,6 @@ class PlaygroundServer {
     }
 }
 const server = PlaygroundServer;
-export { InteractParamsValidationError, PlaygroundServer, buildInteractParams, createManualExecutorContext, server as default, serializeZodField };
+export { InteractParamsValidationError, PlaygroundServer, buildInteractParams, buildPlaygroundBrowserUrl, createManualExecutorContext, server as default, resolvePlaygroundBrowserHost, resolvePlaygroundListenHost, serializeZodField };
 
 //# sourceMappingURL=server.mjs.map