@midscene/core 1.10.6-beta-20260716090839.0 → 1.10.6

package/dist/es/agent/utils.mjs

@@ -181,7 +181,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.10.6-beta-20260716090839.0";
+const getMidsceneVersion = ()=>"1.10.6";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/inspect.mjs

@@ -146,7 +146,8 @@ async function genericLocate(_elementDescription, options, locateRequest) {
             callAi: ()=>callAIWithObjectResponse(msgs, modelRuntime, {
                     abortSignal: options.abortSignal,
                     jsonParserSource: 'locate',
-                    retryTimes: 1
+                    retryTimes: modelRuntime.config.retryCount,
+                    retryInterval: modelRuntime.config.retryInterval
                 }),
             parseResponse: (response)=>{
                 const rawResponse = response.contentString;
@@ -177,7 +178,8 @@ async function genericLocate(_elementDescription, options, locateRequest) {
                 const message = modelErrors && modelErrors.length > 0 ? `${modelErrors.join('\n')} (${parseErrorMessage})` : parseErrorMessage;
                 return new AIResponseParseError(message, response.contentString, response.usage, response.rawChoiceMessage, response.reasoning_content);
             },
-            parseRetryTimes: 1,
+            parseRetryTimes: modelRuntime.config.retryCount,
+            parseRetryInterval: modelRuntime.config.retryInterval,
             abortSignal: options.abortSignal,
             onParseRetry: (error)=>{
                 debugInspect('retrying locate after coordinate parsing failed: %s', error instanceof Error ? error.message : String(error));
@@ -249,7 +251,8 @@ async function AiLocateSection(options) {
             callAi: ()=>callAIWithObjectResponse(msgs, modelRuntime, {
                     abortSignal: options.abortSignal,
                     jsonParserSource: 'section-locator',
-                    retryTimes: 1
+                    retryTimes: modelRuntime.config.retryCount,
+                    retryInterval: modelRuntime.config.retryInterval
                 }),
             parseResponse: (result)=>{
                 const sectionError = result.content.error;
@@ -277,7 +280,8 @@ async function AiLocateSection(options) {
                 const message = result.content.error ? `${result.content.error} (${parseErrorMessage})` : parseErrorMessage;
                 return new AIResponseParseError(message, result.contentString, result.usage, result.rawChoiceMessage, result.reasoning_content);
             },
-            parseRetryTimes: 1,
+            parseRetryTimes: modelRuntime.config.retryCount,
+            parseRetryInterval: modelRuntime.config.retryInterval,
             abortSignal: options.abortSignal,
             onParseRetry: (error)=>{
                 debugSection('retrying section locate after coordinate parsing failed: %s', error instanceof Error ? error.message : String(error));
@@ -415,7 +419,8 @@ async function AiExtractElementInfo(options) {
             const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
             return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawChoiceMessage);
         },
-        parseRetryTimes: 1,
+        parseRetryTimes: modelRuntime.config.retryCount,
+        parseRetryInterval: modelRuntime.config.retryInterval,
         abortSignal: options.abortSignal,
         onParseRetry: (error)=>{
             debugInspect('retrying insight after XML parsing failed: %s', error instanceof Error ? error.message : String(error));

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequestContext,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\ntype SectionLocateObjectResponse = Awaited<\n  ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n>;\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const referenceImageMessages =\n    typeof targetElementDescription === 'string'\n      ? undefined\n      : await multimodalPromptToChatMessages(\n          userPromptToMultimodalPrompt(targetElementDescription),\n        );\n  const locateRequest: LocateRequestContext = {\n    elementDescriptionText: userPromptToString(targetElementDescription),\n    locateImage,\n    referenceImageMessages,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(\n    targetElementDescription,\n    locateOptions,\n    locateRequest,\n  );\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugInspect('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          locateRequest.elementDescriptionText,\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  _elementDescription: TUserPrompt,\n  options: LocateOptions,\n  locateRequest: LocateRequestContext,\n): Promise<LocateModelResponse> {\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const userInstructionPrompt = findElementPrompt(\n    locateRequest.elementDescriptionText,\n  );\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateRequest.locateImage.imageBase64,\n    width: locateRequest.locateImage.width,\n    height: locateRequest.locateImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (locateRequest.referenceImageMessages) {\n    msgs.push(...locateRequest.referenceImageMessages);\n  }\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AIElementLocateResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'locate',\n          retryTimes: 1,\n        }),\n      parseResponse: (response): LocateModelResponse => {\n        const rawResponse = response.contentString;\n        const errors: string[] | undefined =\n          'errors' in response.content ? response.content.errors : [];\n        if (\n          !hasLocateResult(response.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: errors as string[],\n          };\n        }\n\n        const locatedPixelBbox =\n          resultAdapter.adaptElementLocateResultToPixelBbox(response.content, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n        return {\n          locatedPixelBbox,\n          rawResponse,\n          rawChoiceMessage: response.rawChoiceMessage,\n          usage: response.usage,\n          reasoningContent: response.reasoning_content,\n          errors: errors as string[],\n        };\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        const modelErrors =\n          'errors' in response.content ? response.content.errors : undefined;\n        const message =\n          modelErrors && modelErrors.length > 0\n            ? `${modelErrors.join('\\n')} (${parseErrorMessage})`\n            : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          response.contentString,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: 1,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugInspect(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let parsedResult:\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AISectionLocatorResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'section-locator',\n          retryTimes: 1,\n        }),\n      parseResponse: (result) => {\n        const sectionError = result.content.error;\n        if (\n          !hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return { result, sectionError };\n        }\n\n        const adaptedResult =\n          resultAdapter.adaptSectionLocateResultToPixelBboxGroup(\n            result.content,\n            {\n              preparedSize: preparedImage.preparedSize,\n              contentSize: preparedImage.contentSize,\n            },\n          );\n        const mergedRect = mergePixelBboxesToRect([\n          adaptedResult.target,\n          ...(adaptedResult.references ?? []),\n        ]);\n        debugSection('mergedRect %j', mergedRect);\n        return { result, sectionError, mergedRect };\n      },\n      toParseError: (error, result) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        const message = result.content.error\n          ? `${result.content.error} (${parseErrorMessage})`\n          : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          result.contentString,\n          result.usage,\n          result.rawChoiceMessage,\n          result.reasoning_content,\n        );\n      },\n      parseRetryTimes: 1,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(msgs, modelRuntime, {\n        abortSignal: options.abortSignal,\n      }),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseXMLExtractionResponse<T>(rawResponse);\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: 1,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInspect(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","assert","locateImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","locateRequest","userPromptToString","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","error","msg","Error","_elementDescription","modelRuntime","adapter","resultAdapter","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","preparedImage","prepareModelImage","imagePayload","msgs","callAiAndParseWithRetry","callAIWithObjectResponse","response","parseErrorMessage","modelErrors","message","AIResponseParseError","String","callError","errorMessage","AiLocateSection","sectionDescription","screenshotBase64","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","addOns","parsedResult","result","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","searchAreaConfig","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","callAI","reasoning_content","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;;AAkEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAMO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,iBAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,WAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvDa,OACEF,0BACA;IAGF,MAAM,EAAEV,OAAO,EAAE,GAAGW;IACpB,MAAME,cAAcF,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaX,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMc,yBACJ,AAAoC,YAApC,OAAOJ,2BACHb,SACA,MAAMkB,+BACJC,6BAA6BN;IAErC,MAAMO,gBAAsC;QAC1C,wBAAwBC,mBAAmBR;QAC3CG;QACAC;QACA,SAASH;IACX;IAEA,MAAMQ,gBAAgBpB,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMqB,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAC3BV,0BACAC,eACAM;IAEF,MAAM,EACJM,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,SAASA;YACT+B;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAMC,OAAOC,gBACXC,0CACET,kBACAZ,cAAc,YAAY,EAAE;QAGhCvB,aAAa,WAAW0C;QAExB,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,sBACPH,MACAb,cAAc,sBAAsB;gBAEtC,QAAQ,EAAE;YACZ;YACA,GAAGY,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMrC;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ+B,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBgB,mBAAgC,EAChCtC,OAAsB,EACtBkB,aAAmC;IAEnC,MAAMqB,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAME,wBAAwBC,kBAC5BzB,cAAc,sBAAsB;IAEtC,MAAM0B,eAAeC,4BACnBL,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMM,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAa7B,cAAc,WAAW,CAAC,WAAW;QAClD,OAAOA,cAAc,WAAW,CAAC,KAAK;QACtC,QAAQA,cAAc,WAAW,CAAC,MAAM;QACxC,QAAQsB,QAAQ,eAAe;IACjC;IAEA,MAAMQ,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMN;gBACR;aACD;QACH;KACD;IAED,IAAIxB,cAAc,sBAAsB,EACtC+B,KAAK,IAAI,IAAI/B,cAAc,sBAAsB;IAGnD,IAAI;QACF,OAAO,MAAMgC,wBAAwB;YACnC,QAAQ,IACNC,yBAAkDF,MAAMV,cAAc;oBACpE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAY;gBACd;YACF,eAAe,CAACoD;gBACd,MAAM3B,cAAc2B,SAAS,aAAa;gBAC1C,MAAMvB,SACJ,YAAYuB,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAG,EAAE;gBAC7D,IACE,CAAC5D,gBAAgB4D,SAAS,OAAO,EAAEX,cAAc,UAAU,CAAC,SAAS,GAErE,OAAO;oBACLhB;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;gBAGF,MAAML,mBACJiB,cAAc,mCAAmC,CAACW,SAAS,OAAO,EAAE;oBAClE,cAAcN,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBACF,OAAO;oBACLtB;oBACAC;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;YACF;YACA,cAAc,CAACM,OAAOiB;gBACpB,MAAMC,oBACJlB,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,MAAMmB,cACJ,YAAYF,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAGtD;gBAC3D,MAAMyD,UACJD,eAAeA,YAAY,MAAM,GAAG,IAChC,GAAGA,YAAY,IAAI,CAAC,MAAM,EAAE,EAAED,kBAAkB,CAAC,CAAC,GAClDA;gBACN,OAAO,IAAIG,qBACTD,SACAH,SAAS,aAAa,EACtBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiB;YACjB,aAAapD,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb9C,aACE,uDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,sBACvB,OAAO;YACL,aAAaE,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,aAAaC;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF;AAEO,eAAeC,gBAAgB5D,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAE4D,kBAAkB,EAAE,GAAG7D;IACxC,MAAMuC,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAMsB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM6C,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAae;QACb,OAAO7D,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQuC,QAAQ,eAAe;IACjC;IAEA,MAAMI,eAAemB,4BACnBvB,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAMwB,gCAAgCC,0BACpC9C,mBAAmB0C;IAErB,MAAMZ,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMkB;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOH,oBAAiC;QAC1C,MAAMK,SAAS,MAAMlD,+BACnBC,6BAA6B4C;QAE/BZ,KAAK,IAAI,IAAIiB;IACf;IAEA,IAAIC;IAYJ,IAAI;QACFA,eAAe,MAAMjB,wBAAwB;YAC3C,QAAQ,IACNC,yBAAmDF,MAAMV,cAAc;oBACrE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAY;gBACd;YACF,eAAe,CAACoE;gBACd,MAAMC,eAAeD,OAAO,OAAO,CAAC,KAAK;gBACzC,IACE,CAAC5E,gBAAgB4E,OAAO,OAAO,EAAE3B,cAAc,UAAU,CAAC,SAAS,GAEnE,OAAO;oBAAE2B;oBAAQC;gBAAa;gBAGhC,MAAMC,gBACJ7B,cAAc,wCAAwC,CACpD2B,OAAO,OAAO,EACd;oBACE,cAActB,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBAEJ,MAAMyB,aAAaC,uBAAuB;oBACxCF,cAAc,MAAM;uBAChBA,cAAc,UAAU,IAAI,EAAE;iBACnC;gBACD/E,aAAa,iBAAiBgF;gBAC9B,OAAO;oBAAEH;oBAAQC;oBAAcE;gBAAW;YAC5C;YACA,cAAc,CAACpC,OAAOiC;gBACpB,MAAMf,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,MAAMoB,UAAUa,OAAO,OAAO,CAAC,KAAK,GAChC,GAAGA,OAAO,OAAO,CAAC,KAAK,CAAC,EAAE,EAAEf,kBAAkB,CAAC,CAAC,GAChDA;gBACJ,OAAO,IAAIG,qBACTD,SACAa,OAAO,aAAa,EACpBA,OAAO,KAAK,EACZA,OAAO,gBAAgB,EACvBA,OAAO,iBAAiB;YAE5B;YACA,iBAAiB;YACjB,aAAapE,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb5C,aACE,+DACA4C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,sBACvB,OAAO;YACL,kBAAkB1D;YAClB,OAAO4D,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,kBAAkB5D;YAClB,OAAO,CAAC,eAAe,EAAE6D,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAES,MAAM,EAAEC,YAAY,EAAEE,UAAU,EAAE,GAAGJ;IAC7C,IAAI,CAACI,YACH,OAAO;QACL,kBAAkBzE;QAClB,OAAOuE;QACP,aAAaD,OAAO,aAAa;QACjC,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMK,eAAepE,iBAAiBkE,YAAYtE,QAAQ,QAAQ;QAClE,MAAMyE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1ClF,aAAa,2BAA2BkF;QAExC,MAAMG,mBAAmB,MAAM7E,sBAAsB;YACnDE;YACA,UAAUsE;QACZ;QAEAhF,aACE,uDACAmF,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOP;YACP,aAAaD,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOjC,OAAO;QACd,MAAMkB,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMwB,eAAeU,eACjB,GAAGA,aAAa,EAAE,EAAEhB,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBvD;YAClB,OAAO6D;YACP,aAAaS,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF;AAEO,eAAeS,qBAAwB7E,OAQ7C;IACC,MAAM,EAAE8E,SAAS,EAAE7E,OAAO,EAAE8E,aAAa,EAAEC,gBAAgB,EAAEzC,YAAY,EAAE,GACzEvC;IACF,MAAM4C,eAAeqC,sBAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMlB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMiF,wBAAwBC,uBAC5BnF,QAAQ,eAAe,IAAI,IAC3B8E;IAGF,MAAMM,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/C,MAAMM,qBAAqBpF,QAAQ,kBAAkB;QACrD,IAAIoF,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKtB;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAsB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMjC,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASwC;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAMd,SAAS,MAAMlD,+BAA+BgE;QACpD/B,KAAK,IAAI,IAAIiB;IACf;IAEA,OAAOhB,wBAAwB;QAC7B,QAAQ,IACNsC,OAAOvC,MAAMV,cAAc;gBACzB,aAAavC,QAAQ,WAAW;YAClC;QACF,eAAe,CAACoD;YACd,MAAM,EACJ,SAAS3B,WAAW,EACpBE,KAAK,EACL8D,iBAAiB,EACjB/D,gBAAgB,EACjB,GAAG0B;YACJ,MAAMsC,cAAcC,2BAA8BlE;YAClD,OAAO;gBACLiE;gBACAjE;gBACAC;gBACAC;gBACA8D;YACF;QACF;QACA,cAAc,CAACG,YAAYxC;YACzB,MAAMO,eACJiC,sBAAsBvD,QAAQuD,WAAW,OAAO,GAAGnC,OAAOmC;YAC5D,OAAO,IAAIpC,qBACT,CAAC,iBAAiB,EAAEG,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiB;QACjB,aAAapD,QAAQ,WAAW;QAChC,cAAc,CAACmC;YACb9C,aACE,iDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;QAEpD;IACF;AACF;AAEO,eAAe0D,sBACpBC,WAAmB,EACnBvD,YAA0B;IAK1B,MAAMK,eAAemD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAM7C,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASoD;QACX;KACD;IAED3G,aAAa,yCAAyCyG;IAEtD,MAAM1B,SAAS,MAAMjB,yBACnBF,MACAV,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB6B,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequestContext,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\ntype SectionLocateObjectResponse = Awaited<\n  ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n>;\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const referenceImageMessages =\n    typeof targetElementDescription === 'string'\n      ? undefined\n      : await multimodalPromptToChatMessages(\n          userPromptToMultimodalPrompt(targetElementDescription),\n        );\n  const locateRequest: LocateRequestContext = {\n    elementDescriptionText: userPromptToString(targetElementDescription),\n    locateImage,\n    referenceImageMessages,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(\n    targetElementDescription,\n    locateOptions,\n    locateRequest,\n  );\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugInspect('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          locateRequest.elementDescriptionText,\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  _elementDescription: TUserPrompt,\n  options: LocateOptions,\n  locateRequest: LocateRequestContext,\n): Promise<LocateModelResponse> {\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const userInstructionPrompt = findElementPrompt(\n    locateRequest.elementDescriptionText,\n  );\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateRequest.locateImage.imageBase64,\n    width: locateRequest.locateImage.width,\n    height: locateRequest.locateImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (locateRequest.referenceImageMessages) {\n    msgs.push(...locateRequest.referenceImageMessages);\n  }\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AIElementLocateResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'locate',\n          retryTimes: modelRuntime.config.retryCount,\n          retryInterval: modelRuntime.config.retryInterval,\n        }),\n      parseResponse: (response): LocateModelResponse => {\n        const rawResponse = response.contentString;\n        const errors: string[] | undefined =\n          'errors' in response.content ? response.content.errors : [];\n        if (\n          !hasLocateResult(response.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: errors as string[],\n          };\n        }\n\n        const locatedPixelBbox =\n          resultAdapter.adaptElementLocateResultToPixelBbox(response.content, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n        return {\n          locatedPixelBbox,\n          rawResponse,\n          rawChoiceMessage: response.rawChoiceMessage,\n          usage: response.usage,\n          reasoningContent: response.reasoning_content,\n          errors: errors as string[],\n        };\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        const modelErrors =\n          'errors' in response.content ? response.content.errors : undefined;\n        const message =\n          modelErrors && modelErrors.length > 0\n            ? `${modelErrors.join('\\n')} (${parseErrorMessage})`\n            : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          response.contentString,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugInspect(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let parsedResult:\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AISectionLocatorResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'section-locator',\n          retryTimes: modelRuntime.config.retryCount,\n          retryInterval: modelRuntime.config.retryInterval,\n        }),\n      parseResponse: (result) => {\n        const sectionError = result.content.error;\n        if (\n          !hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return { result, sectionError };\n        }\n\n        const adaptedResult =\n          resultAdapter.adaptSectionLocateResultToPixelBboxGroup(\n            result.content,\n            {\n              preparedSize: preparedImage.preparedSize,\n              contentSize: preparedImage.contentSize,\n            },\n          );\n        const mergedRect = mergePixelBboxesToRect([\n          adaptedResult.target,\n          ...(adaptedResult.references ?? []),\n        ]);\n        debugSection('mergedRect %j', mergedRect);\n        return { result, sectionError, mergedRect };\n      },\n      toParseError: (error, result) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        const message = result.content.error\n          ? `${result.content.error} (${parseErrorMessage})`\n          : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          result.contentString,\n          result.usage,\n          result.rawChoiceMessage,\n          result.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(msgs, modelRuntime, {\n        abortSignal: options.abortSignal,\n      }),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseXMLExtractionResponse<T>(rawResponse);\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInspect(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","assert","locateImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","locateRequest","userPromptToString","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","error","msg","Error","_elementDescription","modelRuntime","adapter","resultAdapter","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","preparedImage","prepareModelImage","imagePayload","msgs","callAiAndParseWithRetry","callAIWithObjectResponse","response","parseErrorMessage","modelErrors","message","AIResponseParseError","String","callError","errorMessage","AiLocateSection","sectionDescription","screenshotBase64","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","addOns","parsedResult","result","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","searchAreaConfig","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","callAI","reasoning_content","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;;AAkEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAMO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,iBAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,WAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvDa,OACEF,0BACA;IAGF,MAAM,EAAEV,OAAO,EAAE,GAAGW;IACpB,MAAME,cAAcF,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaX,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMc,yBACJ,AAAoC,YAApC,OAAOJ,2BACHb,SACA,MAAMkB,+BACJC,6BAA6BN;IAErC,MAAMO,gBAAsC;QAC1C,wBAAwBC,mBAAmBR;QAC3CG;QACAC;QACA,SAASH;IACX;IAEA,MAAMQ,gBAAgBpB,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMqB,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAC3BV,0BACAC,eACAM;IAEF,MAAM,EACJM,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,SAASA;YACT+B;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAMC,OAAOC,gBACXC,0CACET,kBACAZ,cAAc,YAAY,EAAE;QAGhCvB,aAAa,WAAW0C;QAExB,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,sBACPH,MACAb,cAAc,sBAAsB;gBAEtC,QAAQ,EAAE;YACZ;YACA,GAAGY,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMrC;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ+B,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBgB,mBAAgC,EAChCtC,OAAsB,EACtBkB,aAAmC;IAEnC,MAAMqB,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAME,wBAAwBC,kBAC5BzB,cAAc,sBAAsB;IAEtC,MAAM0B,eAAeC,4BACnBL,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMM,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAa7B,cAAc,WAAW,CAAC,WAAW;QAClD,OAAOA,cAAc,WAAW,CAAC,KAAK;QACtC,QAAQA,cAAc,WAAW,CAAC,MAAM;QACxC,QAAQsB,QAAQ,eAAe;IACjC;IAEA,MAAMQ,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMN;gBACR;aACD;QACH;KACD;IAED,IAAIxB,cAAc,sBAAsB,EACtC+B,KAAK,IAAI,IAAI/B,cAAc,sBAAsB;IAGnD,IAAI;QACF,OAAO,MAAMgC,wBAAwB;YACnC,QAAQ,IACNC,yBAAkDF,MAAMV,cAAc;oBACpE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAYuC,aAAa,MAAM,CAAC,UAAU;oBAC1C,eAAeA,aAAa,MAAM,CAAC,aAAa;gBAClD;YACF,eAAe,CAACa;gBACd,MAAM3B,cAAc2B,SAAS,aAAa;gBAC1C,MAAMvB,SACJ,YAAYuB,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAG,EAAE;gBAC7D,IACE,CAAC5D,gBAAgB4D,SAAS,OAAO,EAAEX,cAAc,UAAU,CAAC,SAAS,GAErE,OAAO;oBACLhB;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;gBAGF,MAAML,mBACJiB,cAAc,mCAAmC,CAACW,SAAS,OAAO,EAAE;oBAClE,cAAcN,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBACF,OAAO;oBACLtB;oBACAC;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;YACF;YACA,cAAc,CAACM,OAAOiB;gBACpB,MAAMC,oBACJlB,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,MAAMmB,cACJ,YAAYF,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAGtD;gBAC3D,MAAMyD,UACJD,eAAeA,YAAY,MAAM,GAAG,IAChC,GAAGA,YAAY,IAAI,CAAC,MAAM,EAAE,EAAED,kBAAkB,CAAC,CAAC,GAClDA;gBACN,OAAO,IAAIG,qBACTD,SACAH,SAAS,aAAa,EACtBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBb,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAavC,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb9C,aACE,uDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,sBACvB,OAAO;YACL,aAAaE,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,aAAaC;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF;AAEO,eAAeC,gBAAgB5D,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAE4D,kBAAkB,EAAE,GAAG7D;IACxC,MAAMuC,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAMsB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM6C,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAae;QACb,OAAO7D,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQuC,QAAQ,eAAe;IACjC;IAEA,MAAMI,eAAemB,4BACnBvB,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAMwB,gCAAgCC,0BACpC9C,mBAAmB0C;IAErB,MAAMZ,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMkB;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOH,oBAAiC;QAC1C,MAAMK,SAAS,MAAMlD,+BACnBC,6BAA6B4C;QAE/BZ,KAAK,IAAI,IAAIiB;IACf;IAEA,IAAIC;IAYJ,IAAI;QACFA,eAAe,MAAMjB,wBAAwB;YAC3C,QAAQ,IACNC,yBAAmDF,MAAMV,cAAc;oBACrE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAYuC,aAAa,MAAM,CAAC,UAAU;oBAC1C,eAAeA,aAAa,MAAM,CAAC,aAAa;gBAClD;YACF,eAAe,CAAC6B;gBACd,MAAMC,eAAeD,OAAO,OAAO,CAAC,KAAK;gBACzC,IACE,CAAC5E,gBAAgB4E,OAAO,OAAO,EAAE3B,cAAc,UAAU,CAAC,SAAS,GAEnE,OAAO;oBAAE2B;oBAAQC;gBAAa;gBAGhC,MAAMC,gBACJ7B,cAAc,wCAAwC,CACpD2B,OAAO,OAAO,EACd;oBACE,cAActB,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBAEJ,MAAMyB,aAAaC,uBAAuB;oBACxCF,cAAc,MAAM;uBAChBA,cAAc,UAAU,IAAI,EAAE;iBACnC;gBACD/E,aAAa,iBAAiBgF;gBAC9B,OAAO;oBAAEH;oBAAQC;oBAAcE;gBAAW;YAC5C;YACA,cAAc,CAACpC,OAAOiC;gBACpB,MAAMf,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,MAAMoB,UAAUa,OAAO,OAAO,CAAC,KAAK,GAChC,GAAGA,OAAO,OAAO,CAAC,KAAK,CAAC,EAAE,EAAEf,kBAAkB,CAAC,CAAC,GAChDA;gBACJ,OAAO,IAAIG,qBACTD,SACAa,OAAO,aAAa,EACpBA,OAAO,KAAK,EACZA,OAAO,gBAAgB,EACvBA,OAAO,iBAAiB;YAE5B;YACA,iBAAiB7B,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAavC,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb5C,aACE,+DACA4C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,sBACvB,OAAO;YACL,kBAAkB1D;YAClB,OAAO4D,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,kBAAkB5D;YAClB,OAAO,CAAC,eAAe,EAAE6D,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAES,MAAM,EAAEC,YAAY,EAAEE,UAAU,EAAE,GAAGJ;IAC7C,IAAI,CAACI,YACH,OAAO;QACL,kBAAkBzE;QAClB,OAAOuE;QACP,aAAaD,OAAO,aAAa;QACjC,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMK,eAAepE,iBAAiBkE,YAAYtE,QAAQ,QAAQ;QAClE,MAAMyE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1ClF,aAAa,2BAA2BkF;QAExC,MAAMG,mBAAmB,MAAM7E,sBAAsB;YACnDE;YACA,UAAUsE;QACZ;QAEAhF,aACE,uDACAmF,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOP;YACP,aAAaD,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOjC,OAAO;QACd,MAAMkB,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMwB,eAAeU,eACjB,GAAGA,aAAa,EAAE,EAAEhB,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBvD;YAClB,OAAO6D;YACP,aAAaS,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF;AAEO,eAAeS,qBAAwB7E,OAQ7C;IACC,MAAM,EAAE8E,SAAS,EAAE7E,OAAO,EAAE8E,aAAa,EAAEC,gBAAgB,EAAEzC,YAAY,EAAE,GACzEvC;IACF,MAAM4C,eAAeqC,sBAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMlB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMiF,wBAAwBC,uBAC5BnF,QAAQ,eAAe,IAAI,IAC3B8E;IAGF,MAAMM,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/C,MAAMM,qBAAqBpF,QAAQ,kBAAkB;QACrD,IAAIoF,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKtB;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAsB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMjC,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASwC;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAMd,SAAS,MAAMlD,+BAA+BgE;QACpD/B,KAAK,IAAI,IAAIiB;IACf;IAEA,OAAOhB,wBAAwB;QAC7B,QAAQ,IACNsC,OAAOvC,MAAMV,cAAc;gBACzB,aAAavC,QAAQ,WAAW;YAClC;QACF,eAAe,CAACoD;YACd,MAAM,EACJ,SAAS3B,WAAW,EACpBE,KAAK,EACL8D,iBAAiB,EACjB/D,gBAAgB,EACjB,GAAG0B;YACJ,MAAMsC,cAAcC,2BAA8BlE;YAClD,OAAO;gBACLiE;gBACAjE;gBACAC;gBACAC;gBACA8D;YACF;QACF;QACA,cAAc,CAACG,YAAYxC;YACzB,MAAMO,eACJiC,sBAAsBvD,QAAQuD,WAAW,OAAO,GAAGnC,OAAOmC;YAC5D,OAAO,IAAIpC,qBACT,CAAC,iBAAiB,EAAEG,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBb,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAavC,QAAQ,WAAW;QAChC,cAAc,CAACmC;YACb9C,aACE,iDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;QAEpD;IACF;AACF;AAEO,eAAe0D,sBACpBC,WAAmB,EACnBvD,YAA0B;IAK1B,MAAMK,eAAemD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAM7C,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASoD;QACX;KACD;IAED3G,aAAa,yCAAyCyG;IAEtD,MAAM1B,SAAS,MAAMjB,yBACnBF,MACAV,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB6B,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/llm-planning.mjs

@@ -114,7 +114,8 @@ async function callAndParsePlanningResponse(options) {
             const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
             return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
         },
-        parseRetryTimes: 1,
+        parseRetryTimes: modelRuntime.config.retryCount,
+        parseRetryInterval: modelRuntime.config.retryInterval,
         abortSignal,
         onParseRetry: (parseError)=>{
             debug('retrying plan after response parsing failed: %s', parseError instanceof Error ? parseError.message : String(parseError));

package/dist/es/ai-model/llm-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToString } from '@/common';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../common';\nimport { planningModelFamilyRequiredForLocateMessage } from './errors';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport { AIResponseParseError, callAI } from './service-caller/index';\nimport type { JsonParser, JsonParserSource } from './service-caller/json';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport type {\n  LocateResultAdapter,\n  LocateResultContext,\n} from './shared/model-locate-result';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport { normalizePlanningActionLocateFields } from './workflows/planning/locate-normalization';\nimport type { PlanOptions } from './workflows/planning/types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse.\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  jsonParser: JsonParser,\n): RawResponsePlanningAIResponse {\n  // Use <planning> instead of <thought> to avoid colliding with Gemini thought\n  // summaries, which may also be emitted as <thought> in OpenAI-compatible\n  // response content.\n  const thought = extractXMLTag(xmlString, 'planning');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    // Strip any trailing XML tags that LLM might have leaked into the action type\n    // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultAdapter?: LocateResultAdapter;\n  locateResultContext: LocateResultContext;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultAdapter,\n    locateResultContext,\n  } = options;\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(messages, modelRuntime, {\n        abortSignal,\n        requiresOriginalImageDetail: includeLocateInPlanning,\n      }),\n    parseResponse: (response) => {\n      const planFromAI = parseXMLPlanningResponse(\n        response.content,\n        modelRuntime.adapter.jsonParser,\n      );\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      // Keep yamlFlow based on the model's original action params. Coordinate\n      // normalization adds runtime-only locatedPixelBbox fields afterwards.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultAdapter,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: 1,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function plan(\n  userInstruction: TUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultAdapter =\n    modelRuntime.config.modelFamily && adapter.locate.kind === 'standard'\n      ? adapter.locate.resultAdapter\n      : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    locatePromptSpec: locateResultAdapter?.promptSpec,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const userInstructionText = userPromptToString(userInstruction);\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = opts.referenceImageMessages ?? [];\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstructionText}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultAdapter,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: [\n      {\n        type: 'text',\n        text: rawResponse,\n      },\n    ],\n  });\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","parseXMLPlanningResponse","xmlString","jsonParser","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","e","Error","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultAdapter","locateResultContext","callAiAndParseWithRetry","callAI","response","planFromAI","actions","yamlFlow","buildYamlFlowFromPlans","normalizePlanningActionLocateFields","parseError","errorMessage","String","AIResponseParseError","plan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","preparedImage","prepareModelImage","imagePayload","userInstructionText","userPromptToString","actionContext","referenceImageMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","assert","index"],"mappings":";;;;;;;;;;AA4BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAKK,SAASC,yBACdC,SAAiB,EACjBC,UAAsB;IAKtB,MAAMC,UAAUC,cAAcH,WAAW;IACzC,MAAMI,SAASD,cAAcH,WAAW;IACxC,MAAMK,MAAMF,cAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,cAAcH,WAAW;IACvC,MAAMO,aAAaJ,cAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,cAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,cAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,cAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,qBAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,yBAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQrB,WAAWO,gBAAgB;gBACjC,QAAQ;gBACR,yBACEa,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGR;YACjD;QACF,EAAE,OAAOU,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFH,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAcA,eAAeO,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,mBAAmB,EACnBC,mBAAmB,EACpB,GAAGP;IACJ,OAAOQ,wBAAwB;QAC7B,QAAQ,IACNC,OAAOR,UAAUC,cAAc;gBAC7BC;gBACA,6BAA6BC;YAC/B;QACF,eAAe,CAACM;YACd,MAAMC,aAAatC,yBACjBqC,SAAS,OAAO,EAChBR,aAAa,OAAO,CAAC,UAAU;YAEjC,IAAIS,WAAW,MAAM,IAAIA,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;gBACjExC,QACE;gBAEFwC,WAAW,eAAe,GAAGxB;gBAC7BwB,WAAW,eAAe,GAAGxB;YAC/B;YAEA,MAAMyB,UAAUD,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAG5D,MAAME,WAAWC,uBAAuBF,SAASP;YACjDU,oCAAoCH,SAAS;gBAC3CP;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEG;gBAAUC;gBAAYC;gBAASC;YAAS;QACnD;QACA,cAAc,CAACG,YAAYN;YACzB,MAAMO,eACJD,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiB;QACjBP;QACA,cAAc,CAACa;YACb/C,MACE,mDACA+C,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,KACpBC,eAA4B,EAC5BC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAMpB,eAAeoB,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGvB;IACpB,MAAM,EAAEwB,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,IAAID,KAAK,uBAAuB,IAAI,CAACpB,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIJ,MACR8B,4CAA4C1B,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,sBACJJ,aAAa,MAAM,CAAC,WAAW,IAAIuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,GAClDA,QAAQ,MAAM,CAAC,aAAa,GAC5BtC;IAGN,MAAM0C,kBAAkBP,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMQ,eAAe,MAAMC,2BAA2B;QACpD,aAAaT,KAAK,WAAW;QAC7B,kBAAkBhB,qBAAqB;QACvC,yBAAyBgB,KAAK,uBAAuB;QACrD,gBAAgB;QAChBO;IACF;IAEA,MAAMG,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaN;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMS,eAAeF,cAAc,WAAW;IAE9C,MAAMG,sBAAsBC,mBAAmBf;IAC/C,MAAMgB,gBAAgBf,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMgB,yBAAyBhB,KAAK,sBAAsB,IAAI,EAAE;IAChE,MAAMiB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGF,cAAc,kBAAkB,EAAEF,oBAAoB,mBAAmB,CAAC;gBACrF;aACD;QACH;WACGG;KACJ;IAED,IAAIE;IAKJ,MAAMC,wBAAwBZ,kBAC1BL,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpD,uBAAuB;IAGpC,MAAMuE,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAV,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFV,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACDvC,UAAU,EACVC,OAAO,EACPC,QAAQ,EACT,GAAG,MAAMd,6BAA6B;QACrC,UAAU+C;QACV5C;QACA,aAAaoB,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BhB;QACA,qBAAqB;YACnB,cAAc0B,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;IACF;IAEA,IAAImB,yBAAyB;IAG7B,IAAIxC,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;QAC5C1C,MAAM;QACNkF,yBAAyB;QAEzB,IAAItB,iBACFL,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAGzC,UAAU;QACbC;QACAmC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAE,OAAO1C,YAAY;IAGnB,IAAIkB,iBAAiB;QACnB,IAAIlB,WAAW,cAAc,EAAE,QAC7Ba,oBAAoB,aAAa,CAACb,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM2C,SAAS3C,WAAW,mBAAmB,CAChDa,oBAAoB,mBAAmB,CAAC8B;QAI5C,IAAI3C,WAAW,GAAG,EAChBa,oBAAoB,gBAAgB,CAACb,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBa,oBAAoB,mBAAmB,CAACb,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBa,oBAAoB,YAAY,CAACb,WAAW,MAAM;IAGpDa,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAEA,OAAOK;AACT"}
+{"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToString } from '@/common';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../common';\nimport { planningModelFamilyRequiredForLocateMessage } from './errors';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport { AIResponseParseError, callAI } from './service-caller/index';\nimport type { JsonParser, JsonParserSource } from './service-caller/json';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport type {\n  LocateResultAdapter,\n  LocateResultContext,\n} from './shared/model-locate-result';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport { normalizePlanningActionLocateFields } from './workflows/planning/locate-normalization';\nimport type { PlanOptions } from './workflows/planning/types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse.\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  jsonParser: JsonParser,\n): RawResponsePlanningAIResponse {\n  // Use <planning> instead of <thought> to avoid colliding with Gemini thought\n  // summaries, which may also be emitted as <thought> in OpenAI-compatible\n  // response content.\n  const thought = extractXMLTag(xmlString, 'planning');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    // Strip any trailing XML tags that LLM might have leaked into the action type\n    // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultAdapter?: LocateResultAdapter;\n  locateResultContext: LocateResultContext;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultAdapter,\n    locateResultContext,\n  } = options;\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(messages, modelRuntime, {\n        abortSignal,\n        requiresOriginalImageDetail: includeLocateInPlanning,\n      }),\n    parseResponse: (response) => {\n      const planFromAI = parseXMLPlanningResponse(\n        response.content,\n        modelRuntime.adapter.jsonParser,\n      );\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      // Keep yamlFlow based on the model's original action params. Coordinate\n      // normalization adds runtime-only locatedPixelBbox fields afterwards.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultAdapter,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function plan(\n  userInstruction: TUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultAdapter =\n    modelRuntime.config.modelFamily && adapter.locate.kind === 'standard'\n      ? adapter.locate.resultAdapter\n      : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    locatePromptSpec: locateResultAdapter?.promptSpec,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const userInstructionText = userPromptToString(userInstruction);\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = opts.referenceImageMessages ?? [];\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstructionText}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultAdapter,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: [\n      {\n        type: 'text',\n        text: rawResponse,\n      },\n    ],\n  });\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","parseXMLPlanningResponse","xmlString","jsonParser","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","e","Error","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultAdapter","locateResultContext","callAiAndParseWithRetry","callAI","response","planFromAI","actions","yamlFlow","buildYamlFlowFromPlans","normalizePlanningActionLocateFields","parseError","errorMessage","String","AIResponseParseError","plan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","preparedImage","prepareModelImage","imagePayload","userInstructionText","userPromptToString","actionContext","referenceImageMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","assert","index"],"mappings":";;;;;;;;;;AA4BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAKK,SAASC,yBACdC,SAAiB,EACjBC,UAAsB;IAKtB,MAAMC,UAAUC,cAAcH,WAAW;IACzC,MAAMI,SAASD,cAAcH,WAAW;IACxC,MAAMK,MAAMF,cAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,cAAcH,WAAW;IACvC,MAAMO,aAAaJ,cAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,cAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,cAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,cAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,qBAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,yBAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQrB,WAAWO,gBAAgB;gBACjC,QAAQ;gBACR,yBACEa,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGR;YACjD;QACF,EAAE,OAAOU,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFH,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAcA,eAAeO,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,mBAAmB,EACnBC,mBAAmB,EACpB,GAAGP;IACJ,OAAOQ,wBAAwB;QAC7B,QAAQ,IACNC,OAAOR,UAAUC,cAAc;gBAC7BC;gBACA,6BAA6BC;YAC/B;QACF,eAAe,CAACM;YACd,MAAMC,aAAatC,yBACjBqC,SAAS,OAAO,EAChBR,aAAa,OAAO,CAAC,UAAU;YAEjC,IAAIS,WAAW,MAAM,IAAIA,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;gBACjExC,QACE;gBAEFwC,WAAW,eAAe,GAAGxB;gBAC7BwB,WAAW,eAAe,GAAGxB;YAC/B;YAEA,MAAMyB,UAAUD,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAG5D,MAAME,WAAWC,uBAAuBF,SAASP;YACjDU,oCAAoCH,SAAS;gBAC3CP;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEG;gBAAUC;gBAAYC;gBAASC;YAAS;QACnD;QACA,cAAc,CAACG,YAAYN;YACzB,MAAMO,eACJD,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBR,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACa;YACb/C,MACE,mDACA+C,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,KACpBC,eAA4B,EAC5BC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAMpB,eAAeoB,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGvB;IACpB,MAAM,EAAEwB,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,IAAID,KAAK,uBAAuB,IAAI,CAACpB,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIJ,MACR8B,4CAA4C1B,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,sBACJJ,aAAa,MAAM,CAAC,WAAW,IAAIuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,GAClDA,QAAQ,MAAM,CAAC,aAAa,GAC5BtC;IAGN,MAAM0C,kBAAkBP,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMQ,eAAe,MAAMC,2BAA2B;QACpD,aAAaT,KAAK,WAAW;QAC7B,kBAAkBhB,qBAAqB;QACvC,yBAAyBgB,KAAK,uBAAuB;QACrD,gBAAgB;QAChBO;IACF;IAEA,MAAMG,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaN;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMS,eAAeF,cAAc,WAAW;IAE9C,MAAMG,sBAAsBC,mBAAmBf;IAC/C,MAAMgB,gBAAgBf,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMgB,yBAAyBhB,KAAK,sBAAsB,IAAI,EAAE;IAChE,MAAMiB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGF,cAAc,kBAAkB,EAAEF,oBAAoB,mBAAmB,CAAC;gBACrF;aACD;QACH;WACGG;KACJ;IAED,IAAIE;IAKJ,MAAMC,wBAAwBZ,kBAC1BL,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpD,uBAAuB;IAGpC,MAAMuE,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAV,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFV,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACDvC,UAAU,EACVC,OAAO,EACPC,QAAQ,EACT,GAAG,MAAMd,6BAA6B;QACrC,UAAU+C;QACV5C;QACA,aAAaoB,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BhB;QACA,qBAAqB;YACnB,cAAc0B,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;IACF;IAEA,IAAImB,yBAAyB;IAG7B,IAAIxC,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;QAC5C1C,MAAM;QACNkF,yBAAyB;QAEzB,IAAItB,iBACFL,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAGzC,UAAU;QACbC;QACAmC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAE,OAAO1C,YAAY;IAGnB,IAAIkB,iBAAiB;QACnB,IAAIlB,WAAW,cAAc,EAAE,QAC7Ba,oBAAoB,aAAa,CAACb,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM2C,SAAS3C,WAAW,mBAAmB,CAChDa,oBAAoB,mBAAmB,CAAC8B;QAI5C,IAAI3C,WAAW,GAAG,EAChBa,oBAAoB,gBAAgB,CAACb,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBa,oBAAoB,mBAAmB,CAACb,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBa,oBAAoB,YAAY,CAACb,WAAW,MAAM;IAGpDa,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAEA,OAAOK;AACT"}

package/dist/es/ai-model/models/doubao.mjs

@@ -50,6 +50,9 @@ const buildDoubaoChatCompletionParams = (input)=>{
     const { reasoningEnabled, reasoningEffort } = userConfig;
     const commonOverrideConfig = {};
     if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
+    if ('none' !== userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
+        type: 'json_object'
+    };
     const modelSpecificConfig = {};
     if ('default' !== reasoningEnabled) {
         modelSpecificConfig.thinking = {

package/dist/es/ai-model/models/doubao.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/doubao.mjs","sources":["../../../../src/ai-model/models/doubao.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { parseModelResponseJson } from '../service-caller/json';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst doubaoBboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\nconst doubaoPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\n/**\n * Finds a sequence of numbers separated by punctuation or whitespace in a\n * string. Separators cannot be letters, which prevents extracting the `2` in\n * a mixed alphanumeric token such as `bbox_2d` as a coordinate.\n */\nconst coordinateSequencePattern =\n  /(?:^|[^a-zA-Z0-9])(\\d+(?:[^a-zA-Z0-9]+\\d+)+)(?=$|[^a-zA-Z0-9])/g;\n\nfunction isFourFiniteNumberArray(input: unknown): input is number[] {\n  return (\n    Array.isArray(input) &&\n    input.length === 4 &&\n    input.every((value) => typeof value === 'number' && Number.isFinite(value))\n  );\n}\n\nfunction parseNumbersFromUnexpectedBboxStructure(input: unknown): number[] {\n  const serialized = JSON.stringify(input);\n  if (!serialized) {\n    return [];\n  }\n\n  const sequences = Array.from(\n    serialized.matchAll(coordinateSequencePattern),\n    (match) => match[1].match(/\\d+/g)?.map(Number) ?? [],\n  );\n  const longestLength = Math.max(\n    0,\n    ...sequences.map((sequence) => sequence.length),\n  );\n  const longestSequences = sequences.filter(\n    (sequence) => sequence.length === longestLength,\n  );\n\n  if (longestSequences.length !== 1) {\n    return [];\n  }\n\n  return longestSequences[0];\n}\n\nexport function parseDoubaoRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  const bboxList = isFourFiniteNumberArray(bbox)\n    ? bbox\n    : parseNumbersFromUnexpectedBboxStructure(bbox);\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[2],\n      bboxList[3],\n    ]);\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return createLocateResultValue(doubaoPointCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n    ]);\n  }\n\n  if (bboxList.length === 8) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[4],\n      bboxList[5],\n    ]);\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nconst buildDoubaoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n    if (reasoningEffort) {\n      modelSpecificConfig.reasoning_effort = reasoningEffort;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst doubaoVisionAdapter: ModelAdapterDefinition = {\n  jsonParser: parseModelResponseJson,\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningBudget'],\n    buildChatCompletionParams: buildDoubaoChatCompletionParams,\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    resultAdapter: {\n      coordinates: doubaoBboxCoordinatesMeta,\n      parseRawLocateValue: parseDoubaoRawLocateValue,\n    },\n  },\n};\n\nexport const doubaoAdapters = {\n  'doubao-vision': doubaoVisionAdapter,\n  'doubao-seed': doubaoVisionAdapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'doubao-vision' | 'doubao-seed'\n>;\n"],"names":["doubaoBboxCoordinatesMeta","doubaoPointCoordinatesMeta","coordinateSequencePattern","isFourFiniteNumberArray","input","Array","value","Number","parseNumbersFromUnexpectedBboxStructure","serialized","JSON","sequences","match","longestLength","Math","sequence","longestSequences","parseDoubaoRawLocateValue","bbox","unwrapCoordinateListLikeInput","bboxList","createLocateResultValue","msg","Error","buildDoubaoChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","undefined","modelSpecificConfig","doubaoVisionAdapter","parseModelResponseJson","doubaoAdapters"],"mappings":";;AAaA,MAAMA,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAOA,MAAMC,4BACJ;AAEF,SAASC,wBAAwBC,KAAc;IAC7C,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AAExE;AAEA,SAASE,wCAAwCJ,KAAc;IAC7D,MAAMK,aAAaC,KAAK,SAAS,CAACN;IAClC,IAAI,CAACK,YACH,OAAO,EAAE;IAGX,MAAME,YAAYN,MAAM,IAAI,CAC1BI,WAAW,QAAQ,CAACP,4BACpB,CAACU,QAAUA,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,IAAIL,WAAW,EAAE;IAEtD,MAAMM,gBAAgBC,KAAK,GAAG,CAC5B,MACGH,UAAU,GAAG,CAAC,CAACI,WAAaA,SAAS,MAAM;IAEhD,MAAMC,mBAAmBL,UAAU,MAAM,CACvC,CAACI,WAAaA,SAAS,MAAM,KAAKF;IAGpC,IAAIG,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,OAAO,EAAE;IAGX,OAAOA,gBAAgB,CAAC,EAAE;AAC5B;AAEO,SAASC,0BAA0Bb,KAAc;IACtD,MAAMc,OAAOC,8BAA8Bf;IAC3C,MAAMgB,WAAWjB,wBAAwBe,QACrCA,OACAV,wCAAwCU;IAE5C,IAAIE,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAOC,wBAAwBrB,2BAA2B;QACxDoB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAOC,wBAAwBpB,4BAA4B;QACzDmB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAOC,wBAAwBrB,2BAA2B;QACxDoB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,MAAME,MAAM,CAAC,0CAA0C,EAAEZ,KAAK,SAAS,CAACQ,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIK,MAAMD;AAClB;AAEA,MAAME,kCAAkC,CACtCpB;IAEA,MAAM,EAAEqB,gBAAgB,EAAEC,UAAU,EAAE,GAAGtB;IACzC,MAAM,EAAEuB,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,MAAMK,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBAAgC;QAClCI,oBAAoB,QAAQ,GAAG;YAC7B,MAAOJ,oBAAoB,QAAS,YAAY;QAClD;QACA,IAAIC,iBACFG,oBAAoB,gBAAgB,GAAGH;IAE3C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,sBAA8C;IAClD,YAAYC;IACZ,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BT;QAC3B,+BAA+B;IACjC;IACA,QAAQ;QACN,eAAe;YACb,aAAaxB;YACb,qBAAqBiB;QACvB;IACF;AACF;AAEO,MAAMiB,iBAAiB;IAC5B,iBAAiBF;IACjB,eAAeA;AACjB"}
+{"version":3,"file":"ai-model/models/doubao.mjs","sources":["../../../../src/ai-model/models/doubao.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { parseModelResponseJson } from '../service-caller/json';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst doubaoBboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\nconst doubaoPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\n/**\n * Finds a sequence of numbers separated by punctuation or whitespace in a\n * string. Separators cannot be letters, which prevents extracting the `2` in\n * a mixed alphanumeric token such as `bbox_2d` as a coordinate.\n */\nconst coordinateSequencePattern =\n  /(?:^|[^a-zA-Z0-9])(\\d+(?:[^a-zA-Z0-9]+\\d+)+)(?=$|[^a-zA-Z0-9])/g;\n\nfunction isFourFiniteNumberArray(input: unknown): input is number[] {\n  return (\n    Array.isArray(input) &&\n    input.length === 4 &&\n    input.every((value) => typeof value === 'number' && Number.isFinite(value))\n  );\n}\n\nfunction parseNumbersFromUnexpectedBboxStructure(input: unknown): number[] {\n  const serialized = JSON.stringify(input);\n  if (!serialized) {\n    return [];\n  }\n\n  const sequences = Array.from(\n    serialized.matchAll(coordinateSequencePattern),\n    (match) => match[1].match(/\\d+/g)?.map(Number) ?? [],\n  );\n  const longestLength = Math.max(\n    0,\n    ...sequences.map((sequence) => sequence.length),\n  );\n  const longestSequences = sequences.filter(\n    (sequence) => sequence.length === longestLength,\n  );\n\n  if (longestSequences.length !== 1) {\n    return [];\n  }\n\n  return longestSequences[0];\n}\n\nexport function parseDoubaoRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  const bboxList = isFourFiniteNumberArray(bbox)\n    ? bbox\n    : parseNumbersFromUnexpectedBboxStructure(bbox);\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[2],\n      bboxList[3],\n    ]);\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return createLocateResultValue(doubaoPointCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n    ]);\n  }\n\n  if (bboxList.length === 8) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[4],\n      bboxList[5],\n    ]);\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nconst buildDoubaoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // Doubao Chat Completions JSON mode:\n  // https://docs.volcengine.com/docs/82379/1568221?lang=zh\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n    if (reasoningEffort) {\n      modelSpecificConfig.reasoning_effort = reasoningEffort;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst doubaoVisionAdapter: ModelAdapterDefinition = {\n  jsonParser: parseModelResponseJson,\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningBudget'],\n    buildChatCompletionParams: buildDoubaoChatCompletionParams,\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    resultAdapter: {\n      coordinates: doubaoBboxCoordinatesMeta,\n      parseRawLocateValue: parseDoubaoRawLocateValue,\n    },\n  },\n};\n\nexport const doubaoAdapters = {\n  'doubao-vision': doubaoVisionAdapter,\n  'doubao-seed': doubaoVisionAdapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'doubao-vision' | 'doubao-seed'\n>;\n"],"names":["doubaoBboxCoordinatesMeta","doubaoPointCoordinatesMeta","coordinateSequencePattern","isFourFiniteNumberArray","input","Array","value","Number","parseNumbersFromUnexpectedBboxStructure","serialized","JSON","sequences","match","longestLength","Math","sequence","longestSequences","parseDoubaoRawLocateValue","bbox","unwrapCoordinateListLikeInput","bboxList","createLocateResultValue","msg","Error","buildDoubaoChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","undefined","modelSpecificConfig","doubaoVisionAdapter","parseModelResponseJson","doubaoAdapters"],"mappings":";;AAaA,MAAMA,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAOA,MAAMC,4BACJ;AAEF,SAASC,wBAAwBC,KAAc;IAC7C,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AAExE;AAEA,SAASE,wCAAwCJ,KAAc;IAC7D,MAAMK,aAAaC,KAAK,SAAS,CAACN;IAClC,IAAI,CAACK,YACH,OAAO,EAAE;IAGX,MAAME,YAAYN,MAAM,IAAI,CAC1BI,WAAW,QAAQ,CAACP,4BACpB,CAACU,QAAUA,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,IAAIL,WAAW,EAAE;IAEtD,MAAMM,gBAAgBC,KAAK,GAAG,CAC5B,MACGH,UAAU,GAAG,CAAC,CAACI,WAAaA,SAAS,MAAM;IAEhD,MAAMC,mBAAmBL,UAAU,MAAM,CACvC,CAACI,WAAaA,SAAS,MAAM,KAAKF;IAGpC,IAAIG,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,OAAO,EAAE;IAGX,OAAOA,gBAAgB,CAAC,EAAE;AAC5B;AAEO,SAASC,0BAA0Bb,KAAc;IACtD,MAAMc,OAAOC,8BAA8Bf;IAC3C,MAAMgB,WAAWjB,wBAAwBe,QACrCA,OACAV,wCAAwCU;IAE5C,IAAIE,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAOC,wBAAwBrB,2BAA2B;QACxDoB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAOC,wBAAwBpB,4BAA4B;QACzDmB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAOC,wBAAwBrB,2BAA2B;QACxDoB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,MAAME,MAAM,CAAC,0CAA0C,EAAEZ,KAAK,SAAS,CAACQ,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIK,MAAMD;AAClB;AAEA,MAAME,kCAAkC,CACtCpB;IAEA,MAAM,EAAEqB,gBAAgB,EAAEC,UAAU,EAAE,GAAGtB;IACzC,MAAM,EAAEuB,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAK3D,IACEA,AAA8B,WAA9BA,WAAW,cAAc,IACzBtB,MAAM,0BAA0B,EAEhCyB,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBAAgC;QAClCI,oBAAoB,QAAQ,GAAG;YAC7B,MAAOJ,oBAAoB,QAAS,YAAY;QAClD;QACA,IAAIC,iBACFG,oBAAoB,gBAAgB,GAAGH;IAE3C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,sBAA8C;IAClD,YAAYC;IACZ,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BT;QAC3B,+BAA+B;IACjC;IACA,QAAQ;QACN,eAAe;YACb,aAAaxB;YACb,qBAAqBiB;QACvB;IACF;AACF;AAEO,MAAMiB,iBAAiB;IAC5B,iBAAiBF;IACjB,eAAeA;AACjB"}

package/dist/es/ai-model/models/glm.mjs

@@ -1,10 +1,9 @@
-import { isLocateIntent } from "./utils/intent.mjs";
 const buildGlmChatCompletionParams = (input)=>{
     const { midsceneDefaults, userConfig } = input;
     const { reasoningEnabled } = userConfig;
     const commonOverrideConfig = {};
     if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
-    if (isLocateIntent(input.intent)) commonOverrideConfig.response_format = {
+    if ('none' !== userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
         type: 'json_object'
     };
     const modelSpecificConfig = {};

package/dist/es/ai-model/models/glm.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/glm.mjs","sources":["../../../../src/ai-model/models/glm.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { isLocateIntent } from './utils/intent';\n\nconst buildGlmChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // Zhipu structured output JSON mode:\n  // https://docs.bigmodel.cn/cn/guide/capabilities/struct-output\n  if (isLocateIntent(input.intent)) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const glmAdapters = {\n  'glm-v': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildGlmChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'glm-v'>;\n"],"names":["buildGlmChatCompletionParams","input","midsceneDefaults","userConfig","reasoningEnabled","commonOverrideConfig","undefined","isLocateIntent","modelSpecificConfig","glmAdapters"],"mappings":";AAQA,MAAMA,+BAA+B,CACnCC;IAEA,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGF;IACzC,MAAM,EAAEG,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,uBAAgD,CAAC;IAEvD,IAAIF,AAA2BG,WAA3BH,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAK3D,IAAII,eAAeN,MAAM,MAAM,GAC7BI,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAMG,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBACFI,oBAAoB,QAAQ,GAAG;QAC7B,MAAOJ,oBAAoB,QAAS,YAAY;IAClD;IAGF,OAAO;QACL,QAAQ;YACN,GAAGF,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,GAAGG,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,cAAc;IACzB,SAAS;QACP,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BT;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;oBAAM,cAAc;gBAAK;YAChE;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/models/glm.mjs","sources":["../../../../src/ai-model/models/glm.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\n\nconst buildGlmChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // Zhipu structured output JSON mode:\n  // https://docs.bigmodel.cn/cn/guide/capabilities/struct-output\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const glmAdapters = {\n  'glm-v': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildGlmChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'glm-v'>;\n"],"names":["buildGlmChatCompletionParams","input","midsceneDefaults","userConfig","reasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","glmAdapters"],"mappings":"AAOA,MAAMA,+BAA+B,CACnCC;IAEA,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGF;IACzC,MAAM,EAAEG,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,uBAAgD,CAAC;IAEvD,IAAIF,AAA2BG,WAA3BH,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAK3D,IACEA,AAA8B,WAA9BA,WAAW,cAAc,IACzBF,MAAM,0BAA0B,EAEhCI,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C,CAAC;IAEtD,IAAIH,AAAqB,cAArBA,kBACFG,oBAAoB,QAAQ,GAAG;QAC7B,MAAOH,oBAAoB,QAAS,YAAY;IAClD;IAGF,OAAO;QACL,QAAQ;YACN,GAAGF,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,cAAc;IACzB,SAAS;QACP,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BR;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;oBAAM,cAAc;gBAAK;YAChE;QACF;IACF;AACF"}

package/dist/es/ai-model/models/gpt.mjs

@@ -5,7 +5,7 @@ const buildGpt5ChatCompletionParams = (input)=>{
     const { reasoningEnabled, reasoningEffort } = userConfig;
     const commonOverrideConfig = {};
     if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
-    if (isLocateIntent(input.intent)) commonOverrideConfig.response_format = {
+    if ('none' !== input.userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
         type: 'json_object'
     };
     const effectiveReasoningEffort = true === reasoningEnabled ? reasoningEffort ?? 'medium' : 'none';

package/dist/es/ai-model/models/gpt.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/gpt.mjs","sources":["../../../../src/ai-model/models/gpt.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ImageDetail,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { isLocateIntent } from './utils/intent';\n\nconst originalImageDetailForDefaultIntent = (\n  input: ChatCompletionCallContext,\n): ImageDetail | undefined =>\n  isLocateIntent(input.intent) || input.requiresOriginalImageDetail\n    ? 'original'\n    : undefined;\n\nconst buildGpt5ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // OpenAI Chat Completions JSON mode:\n  // https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat#json-mode\n  if (isLocateIntent(input.intent)) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const effectiveReasoningEffort =\n    reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'none';\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      reasoning_effort: effectiveReasoningEffort,\n    },\n  };\n};\n\nexport const gptAdapters = {\n  'gpt-5': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningBudget'],\n      buildChatCompletionParams: buildGpt5ChatCompletionParams,\n      resolveImageDetail: originalImageDetailForDefaultIntent,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy' },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'gpt-5'>;\n"],"names":["originalImageDetailForDefaultIntent","input","isLocateIntent","undefined","buildGpt5ChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","effectiveReasoningEffort","gptAdapters"],"mappings":";AASA,MAAMA,sCAAsC,CAC1CC,QAEAC,eAAeD,MAAM,MAAM,KAAKA,MAAM,2BAA2B,GAC7D,aACAE;AAEN,MAAMC,gCAAgC,CACpCH;IAEA,MAAM,EAAEI,gBAAgB,EAAEC,UAAU,EAAE,GAAGL;IACzC,MAAM,EAAEM,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BH,WAA3BG,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAK3D,IAAIJ,eAAeD,MAAM,MAAM,GAC7BQ,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAMC,2BACJH,AAAqB,SAArBA,mBAA6BC,mBAAmB,WAAY;IAE9D,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,kBAAkBC;QACpB;IACF;AACF;AAEO,MAAMC,cAAc;IACzB,SAAS;QACP,gBAAgB;YACd,uBAAuB;gBAAC;aAAkB;YAC1C,2BAA2BP;YAC3B,oBAAoBJ;QACtB;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;gBAAK;YAC5C;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/models/gpt.mjs","sources":["../../../../src/ai-model/models/gpt.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ImageDetail,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { isLocateIntent } from './utils/intent';\n\nconst originalImageDetailForDefaultIntent = (\n  input: ChatCompletionCallContext,\n): ImageDetail | undefined =>\n  isLocateIntent(input.intent) || input.requiresOriginalImageDetail\n    ? 'original'\n    : undefined;\n\nconst buildGpt5ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // OpenAI Chat Completions JSON mode:\n  // https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat#json-mode\n  if (\n    input.userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const effectiveReasoningEffort =\n    reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'none';\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      reasoning_effort: effectiveReasoningEffort,\n    },\n  };\n};\n\nexport const gptAdapters = {\n  'gpt-5': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningBudget'],\n      buildChatCompletionParams: buildGpt5ChatCompletionParams,\n      resolveImageDetail: originalImageDetailForDefaultIntent,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy' },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'gpt-5'>;\n"],"names":["originalImageDetailForDefaultIntent","input","isLocateIntent","undefined","buildGpt5ChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","effectiveReasoningEffort","gptAdapters"],"mappings":";AASA,MAAMA,sCAAsC,CAC1CC,QAEAC,eAAeD,MAAM,MAAM,KAAKA,MAAM,2BAA2B,GAC7D,aACAE;AAEN,MAAMC,gCAAgC,CACpCH;IAEA,MAAM,EAAEI,gBAAgB,EAAEC,UAAU,EAAE,GAAGL;IACzC,MAAM,EAAEM,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BH,WAA3BG,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAK3D,IACEL,AAAoC,WAApCA,MAAM,UAAU,CAAC,cAAc,IAC/BA,MAAM,0BAA0B,EAEhCQ,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAMC,2BACJH,AAAqB,SAArBA,mBAA6BC,mBAAmB,WAAY;IAE9D,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,kBAAkBC;QACpB;IACF;AACF;AAEO,MAAMC,cAAc;IACzB,SAAS;QACP,gBAAgB;YACd,uBAAuB;gBAAC;aAAkB;YAC1C,2BAA2BP;YAC3B,oBAAoBJ;QACtB;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;gBAAK;YAC5C;QACF;IACF;AACF"}

package/dist/es/ai-model/models/kimi.mjs

@@ -1,5 +1,4 @@
 import { createLocateResultValue, parseCoordinateList } from "../shared/model-locate-result/index.mjs";
-import { isLocateIntent } from "./utils/intent.mjs";
 const kimiNormalizedPointCoordinatesMeta = {
     shape: 'point',
     order: 'xy',
@@ -25,7 +24,7 @@ const buildKimiChatCompletionParams = (input)=>{
     const effectiveReasoningEnabled = reasoningEnabled ?? false;
     const commonOverrideConfig = {};
     commonOverrideConfig.temperature = void 0;
-    if (isLocateIntent(input.intent)) commonOverrideConfig.response_format = {
+    if ('none' !== userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
         type: 'json_object'
     };
     const modelSpecificConfig = {

package/dist/es/ai-model/models/kimi.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/kimi.mjs","sources":["../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  parseCoordinateList,\n} from '../shared/model-locate-result';\nimport { isLocateIntent } from './utils/intent';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1,\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n  const point = parseCoordinateList(input, 'point');\n  if (point.length < 2) {\n    throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n  }\n  const [x, y] = point;\n  // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n  // normalized coordinates, otherwise they are treated as screenshot pixels.\n  const coordinatesMeta =\n    x <= 1 && y <= 1\n      ? kimiNormalizedPointCoordinatesMeta\n      : kimiPixelPointCoordinatesMeta;\n  return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const effectiveReasoningEnabled = reasoningEnabled ?? false;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // kimi disallow custom temperature\n  commonOverrideConfig.temperature = undefined;\n\n  // Kimi Chat Completions response_format:\n  // https://platform.kimi.com/docs/api/chat\n  if (isLocateIntent(input.intent)) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const kimiAdapters = {\n  kimi: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimiChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: kimiNormalizedPointCoordinatesMeta,\n        parseRawLocateValue: parseKimiRawLocateValue,\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'kimi'>;\n"],"names":["kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","isLocateIntent","modelSpecificConfig","kimiAdapters"],"mappings":";;AAaA,MAAMA,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;AACT;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,oBAAoBF,OAAO;IACzC,IAAIC,MAAM,MAAM,GAAG,GACjB,MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,wBAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAInC,IAAIC,eAAehB,MAAM,MAAM,GAC7Bc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAMG,sBAA+C;QACnD,UAAU;YACR,MAAMJ,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGG,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BT;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAaZ;gBACb,qBAAqBE;YACvB;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/models/kimi.mjs","sources":["../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  parseCoordinateList,\n} from '../shared/model-locate-result';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1,\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n  const point = parseCoordinateList(input, 'point');\n  if (point.length < 2) {\n    throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n  }\n  const [x, y] = point;\n  // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n  // normalized coordinates, otherwise they are treated as screenshot pixels.\n  const coordinatesMeta =\n    x <= 1 && y <= 1\n      ? kimiNormalizedPointCoordinatesMeta\n      : kimiPixelPointCoordinatesMeta;\n  return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const effectiveReasoningEnabled = reasoningEnabled ?? false;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // kimi disallow custom temperature\n  commonOverrideConfig.temperature = undefined;\n\n  // Kimi Chat Completions response_format:\n  // https://platform.kimi.com/docs/api/chat\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const kimiAdapters = {\n  kimi: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimiChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: kimiNormalizedPointCoordinatesMeta,\n        parseRawLocateValue: parseKimiRawLocateValue,\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'kimi'>;\n"],"names":["kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","kimiAdapters"],"mappings":";AAYA,MAAMA,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;AACT;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,oBAAoBF,OAAO;IACzC,IAAIC,MAAM,MAAM,GAAG,GACjB,MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,wBAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAInC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C;QACnD,UAAU;YACR,MAAMH,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BR;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAaZ;gBACb,qBAAqBE;YACvB;QACF;IACF;AACF"}

package/dist/es/ai-model/models/mimo.mjs

@@ -1,9 +1,8 @@
-import { isLocateIntent } from "./utils/intent.mjs";
 const buildMimoChatCompletionParams = (input)=>{
-    const { intent, midsceneDefaults, userConfig } = input;
+    const { midsceneDefaults, userConfig } = input;
     const { reasoningEnabled } = userConfig;
     const commonOverrideConfig = {};
-    if (isLocateIntent(intent)) commonOverrideConfig.response_format = {
+    if ('none' !== userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
         type: 'json_object'
     };
     if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;

package/dist/es/ai-model/models/mimo.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/mimo.mjs","sources":["../../../../src/ai-model/models/mimo.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { isLocateIntent } from './utils/intent';\n\nconst buildMimoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { intent, midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // https://platform.xiaomimimo.com/docs/zh-CN/api/chat/openai-api\n  // Observed with thinking disabled: Mimo needs json_object to return JSON.\n  if (isLocateIntent(intent)) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const mimoAdapters = {\n  'xiaomi-mimo': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildMimoChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'xiaomi-mimo'>;\n"],"names":["buildMimoChatCompletionParams","input","intent","midsceneDefaults","userConfig","reasoningEnabled","commonOverrideConfig","isLocateIntent","undefined","modelSpecificConfig","mimoAdapters"],"mappings":";AAQA,MAAMA,gCAAgC,CACpCC;IAEA,MAAM,EAAEC,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGH;IACjD,MAAM,EAAEI,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,uBAAgD,CAAC;IAIvD,IAAIC,eAAeL,SACjBI,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,IAAIF,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAG3D,MAAMK,sBAA+C;QACnD,UAAU;YACR,MAAOJ,oBAAoB,QAAS,YAAY;QAClD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGF,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,GAAGG,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,eAAe;QACb,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BV;YAC3B,+BAA+B;QACjC;IACF;AACF"}
+{"version":3,"file":"ai-model/models/mimo.mjs","sources":["../../../../src/ai-model/models/mimo.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\n\nconst buildMimoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // https://platform.xiaomimimo.com/docs/zh-CN/api/chat/openai-api\n  // Observed with thinking disabled: Mimo needs json_object to return JSON.\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const mimoAdapters = {\n  'xiaomi-mimo': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildMimoChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'xiaomi-mimo'>;\n"],"names":["buildMimoChatCompletionParams","input","midsceneDefaults","userConfig","reasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","mimoAdapters"],"mappings":"AAOA,MAAMA,gCAAgC,CACpCC;IAEA,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGF;IACzC,MAAM,EAAEG,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,uBAAgD,CAAC;IAIvD,IACEF,AAA8B,WAA9BA,WAAW,cAAc,IACzBF,MAAM,0BAA0B,EAEhCI,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,IAAIF,AAA2BG,WAA3BH,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAG3D,MAAMI,sBAA+C;QACnD,UAAU;YACR,MAAOH,oBAAoB,QAAS,YAAY;QAClD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGF,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,eAAe;QACb,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BR;YAC3B,+BAA+B;QACjC;IACF;AACF"}

package/dist/es/ai-model/service-caller/index.mjs

@@ -4,7 +4,7 @@ import { assert, ifInBrowser } from "@midscene/shared/utils";
 import openai_0 from "openai";
 import { callAIWithCodexAppServer, isCodexAppServerProvider } from "./codex-app-server.mjs";
 import { formatOpenAIAPIErrorDetails, wrapOpenAICompatibleFetch } from "./openai-error.mjs";
-import { buildRequestAbortSignal, isHardTimeoutError, resolveEffectiveTimeoutMs } from "./request-timeout.mjs";
+import { buildRequestAbortSignal, isHardTimeoutError, resolveEffectiveTimeoutMs, restoreHardTimeoutError } from "./request-timeout.mjs";
 import { callAiAndParseWithRetry } from "./semantic-retry.mjs";
 import { extractJSONFromCodeBlock, parseModelResponseJson } from "./json.mjs";
 function _define_property(obj, key, value) {
@@ -211,9 +211,11 @@ async function callAI(messages, modelRuntime, options) {
             temperature: modelConfig.temperature,
             reasoningEnabled: modelConfig.reasoningEnabled,
             reasoningEffort: modelConfig.reasoningEffort,
-            reasoningBudget: modelConfig.reasoningBudget
+            reasoningBudget: modelConfig.reasoningBudget,
+            responseFormat: modelConfig.responseFormat
         },
-        requiresOriginalImageDetail: options?.requiresOriginalImageDetail
+        requiresOriginalImageDetail: options?.requiresOriginalImageDetail,
+        expectedJsonObjectResponse: options?.expectedJsonObjectResponse
     };
     const { config: adapterChatCompletionParams } = adapter.chatCompletion.buildChatCompletionParams(chatCompletionInput);
     debugCall(`adapter chat completion params: ${stringifyForDebug({
@@ -342,6 +344,8 @@ async function callAI(messages, modelRuntime, options) {
                         break;
                     }
                 }
+            } catch (error) {
+                throw restoreHardTimeoutError(toError(error), streamSignal);
             } finally{
                 cleanupStreamSignal();
             }
@@ -383,10 +387,10 @@ async function callAI(messages, modelRuntime, options) {
                     }
                     break;
                 } catch (error) {
-                    lastError = toError(error);
+                    lastError = restoreHardTimeoutError(toError(error), attemptSignal);
                     attemptErrors.push({
                         attempt,
-                        error
+                        error: lastError
                     });
                     const wasHardTimeout = isHardTimeoutError(lastError);
                     if (wasHardTimeout) warnCall(`AI call hit hard timeout (${effectiveTimeoutMs}ms, attempt ${attempt}/${maxAttempts}, model ${modelName}, slot ${modelConfig.slot})`);
@@ -436,7 +440,8 @@ async function callAIWithObjectResponse(messages, modelRuntime, options) {
     const { config: modelConfig, adapter } = modelRuntime;
     return callAiAndParseWithRetry({
         callAi: ()=>callAI(messages, modelRuntime, {
-                abortSignal: options?.abortSignal
+                abortSignal: options?.abortSignal,
+                expectedJsonObjectResponse: true
             }),
         parseResponse: (response)=>{
             assert(response, 'empty response');
@@ -456,7 +461,8 @@ async function callAIWithObjectResponse(messages, modelRuntime, options) {
             const errorMessage = error instanceof Error ? error.message : String(error);
             return new AIResponseParseError(errorMessage, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
         },
-        parseRetryTimes: options?.retryTimes,
+        parseRetryTimes: options?.retryTimes ?? modelConfig.retryCount,
+        parseRetryInterval: options?.retryInterval ?? modelConfig.retryInterval,
         abortSignal: options?.abortSignal
     });
 }