@midscene/core 1.13.1 → 1.13.2-beta-20260923031905.0

package/dist/es/ai-model/service-caller/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/utils.mjs","sources":["../../../../src/ai-model/service-caller/utils.ts"],"sourcesContent":["import type { AIUsageInfo } from '@/types';\nimport type OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { ImageDetail } from '../model-adapter/types';\nimport type { ModelRuntime } from '../models';\n\n// Error class that preserves usage and rawResponse when AI call parsing fails\nexport class AIResponseParseError extends Error {\n  usage?: AIUsageInfo;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  reasoningContent?: string;\n\n  constructor(\n    message: string,\n    rawResponse: string,\n    usage?: AIUsageInfo,\n    rawChoiceMessage?: unknown,\n    reasoningContent?: string,\n  ) {\n    super(message);\n    this.name = 'AIResponseParseError';\n    this.rawResponse = rawResponse;\n    this.usage = usage;\n    this.rawChoiceMessage = rawChoiceMessage;\n    this.reasoningContent = reasoningContent;\n  }\n}\n\n/**\n * Internal field name stamped onto every AIUsageInfo shaped by callAI().\n * Used for cross-path dedup when the provider does not return a request_id.\n */\nexport const INTERNAL_CALL_ID_FIELD = '_midscene_call_id';\n\nlet internalCallIdCounter = 0;\nexport function nextInternalCallId(): string {\n  internalCallIdCounter += 1;\n  return `call_${internalCallIdCounter}`;\n}\n\nexport function stringifyForDebug(value: unknown): string {\n  try {\n    return JSON.stringify(value);\n  } catch (_error) {\n    return String(value);\n  }\n}\n\nfunction getErrorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nexport function toError(error: unknown): Error {\n  return error instanceof Error ? error : new Error(String(error));\n}\n\nexport function normalizeRetryCount(retryCount: unknown): number {\n  if (typeof retryCount !== 'number' || !Number.isFinite(retryCount)) {\n    return 1;\n  }\n\n  return Math.max(0, Math.floor(retryCount));\n}\n\nexport function appendAIRequestFailureSummary<T extends Error>(\n  error: T,\n  attemptErrors: Array<{ attempt: number; error: unknown }>,\n  maxAttempts: number,\n): T {\n  const failedAttempts = attemptErrors.length;\n  const retries = Math.max(0, failedAttempts - 1);\n  const retryLabel = retries === 1 ? 'retry' : 'retries';\n  const originalMessage = error.message;\n  const previousAttemptErrors = attemptErrors.slice(0, -1);\n\n  error.message = `AI model request failed after ${retries} ${retryLabel} (${failedAttempts}/${maxAttempts} attempts). Last error: ${originalMessage}`;\n\n  if (previousAttemptErrors.length === 0) {\n    return error;\n  }\n\n  const details = previousAttemptErrors\n    .map(\n      ({ attempt, error }) => `Attempt ${attempt}: ${getErrorMessage(error)}`,\n    )\n    .join('\\n');\n\n  error.message = `${error.message}\\nPrevious AI call attempt errors:\\n${details}`;\n  return error;\n}\n\nexport const buildUsageInfo = ({\n  usageData,\n  requestId,\n  timeCost,\n  totalTimeCost,\n  retryCount,\n  modelName,\n  modelDescription,\n  responseModelName,\n  slot,\n  internalCallId,\n}: {\n  usageData?: OpenAI.CompletionUsage;\n  requestId?: string | null;\n  timeCost?: number;\n  totalTimeCost: number;\n  retryCount: number;\n  modelName: string;\n  modelDescription: string;\n  responseModelName?: string;\n  slot: ModelRuntime['config']['slot'];\n  internalCallId: string;\n}): AIUsageInfo | undefined => {\n  if (!usageData) return undefined;\n\n  const cachedInputTokens = (\n    usageData as { prompt_tokens_details?: { cached_tokens?: number } }\n  )?.prompt_tokens_details?.cached_tokens;\n\n  return {\n    ...usageData,\n    prompt_tokens: usageData.prompt_tokens ?? 0,\n    completion_tokens: usageData.completion_tokens ?? 0,\n    total_tokens: usageData.total_tokens ?? 0,\n    cached_input: cachedInputTokens ?? 0,\n    time_cost: timeCost ?? 0,\n    total_time_cost: totalTimeCost,\n    retry_count: retryCount,\n    model_name: modelName,\n    model_description: modelDescription,\n    response_model_name: responseModelName,\n    slot,\n    // Left undefined at the raw call layer. The agent's onUsage callback\n    // fills it from modelConfig.slot for metrics collection, and task\n    // layers use withUsageIntent() to stamp a more specific semantic\n    // intent (e.g. 'planning', 'insight') when attaching usage to tasks.\n    intent: undefined,\n    request_id: requestId ?? undefined,\n    // Internal stable ID for cross-path dedup when request_id is absent.\n    [INTERNAL_CALL_ID_FIELD]: internalCallId,\n  } satisfies AIUsageInfo;\n};\n\nexport const applyImageDetail = ({\n  imageDetail,\n  messages,\n}: {\n  imageDetail?: ImageDetail;\n  messages: ChatCompletionMessageParam[];\n}): ChatCompletionMessageParam[] => {\n  if (!imageDetail) {\n    return messages;\n  }\n\n  return messages.map((msg) => {\n    if (!Array.isArray(msg.content)) {\n      return msg;\n    }\n\n    const content = msg.content.map((part) => {\n      if (part && part.type === 'image_url' && part.image_url?.url) {\n        return {\n          ...part,\n          image_url: {\n            ...part.image_url,\n            detail: imageDetail,\n          },\n        };\n      }\n      return part;\n    });\n\n    return {\n      ...msg,\n      content,\n    } as ChatCompletionMessageParam;\n  });\n};\n\nexport const hasUsableText = (\n  value: string | null | undefined,\n): value is string => typeof value === 'string' && value.trim().length > 0;\n"],"names":["AIResponseParseError","Error","message","rawResponse","usage","rawChoiceMessage","reasoningContent","INTERNAL_CALL_ID_FIELD","internalCallIdCounter","nextInternalCallId","stringifyForDebug","value","JSON","_error","String","getErrorMessage","error","toError","normalizeRetryCount","retryCount","Number","Math","appendAIRequestFailureSummary","attemptErrors","maxAttempts","failedAttempts","retries","retryLabel","originalMessage","previousAttemptErrors","details","attempt","buildUsageInfo","usageData","requestId","timeCost","totalTimeCost","modelName","modelDescription","responseModelName","slot","internalCallId","cachedInputTokens","undefined","applyImageDetail","imageDetail","messages","msg","Array","content","part","hasUsableText"],"mappings":";;;;;;;;;;AAOO,MAAMA,6BAA6BC;IAUxC,YACEC,OAAe,EACfC,WAAmB,EACnBC,KAAmB,EACnBC,gBAA0B,EAC1BC,gBAAyB,CACzB;QACA,KAAK,CAACJ,UAhBR,yCAKA,+CACA,oDACA;QAUE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,WAAW,GAAGC;QACnB,IAAI,CAAC,KAAK,GAAGC;QACb,IAAI,CAAC,gBAAgB,GAAGC;QACxB,IAAI,CAAC,gBAAgB,GAAGC;IAC1B;AACF;AAMO,MAAMC,yBAAyB;AAEtC,IAAIC,wBAAwB;AACrB,SAASC;IACdD,yBAAyB;IACzB,OAAO,CAAC,KAAK,EAAEA,uBAAuB;AACxC;AAEO,SAASE,kBAAkBC,KAAc;IAC9C,IAAI;QACF,OAAOC,KAAK,SAAS,CAACD;IACxB,EAAE,OAAOE,QAAQ;QACf,OAAOC,OAAOH;IAChB;AACF;AAEA,SAASI,gBAAgBC,KAAc;IACrC,OAAOA,iBAAiBf,QAAQe,MAAM,OAAO,GAAGF,OAAOE;AACzD;AAEO,SAASC,QAAQD,KAAc;IACpC,OAAOA,iBAAiBf,QAAQe,QAAQ,IAAIf,MAAMa,OAAOE;AAC3D;AAEO,SAASE,oBAAoBC,UAAmB;IACrD,IAAI,AAAsB,YAAtB,OAAOA,cAA2B,CAACC,OAAO,QAAQ,CAACD,aACrD,OAAO;IAGT,OAAOE,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACF;AAChC;AAEO,SAASG,8BACdN,KAAQ,EACRO,aAAyD,EACzDC,WAAmB;IAEnB,MAAMC,iBAAiBF,cAAc,MAAM;IAC3C,MAAMG,UAAUL,KAAK,GAAG,CAAC,GAAGI,iBAAiB;IAC7C,MAAME,aAAaD,AAAY,MAAZA,UAAgB,UAAU;IAC7C,MAAME,kBAAkBZ,MAAM,OAAO;IACrC,MAAMa,wBAAwBN,cAAc,KAAK,CAAC,GAAG;IAErDP,MAAM,OAAO,GAAG,CAAC,8BAA8B,EAAEU,QAAQ,CAAC,EAAEC,WAAW,EAAE,EAAEF,eAAe,CAAC,EAAED,YAAY,wBAAwB,EAAEI,iBAAiB;IAEpJ,IAAIC,AAAiC,MAAjCA,sBAAsB,MAAM,EAC9B,OAAOb;IAGT,MAAMc,UAAUD,sBACb,GAAG,CACF,CAAC,EAAEE,OAAO,EAAEf,KAAK,EAAE,GAAK,CAAC,QAAQ,EAAEe,QAAQ,EAAE,EAAEhB,gBAAgBC,QAAQ,EAExE,IAAI,CAAC;IAERA,MAAM,OAAO,GAAG,GAAGA,MAAM,OAAO,CAAC,oCAAoC,EAAEc,SAAS;IAChF,OAAOd;AACT;AAEO,MAAMgB,iBAAiB,CAAC,EAC7BC,SAAS,EACTC,SAAS,EACTC,QAAQ,EACRC,aAAa,EACbjB,UAAU,EACVkB,SAAS,EACTC,gBAAgB,EAChBC,iBAAiB,EACjBC,IAAI,EACJC,cAAc,EAYf;IACC,IAAI,CAACR,WAAW;IAEhB,MAAMS,oBACJT,WACC,uBAAuB;IAE1B,OAAO;QACL,GAAGA,SAAS;QACZ,eAAeA,UAAU,aAAa,IAAI;QAC1C,mBAAmBA,UAAU,iBAAiB,IAAI;QAClD,cAAcA,UAAU,YAAY,IAAI;QACxC,cAAcS,qBAAqB;QACnC,WAAWP,YAAY;QACvB,iBAAiBC;QACjB,aAAajB;QACb,YAAYkB;QACZ,mBAAmBC;QACnB,qBAAqBC;QACrBC;QAKA,QAAQG;QACR,YAAYT,aAAaS;QAEzB,CAACpC,uBAAuB,EAAEkC;IAC5B;AACF;AAEO,MAAMG,mBAAmB,CAAC,EAC/BC,WAAW,EACXC,QAAQ,EAIT;IACC,IAAI,CAACD,aACH,OAAOC;IAGT,OAAOA,SAAS,GAAG,CAAC,CAACC;QACnB,IAAI,CAACC,MAAM,OAAO,CAACD,IAAI,OAAO,GAC5B,OAAOA;QAGT,MAAME,UAAUF,IAAI,OAAO,CAAC,GAAG,CAAC,CAACG;YAC/B,IAAIA,QAAQA,AAAc,gBAAdA,KAAK,IAAI,IAAoBA,KAAK,SAAS,EAAE,KACvD,OAAO;gBACL,GAAGA,IAAI;gBACP,WAAW;oBACT,GAAGA,KAAK,SAAS;oBACjB,QAAQL;gBACV;YACF;YAEF,OAAOK;QACT;QAEA,OAAO;YACL,GAAGH,GAAG;YACNE;QACF;IACF;AACF;AAEO,MAAME,gBAAgB,CAC3BxC,QACoB,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,IAAI,GAAG,MAAM,GAAG"}
+{"version":3,"file":"ai-model/service-caller/utils.mjs","sources":["../../../../src/ai-model/service-caller/utils.ts"],"sourcesContent":["import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo } from '@/types';\nimport type OpenAI from 'openai';\nimport type { ModelRuntime } from '../models';\n\n// Error class that preserves usage and rawResponse when AI call parsing fails\nexport class AIResponseParseError extends Error {\n  usage?: AIUsageInfo;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  reasoningContent?: string;\n\n  constructor(\n    message: string,\n    rawResponse: string,\n    usage?: AIUsageInfo,\n    rawAssistantOutput?: RawAssistantOutput,\n    reasoningContent?: string,\n  ) {\n    super(message);\n    this.name = 'AIResponseParseError';\n    this.rawResponse = rawResponse;\n    this.usage = usage;\n    this.rawAssistantOutput = rawAssistantOutput;\n    this.reasoningContent = reasoningContent;\n  }\n}\n\n/**\n * Internal field name stamped onto every AIUsageInfo shaped by callAI().\n * Used for cross-path dedup when the provider does not return a request_id.\n */\nexport const INTERNAL_CALL_ID_FIELD = '_midscene_call_id';\n\nlet internalCallIdCounter = 0;\nexport function nextInternalCallId(): string {\n  internalCallIdCounter += 1;\n  return `call_${internalCallIdCounter}`;\n}\n\nexport function stringifyForDebug(value: unknown): string {\n  try {\n    return JSON.stringify(value);\n  } catch (_error) {\n    return String(value);\n  }\n}\n\nfunction getErrorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nexport function toError(error: unknown): Error {\n  return error instanceof Error ? error : new Error(String(error));\n}\n\nexport function normalizeRetryCount(retryCount: unknown): number {\n  if (typeof retryCount !== 'number' || !Number.isFinite(retryCount)) {\n    return 1;\n  }\n\n  return Math.max(0, Math.floor(retryCount));\n}\n\nexport function appendAIRequestFailureSummary<T extends Error>(\n  error: T,\n  attemptErrors: Array<{ attempt: number; error: unknown }>,\n  maxAttempts: number,\n): T {\n  const failedAttempts = attemptErrors.length;\n  const retries = Math.max(0, failedAttempts - 1);\n  const retryLabel = retries === 1 ? 'retry' : 'retries';\n  const originalMessage = error.message;\n  const previousAttemptErrors = attemptErrors.slice(0, -1);\n\n  error.message = `AI model request failed after ${retries} ${retryLabel} (${failedAttempts}/${maxAttempts} attempts). Last error: ${originalMessage}`;\n\n  if (previousAttemptErrors.length === 0) {\n    return error;\n  }\n\n  const details = previousAttemptErrors\n    .map(\n      ({ attempt, error }) => `Attempt ${attempt}: ${getErrorMessage(error)}`,\n    )\n    .join('\\n');\n\n  error.message = `${error.message}\\nPrevious AI call attempt errors:\\n${details}`;\n  return error;\n}\n\nexport const buildUsageInfo = ({\n  apiType,\n  usageData,\n  requestId,\n  timeCost,\n  totalTimeCost,\n  retryCount,\n  modelName,\n  modelDescription,\n  responseModelName,\n  slot,\n  internalCallId,\n}: {\n  apiType: NonNullable<AIUsageInfo['api_type']>;\n  usageData?: OpenAI.CompletionUsage;\n  requestId?: string | null;\n  timeCost?: number;\n  totalTimeCost: number;\n  retryCount: number;\n  modelName: string;\n  modelDescription: string;\n  responseModelName?: string;\n  slot: ModelRuntime['config']['slot'];\n  internalCallId: string;\n}): AIUsageInfo | undefined => {\n  if (!usageData) return undefined;\n\n  const cachedInputTokens = (\n    usageData as { prompt_tokens_details?: { cached_tokens?: number } }\n  )?.prompt_tokens_details?.cached_tokens;\n\n  return {\n    ...usageData,\n    api_type: apiType,\n    prompt_tokens: usageData.prompt_tokens ?? 0,\n    completion_tokens: usageData.completion_tokens ?? 0,\n    total_tokens: usageData.total_tokens ?? 0,\n    cached_input: cachedInputTokens ?? 0,\n    time_cost: timeCost ?? 0,\n    total_time_cost: totalTimeCost,\n    retry_count: retryCount,\n    model_name: modelName,\n    model_description: modelDescription,\n    response_model_name: responseModelName,\n    slot,\n    // Left undefined at the raw call layer. The agent's onUsage callback\n    // fills it from modelConfig.slot for metrics collection, and task\n    // layers use withUsageIntent() to stamp a more specific semantic\n    // intent (e.g. 'planning', 'insight') when attaching usage to tasks.\n    intent: undefined,\n    request_id: requestId ?? undefined,\n    // Internal stable ID for cross-path dedup when request_id is absent.\n    [INTERNAL_CALL_ID_FIELD]: internalCallId,\n  } satisfies AIUsageInfo;\n};\n\nexport const hasUsableText = (\n  value: string | null | undefined,\n): value is string => typeof value === 'string' && value.trim().length > 0;\n"],"names":["AIResponseParseError","Error","message","rawResponse","usage","rawAssistantOutput","reasoningContent","INTERNAL_CALL_ID_FIELD","internalCallIdCounter","nextInternalCallId","stringifyForDebug","value","JSON","_error","String","getErrorMessage","error","toError","normalizeRetryCount","retryCount","Number","Math","appendAIRequestFailureSummary","attemptErrors","maxAttempts","failedAttempts","retries","retryLabel","originalMessage","previousAttemptErrors","details","attempt","buildUsageInfo","apiType","usageData","requestId","timeCost","totalTimeCost","modelName","modelDescription","responseModelName","slot","internalCallId","cachedInputTokens","undefined","hasUsableText"],"mappings":";;;;;;;;;;AAMO,MAAMA,6BAA6BC;IAUxC,YACEC,OAAe,EACfC,WAAmB,EACnBC,KAAmB,EACnBC,kBAAuC,EACvCC,gBAAyB,CACzB;QACA,KAAK,CAACJ,UAhBR,yCAKA,+CACA,sDACA;QAUE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,WAAW,GAAGC;QACnB,IAAI,CAAC,KAAK,GAAGC;QACb,IAAI,CAAC,kBAAkB,GAAGC;QAC1B,IAAI,CAAC,gBAAgB,GAAGC;IAC1B;AACF;AAMO,MAAMC,yBAAyB;AAEtC,IAAIC,wBAAwB;AACrB,SAASC;IACdD,yBAAyB;IACzB,OAAO,CAAC,KAAK,EAAEA,uBAAuB;AACxC;AAEO,SAASE,kBAAkBC,KAAc;IAC9C,IAAI;QACF,OAAOC,KAAK,SAAS,CAACD;IACxB,EAAE,OAAOE,QAAQ;QACf,OAAOC,OAAOH;IAChB;AACF;AAEA,SAASI,gBAAgBC,KAAc;IACrC,OAAOA,iBAAiBf,QAAQe,MAAM,OAAO,GAAGF,OAAOE;AACzD;AAEO,SAASC,QAAQD,KAAc;IACpC,OAAOA,iBAAiBf,QAAQe,QAAQ,IAAIf,MAAMa,OAAOE;AAC3D;AAEO,SAASE,oBAAoBC,UAAmB;IACrD,IAAI,AAAsB,YAAtB,OAAOA,cAA2B,CAACC,OAAO,QAAQ,CAACD,aACrD,OAAO;IAGT,OAAOE,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACF;AAChC;AAEO,SAASG,8BACdN,KAAQ,EACRO,aAAyD,EACzDC,WAAmB;IAEnB,MAAMC,iBAAiBF,cAAc,MAAM;IAC3C,MAAMG,UAAUL,KAAK,GAAG,CAAC,GAAGI,iBAAiB;IAC7C,MAAME,aAAaD,AAAY,MAAZA,UAAgB,UAAU;IAC7C,MAAME,kBAAkBZ,MAAM,OAAO;IACrC,MAAMa,wBAAwBN,cAAc,KAAK,CAAC,GAAG;IAErDP,MAAM,OAAO,GAAG,CAAC,8BAA8B,EAAEU,QAAQ,CAAC,EAAEC,WAAW,EAAE,EAAEF,eAAe,CAAC,EAAED,YAAY,wBAAwB,EAAEI,iBAAiB;IAEpJ,IAAIC,AAAiC,MAAjCA,sBAAsB,MAAM,EAC9B,OAAOb;IAGT,MAAMc,UAAUD,sBACb,GAAG,CACF,CAAC,EAAEE,OAAO,EAAEf,KAAK,EAAE,GAAK,CAAC,QAAQ,EAAEe,QAAQ,EAAE,EAAEhB,gBAAgBC,QAAQ,EAExE,IAAI,CAAC;IAERA,MAAM,OAAO,GAAG,GAAGA,MAAM,OAAO,CAAC,oCAAoC,EAAEc,SAAS;IAChF,OAAOd;AACT;AAEO,MAAMgB,iBAAiB,CAAC,EAC7BC,OAAO,EACPC,SAAS,EACTC,SAAS,EACTC,QAAQ,EACRC,aAAa,EACblB,UAAU,EACVmB,SAAS,EACTC,gBAAgB,EAChBC,iBAAiB,EACjBC,IAAI,EACJC,cAAc,EAaf;IACC,IAAI,CAACR,WAAW;IAEhB,MAAMS,oBACJT,WACC,uBAAuB;IAE1B,OAAO;QACL,GAAGA,SAAS;QACZ,UAAUD;QACV,eAAeC,UAAU,aAAa,IAAI;QAC1C,mBAAmBA,UAAU,iBAAiB,IAAI;QAClD,cAAcA,UAAU,YAAY,IAAI;QACxC,cAAcS,qBAAqB;QACnC,WAAWP,YAAY;QACvB,iBAAiBC;QACjB,aAAalB;QACb,YAAYmB;QACZ,mBAAmBC;QACnB,qBAAqBC;QACrBC;QAKA,QAAQG;QACR,YAAYT,aAAaS;QAEzB,CAACrC,uBAAuB,EAAEmC;IAC5B;AACF;AAEO,MAAMG,gBAAgB,CAC3BlC,QACoB,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,IAAI,GAAG,MAAM,GAAG"}

package/dist/es/ai-model/shared/multimodal-prompt.mjs

@@ -12,7 +12,7 @@ const prepareUserPrompt = async (userPrompt)=>({
         text: userPromptToString(userPrompt),
         referenceImages: await prepareReferenceImages(userPromptToMultimodalPrompt(userPrompt))
     });
-const preparedReferenceImagesToChatMessages = (referenceImages)=>{
+const preparedReferenceImagesToMessages = (referenceImages)=>{
     if (0 === referenceImages.length) return [];
     return [
         {
@@ -38,18 +38,15 @@ const preparedReferenceImagesToChatMessages = (referenceImages)=>{
                     role: 'user',
                     content: [
                         {
-                            type: 'image_url',
-                            image_url: {
-                                url: image.url,
-                                detail: 'high'
-                            }
+                            type: 'image',
+                            url: image.url
                         }
                     ]
                 }
             ])
     ];
 };
-const multimodalPromptToChatMessages = async (multimodalPrompt)=>preparedReferenceImagesToChatMessages(await prepareReferenceImages(multimodalPrompt));
-export { multimodalPromptToChatMessages, prepareUserPrompt, preparedReferenceImagesToChatMessages };
+const multimodalPromptToMessages = async (multimodalPrompt)=>preparedReferenceImagesToMessages(await prepareReferenceImages(multimodalPrompt));
+export { multimodalPromptToMessages, prepareUserPrompt, preparedReferenceImagesToMessages };
 
 //# sourceMappingURL=multimodal-prompt.mjs.map

package/dist/es/ai-model/shared/multimodal-prompt.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/multimodal-prompt.mjs","sources":["../../../../src/ai-model/shared/multimodal-prompt.ts"],"sourcesContent":["import {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport { preProcessImageUrl } from '@midscene/shared/img';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\n\nexport interface PreparedReferenceImage {\n  name: string;\n  url: string;\n}\n\nexport interface PreparedUserPrompt {\n  text: string;\n  referenceImages: PreparedReferenceImage[];\n}\n\nconst prepareReferenceImages = async (\n  multimodalPrompt?: TMultimodalPrompt,\n): Promise<PreparedReferenceImage[]> => {\n  const referenceImages: PreparedReferenceImage[] = [];\n  for (const image of multimodalPrompt?.images ?? []) {\n    referenceImages.push({\n      name: image.name,\n      url: await preProcessImageUrl(\n        image.url,\n        !!multimodalPrompt?.convertHttpImage2Base64,\n      ),\n    });\n  }\n  return referenceImages;\n};\n\nexport const prepareUserPrompt = async (\n  userPrompt: TUserPrompt,\n): Promise<PreparedUserPrompt> => ({\n  text: userPromptToString(userPrompt),\n  referenceImages: await prepareReferenceImages(\n    userPromptToMultimodalPrompt(userPrompt),\n  ),\n});\n\nexport const preparedReferenceImagesToChatMessages = (\n  referenceImages: PreparedReferenceImage[],\n): ChatCompletionUserMessageParam[] => {\n  if (referenceImages.length === 0) {\n    return [];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images. These reference images are supporting context only, not the current screenshot being evaluated, unless the task explicitly asks for comparison or matching.',\n        },\n      ],\n    },\n    ...referenceImages.flatMap((image): ChatCompletionUserMessageParam[] => [\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${image.name}'. It is a reference image, not the current screenshot:`,\n          },\n        ],\n      },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: image.url,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ]),\n  ];\n};\n\nexport const multimodalPromptToChatMessages = async (\n  multimodalPrompt?: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> =>\n  preparedReferenceImagesToChatMessages(\n    await prepareReferenceImages(multimodalPrompt),\n  );\n"],"names":["prepareReferenceImages","multimodalPrompt","referenceImages","image","preProcessImageUrl","prepareUserPrompt","userPrompt","userPromptToString","userPromptToMultimodalPrompt","preparedReferenceImagesToChatMessages","multimodalPromptToChatMessages"],"mappings":";;AAmBA,MAAMA,yBAAyB,OAC7BC;IAEA,MAAMC,kBAA4C,EAAE;IACpD,KAAK,MAAMC,SAASF,kBAAkB,UAAU,EAAE,CAChDC,gBAAgB,IAAI,CAAC;QACnB,MAAMC,MAAM,IAAI;QAChB,KAAK,MAAMC,mBACTD,MAAM,GAAG,EACT,CAAC,CAACF,kBAAkB;IAExB;IAEF,OAAOC;AACT;AAEO,MAAMG,oBAAoB,OAC/BC,aACiC;QACjC,MAAMC,mBAAmBD;QACzB,iBAAiB,MAAMN,uBACrBQ,6BAA6BF;IAEjC;AAEO,MAAMG,wCAAwC,CACnDP;IAEA,IAAIA,AAA2B,MAA3BA,gBAAgB,MAAM,EACxB,OAAO,EAAE;IAGX,OAAO;QACL;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;WACGA,gBAAgB,OAAO,CAAC,CAACC,QAA4C;gBACtE;oBACE,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,MAAM,CAAC,mCAAmC,EAAEA,MAAM,IAAI,CAAC,uDAAuD,CAAC;wBACjH;qBACD;gBACH;gBACA;oBACE,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,WAAW;gCACT,KAAKA,MAAM,GAAG;gCACd,QAAQ;4BACV;wBACF;qBACD;gBACH;aACD;KACF;AACH;AAEO,MAAMO,iCAAiC,OAC5CT,mBAEAQ,sCACE,MAAMT,uBAAuBC"}
+{"version":3,"file":"ai-model/shared/multimodal-prompt.mjs","sources":["../../../../src/ai-model/shared/multimodal-prompt.ts"],"sourcesContent":["import type { ConversationUserMessage } from '@/ai-model/service-caller/types';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport { preProcessImageUrl } from '@midscene/shared/img';\n\nexport interface PreparedReferenceImage {\n  name: string;\n  url: string;\n}\n\nexport interface PreparedUserPrompt {\n  text: string;\n  referenceImages: PreparedReferenceImage[];\n}\n\nconst prepareReferenceImages = async (\n  multimodalPrompt?: TMultimodalPrompt,\n): Promise<PreparedReferenceImage[]> => {\n  const referenceImages: PreparedReferenceImage[] = [];\n  for (const image of multimodalPrompt?.images ?? []) {\n    referenceImages.push({\n      name: image.name,\n      url: await preProcessImageUrl(\n        image.url,\n        !!multimodalPrompt?.convertHttpImage2Base64,\n      ),\n    });\n  }\n  return referenceImages;\n};\n\nexport const prepareUserPrompt = async (\n  userPrompt: TUserPrompt,\n): Promise<PreparedUserPrompt> => ({\n  text: userPromptToString(userPrompt),\n  referenceImages: await prepareReferenceImages(\n    userPromptToMultimodalPrompt(userPrompt),\n  ),\n});\n\nexport const preparedReferenceImagesToMessages = (\n  referenceImages: PreparedReferenceImage[],\n): ConversationUserMessage[] => {\n  if (referenceImages.length === 0) {\n    return [];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images. These reference images are supporting context only, not the current screenshot being evaluated, unless the task explicitly asks for comparison or matching.',\n        },\n      ],\n    },\n    ...referenceImages.flatMap((image): ConversationUserMessage[] => [\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${image.name}'. It is a reference image, not the current screenshot:`,\n          },\n        ],\n      },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image',\n            url: image.url,\n          },\n        ],\n      },\n    ]),\n  ];\n};\n\nexport const multimodalPromptToMessages = async (\n  multimodalPrompt?: TMultimodalPrompt,\n): Promise<ConversationUserMessage[]> =>\n  preparedReferenceImagesToMessages(\n    await prepareReferenceImages(multimodalPrompt),\n  );\n"],"names":["prepareReferenceImages","multimodalPrompt","referenceImages","image","preProcessImageUrl","prepareUserPrompt","userPrompt","userPromptToString","userPromptToMultimodalPrompt","preparedReferenceImagesToMessages","multimodalPromptToMessages"],"mappings":";;AAmBA,MAAMA,yBAAyB,OAC7BC;IAEA,MAAMC,kBAA4C,EAAE;IACpD,KAAK,MAAMC,SAASF,kBAAkB,UAAU,EAAE,CAChDC,gBAAgB,IAAI,CAAC;QACnB,MAAMC,MAAM,IAAI;QAChB,KAAK,MAAMC,mBACTD,MAAM,GAAG,EACT,CAAC,CAACF,kBAAkB;IAExB;IAEF,OAAOC;AACT;AAEO,MAAMG,oBAAoB,OAC/BC,aACiC;QACjC,MAAMC,mBAAmBD;QACzB,iBAAiB,MAAMN,uBACrBQ,6BAA6BF;IAEjC;AAEO,MAAMG,oCAAoC,CAC/CP;IAEA,IAAIA,AAA2B,MAA3BA,gBAAgB,MAAM,EACxB,OAAO,EAAE;IAGX,OAAO;QACL;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;WACGA,gBAAgB,OAAO,CAAC,CAACC,QAAqC;gBAC/D;oBACE,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,MAAM,CAAC,mCAAmC,EAAEA,MAAM,IAAI,CAAC,uDAAuD,CAAC;wBACjH;qBACD;gBACH;gBACA;oBACE,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,KAAKA,MAAM,GAAG;wBAChB;qBACD;gBACH;aACD;KACF;AACH;AAEO,MAAMO,6BAA6B,OACxCT,mBAEAQ,kCACE,MAAMT,uBAAuBC"}

package/dist/es/ai-model/workflows/grounding/locate.mjs

@@ -24,10 +24,10 @@ async function AiLocateElement(options) {
     const locateAdapter = options.modelRuntime.adapter.locate;
     const locateFn = 'custom' === locateAdapter.kind ? locateAdapter.locateFn : genericLocate;
     const locateResponse = await locateFn(locateRequest);
-    const { locatedPixelResult, rawResponse, rawChoiceMessage, usage, reasoningContent, errors = [] } = locateResponse;
+    const { locatedPixelResult, rawResponse, rawAssistantOutput, usage, reasoningContent, errors = [] } = locateResponse;
     const baseLocateResult = {
         rawResponse,
-        rawChoiceMessage,
+        rawAssistantOutput,
         usage,
         reasoning_content: reasoningContent
     };
@@ -99,7 +99,7 @@ async function genericLocate(locateRequest) {
                 const locateError = parsedLocateResult.error;
                 if ('not-found' === parsedLocateResult.kind) return {
                     rawResponse,
-                    rawChoiceMessage: response.rawChoiceMessage,
+                    rawAssistantOutput: response.rawAssistantOutput,
                     usage: response.usage,
                     reasoningContent: response.reasoning_content,
                     errors: locateError ? [
@@ -114,7 +114,7 @@ async function genericLocate(locateRequest) {
                     return {
                         locatedPixelResult: result,
                         rawResponse,
-                        rawChoiceMessage: response.rawChoiceMessage,
+                        rawAssistantOutput: response.rawAssistantOutput,
                         usage: response.usage,
                         reasoningContent: response.reasoning_content,
                         errors: []
@@ -133,7 +133,7 @@ async function genericLocate(locateRequest) {
             },
             toParseError: (error, response)=>{
                 const parseErrorMessage = error instanceof Error ? `Failed to parse locate result: ${error.message}` : 'unknown error in locate result';
-                return new AIResponseParseError(parseErrorMessage, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
+                return new AIResponseParseError(parseErrorMessage, response.content, response.usage, response.rawAssistantOutput, response.reasoning_content);
             },
             parseRetryTimes: modelRuntime.config.retryCount,
             parseRetryInterval: modelRuntime.config.retryInterval,
@@ -145,7 +145,7 @@ async function genericLocate(locateRequest) {
     } catch (callError) {
         if (callError instanceof AIResponseParseError) return {
             rawResponse: callError.rawResponse,
-            rawChoiceMessage: callError.rawChoiceMessage,
+            rawAssistantOutput: callError.rawAssistantOutput,
             usage: callError.usage,
             reasoningContent: callError.reasoningContent,
             errors: [

package/dist/es/ai-model/workflows/grounding/locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;AAmBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,oCACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,mBAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,OACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,yBAAyByB,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,kBAAkB4B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,kBAAkB4B,SAAS,gBAAgB;wBAC3C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,yBAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,qBACTD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;AAmBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,oCACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,mBAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,OACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,yBAAyByB,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,yBAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,qBACTD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/planning-action-locate.mjs

@@ -52,13 +52,13 @@ function resolvePlanningTapLocator(definition, planner) {
         let errors = [];
         let reasoningContent = '';
         let rawResponse = '';
-        let rawChoiceMessage;
+        let rawAssistantOutput;
         let usage;
         try {
             const locatePlanOptions = await buildPlanningTapLocatorPlanOptions(locateRequest);
             const planningResponse = await runCustomPlanning(await prepareUserPrompt(targetElementDescription), locatePlanOptions, locatorPlanner);
             rawResponse = planningResponse.rawResponse ?? '';
-            rawChoiceMessage = planningResponse.rawChoiceMessage;
+            rawAssistantOutput = planningResponse.rawAssistantOutput;
             usage = planningResponse.usage;
             reasoningContent = planningResponse.log;
             debugGrounding('planning-tap-locator rawResponse:', rawResponse);
@@ -67,7 +67,7 @@ function resolvePlanningTapLocator(definition, planner) {
             return {
                 locatedPixelResult,
                 rawResponse,
-                rawChoiceMessage,
+                rawAssistantOutput,
                 usage,
                 reasoningContent
             };
@@ -75,7 +75,7 @@ function resolvePlanningTapLocator(definition, planner) {
             const errorMessage = error instanceof Error ? error.message : String(error);
             if (error instanceof AIResponseParseError) {
                 rawResponse = error.rawResponse;
-                rawChoiceMessage = error.rawChoiceMessage;
+                rawAssistantOutput = error.rawAssistantOutput;
                 usage = error.usage;
             }
             errors = [
@@ -85,7 +85,7 @@ function resolvePlanningTapLocator(definition, planner) {
         }
         return {
             rawResponse,
-            rawChoiceMessage,
+            rawAssistantOutput,
             usage,
             reasoningContent,
             errors

package/dist/es/ai-model/workflows/grounding/planning-action-locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/planning-action-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/planning-action-locate.ts"],"sourcesContent":["import type { DeviceAction } from '@/device';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { z } from 'zod';\nimport { getMidsceneLocationSchema } from '../../../common';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type { ResolvedCustomPlanningDefinition } from '../../model-adapter/custom-planning-types';\nimport type { PlanningTapLocatorDefinition } from '../../model-adapter/types';\nimport { AIResponseParseError } from '../../service-caller/index';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\nimport { ConversationHistory } from '../planning/conversation-history';\nimport { runCustomPlanning } from '../planning/custom-planning';\nimport type { PlanOptions } from '../planning/types';\nimport type { LocateFn, LocateModelResponse, LocateRequest } from './types';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nconst planningActionLocatorActionSpace: DeviceAction[] = [\n  {\n    name: 'Tap',\n    description: 'Tap the element',\n    paramSchema: z.object({\n      locate: getMidsceneLocationSchema(),\n    }),\n    call: async () => undefined,\n  },\n];\n\nasync function buildPlanningTapLocatorPlanOptions(\n  locateRequest: LocateRequest,\n): Promise<PlanOptions> {\n  const { options, locateImage } = locateRequest;\n  const { context } = options;\n\n  return {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        locateImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: {\n        width: locateImage.width,\n        height: locateImage.height,\n      },\n    },\n    actionSpace: planningActionLocatorActionSpace,\n    conversationHistory: new ConversationHistory(),\n    includeLocateInPlanning: true,\n    effort: 'balance',\n  };\n}\n\nexport function resolvePlanningTapLocator<TParsed>(\n  definition: PlanningTapLocatorDefinition,\n  planner: ResolvedCustomPlanningDefinition<TParsed>,\n): LocateFn {\n  const locatorPlanner: ResolvedCustomPlanningDefinition<TParsed> = {\n    ...planner,\n    messages: {\n      ...planner.messages,\n      buildSystemPrompt: definition.buildSystemPrompt,\n      buildUserInstruction: (instruction) => `Tap: ${instruction}`,\n    },\n  };\n\n  return async (locateRequest: LocateRequest): Promise<LocateModelResponse> => {\n    const { targetElementDescription } = locateRequest;\n    assert(\n      targetElementDescription,\n      'cannot find the target element description',\n    );\n\n    let errors: string[] = [];\n    let reasoningContent = '';\n    let rawResponse = '';\n    let rawChoiceMessage: unknown;\n    let usage: LocateModelResponse['usage'];\n\n    try {\n      const locatePlanOptions =\n        await buildPlanningTapLocatorPlanOptions(locateRequest);\n      const planningResponse = await runCustomPlanning(\n        await prepareUserPrompt(targetElementDescription),\n        locatePlanOptions,\n        locatorPlanner,\n      );\n\n      rawResponse = planningResponse.rawResponse ?? '';\n      rawChoiceMessage = planningResponse.rawChoiceMessage;\n      usage = planningResponse.usage;\n      reasoningContent = planningResponse.log;\n\n      debugGrounding('planning-tap-locator rawResponse:', rawResponse);\n\n      const locatedPixelResult = definition.getLocatedPixelResult(\n        planningResponse.actions ?? [],\n      );\n\n      if (!locatedPixelResult) {\n        throw new Error('No locatedPixelResult found in planner response');\n      }\n\n      return {\n        locatedPixelResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoningContent,\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      if (error instanceof AIResponseParseError) {\n        rawResponse = error.rawResponse;\n        rawChoiceMessage = error.rawChoiceMessage;\n        usage = error.usage;\n      }\n      errors = [\n        errorMessage || 'Failed to parse planning tap locator response',\n      ];\n      debugGrounding('planning-tap-locator parse error:', errors[0]);\n    }\n\n    return {\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoningContent,\n      errors,\n    };\n  };\n}\n"],"names":["debugGrounding","getDebug","planningActionLocatorActionSpace","z","getMidsceneLocationSchema","undefined","buildPlanningTapLocatorPlanOptions","locateRequest","options","locateImage","context","ScreenshotItem","ConversationHistory","resolvePlanningTapLocator","definition","planner","locatorPlanner","instruction","targetElementDescription","assert","errors","reasoningContent","rawResponse","rawChoiceMessage","usage","locatePlanOptions","planningResponse","runCustomPlanning","prepareUserPrompt","locatedPixelResult","Error","error","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;;;AAeA,MAAMA,iBAAiBC,SAAS;AAEhC,MAAMC,mCAAmD;IACvD;QACE,MAAM;QACN,aAAa;QACb,aAAaC,EAAE,MAAM,CAAC;YACpB,QAAQC;QACV;QACA,MAAM,UAAYC;IACpB;CACD;AAED,eAAeC,mCACbC,aAA4B;IAE5B,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGF;IACjC,MAAM,EAAEG,OAAO,EAAE,GAAGF;IAEpB,OAAO;QACL,GAAGA,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYC,eAAe,MAAM,CAC/BF,YAAY,WAAW,EACvBC,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAU;gBACR,OAAOD,YAAY,KAAK;gBACxB,QAAQA,YAAY,MAAM;YAC5B;QACF;QACA,aAAaP;QACb,qBAAqB,IAAIU;QACzB,yBAAyB;QACzB,QAAQ;IACV;AACF;AAEO,SAASC,0BACdC,UAAwC,EACxCC,OAAkD;IAElD,MAAMC,iBAA4D;QAChE,GAAGD,OAAO;QACV,UAAU;YACR,GAAGA,QAAQ,QAAQ;YACnB,mBAAmBD,WAAW,iBAAiB;YAC/C,sBAAsB,CAACG,cAAgB,CAAC,KAAK,EAAEA,aAAa;QAC9D;IACF;IAEA,OAAO,OAAOV;QACZ,MAAM,EAAEW,wBAAwB,EAAE,GAAGX;QACrCY,OACED,0BACA;QAGF,IAAIE,SAAmB,EAAE;QACzB,IAAIC,mBAAmB;QACvB,IAAIC,cAAc;QAClB,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMC,oBACJ,MAAMnB,mCAAmCC;YAC3C,MAAMmB,mBAAmB,MAAMC,kBAC7B,MAAMC,kBAAkBV,2BACxBO,mBACAT;YAGFM,cAAcI,iBAAiB,WAAW,IAAI;YAC9CH,mBAAmBG,iBAAiB,gBAAgB;YACpDF,QAAQE,iBAAiB,KAAK;YAC9BL,mBAAmBK,iBAAiB,GAAG;YAEvC1B,eAAe,qCAAqCsB;YAEpD,MAAMO,qBAAqBf,WAAW,qBAAqB,CACzDY,iBAAiB,OAAO,IAAI,EAAE;YAGhC,IAAI,CAACG,oBACH,MAAM,IAAIC,MAAM;YAGlB,OAAO;gBACLD;gBACAP;gBACAC;gBACAC;gBACAH;YACF;QACF,EAAE,OAAOU,OAAO;YACd,MAAMC,eACJD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;YAClD,IAAIA,iBAAiBG,sBAAsB;gBACzCZ,cAAcS,MAAM,WAAW;gBAC/BR,mBAAmBQ,MAAM,gBAAgB;gBACzCP,QAAQO,MAAM,KAAK;YACrB;YACAX,SAAS;gBACPY,gBAAgB;aACjB;YACDhC,eAAe,qCAAqCoB,MAAM,CAAC,EAAE;QAC/D;QAEA,OAAO;YACLE;YACAC;YACAC;YACAH;YACAD;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/planning-action-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/planning-action-locate.ts"],"sourcesContent":["import type { DeviceAction } from '@/device';\nimport type { RawAssistantOutput } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { z } from 'zod';\nimport { getMidsceneLocationSchema } from '../../../common';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type { ResolvedCustomPlanningDefinition } from '../../model-adapter/custom-planning-types';\nimport type { PlanningTapLocatorDefinition } from '../../model-adapter/types';\nimport { AIResponseParseError } from '../../service-caller/index';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\nimport { ConversationHistory } from '../planning/conversation-history';\nimport { runCustomPlanning } from '../planning/custom-planning';\nimport type { PlanOptions } from '../planning/types';\nimport type { LocateFn, LocateModelResponse, LocateRequest } from './types';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nconst planningActionLocatorActionSpace: DeviceAction[] = [\n  {\n    name: 'Tap',\n    description: 'Tap the element',\n    paramSchema: z.object({\n      locate: getMidsceneLocationSchema(),\n    }),\n    call: async () => undefined,\n  },\n];\n\nasync function buildPlanningTapLocatorPlanOptions(\n  locateRequest: LocateRequest,\n): Promise<PlanOptions> {\n  const { options, locateImage } = locateRequest;\n  const { context } = options;\n\n  return {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        locateImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: {\n        width: locateImage.width,\n        height: locateImage.height,\n      },\n    },\n    actionSpace: planningActionLocatorActionSpace,\n    conversationHistory: new ConversationHistory(),\n    includeLocateInPlanning: true,\n    effort: 'balance',\n  };\n}\n\nexport function resolvePlanningTapLocator<TParsed>(\n  definition: PlanningTapLocatorDefinition,\n  planner: ResolvedCustomPlanningDefinition<TParsed>,\n): LocateFn {\n  const locatorPlanner: ResolvedCustomPlanningDefinition<TParsed> = {\n    ...planner,\n    messages: {\n      ...planner.messages,\n      buildSystemPrompt: definition.buildSystemPrompt,\n      buildUserInstruction: (instruction) => `Tap: ${instruction}`,\n    },\n  };\n\n  return async (locateRequest: LocateRequest): Promise<LocateModelResponse> => {\n    const { targetElementDescription } = locateRequest;\n    assert(\n      targetElementDescription,\n      'cannot find the target element description',\n    );\n\n    let errors: string[] = [];\n    let reasoningContent = '';\n    let rawResponse = '';\n    let rawAssistantOutput: RawAssistantOutput | undefined;\n    let usage: LocateModelResponse['usage'];\n\n    try {\n      const locatePlanOptions =\n        await buildPlanningTapLocatorPlanOptions(locateRequest);\n      const planningResponse = await runCustomPlanning(\n        await prepareUserPrompt(targetElementDescription),\n        locatePlanOptions,\n        locatorPlanner,\n      );\n\n      rawResponse = planningResponse.rawResponse ?? '';\n      rawAssistantOutput = planningResponse.rawAssistantOutput;\n      usage = planningResponse.usage;\n      reasoningContent = planningResponse.log;\n\n      debugGrounding('planning-tap-locator rawResponse:', rawResponse);\n\n      const locatedPixelResult = definition.getLocatedPixelResult(\n        planningResponse.actions ?? [],\n      );\n\n      if (!locatedPixelResult) {\n        throw new Error('No locatedPixelResult found in planner response');\n      }\n\n      return {\n        locatedPixelResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoningContent,\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      if (error instanceof AIResponseParseError) {\n        rawResponse = error.rawResponse;\n        rawAssistantOutput = error.rawAssistantOutput;\n        usage = error.usage;\n      }\n      errors = [\n        errorMessage || 'Failed to parse planning tap locator response',\n      ];\n      debugGrounding('planning-tap-locator parse error:', errors[0]);\n    }\n\n    return {\n      rawResponse,\n      rawAssistantOutput,\n      usage,\n      reasoningContent,\n      errors,\n    };\n  };\n}\n"],"names":["debugGrounding","getDebug","planningActionLocatorActionSpace","z","getMidsceneLocationSchema","undefined","buildPlanningTapLocatorPlanOptions","locateRequest","options","locateImage","context","ScreenshotItem","ConversationHistory","resolvePlanningTapLocator","definition","planner","locatorPlanner","instruction","targetElementDescription","assert","errors","reasoningContent","rawResponse","rawAssistantOutput","usage","locatePlanOptions","planningResponse","runCustomPlanning","prepareUserPrompt","locatedPixelResult","Error","error","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;;;AAgBA,MAAMA,iBAAiBC,SAAS;AAEhC,MAAMC,mCAAmD;IACvD;QACE,MAAM;QACN,aAAa;QACb,aAAaC,EAAE,MAAM,CAAC;YACpB,QAAQC;QACV;QACA,MAAM,UAAYC;IACpB;CACD;AAED,eAAeC,mCACbC,aAA4B;IAE5B,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGF;IACjC,MAAM,EAAEG,OAAO,EAAE,GAAGF;IAEpB,OAAO;QACL,GAAGA,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYC,eAAe,MAAM,CAC/BF,YAAY,WAAW,EACvBC,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAU;gBACR,OAAOD,YAAY,KAAK;gBACxB,QAAQA,YAAY,MAAM;YAC5B;QACF;QACA,aAAaP;QACb,qBAAqB,IAAIU;QACzB,yBAAyB;QACzB,QAAQ;IACV;AACF;AAEO,SAASC,0BACdC,UAAwC,EACxCC,OAAkD;IAElD,MAAMC,iBAA4D;QAChE,GAAGD,OAAO;QACV,UAAU;YACR,GAAGA,QAAQ,QAAQ;YACnB,mBAAmBD,WAAW,iBAAiB;YAC/C,sBAAsB,CAACG,cAAgB,CAAC,KAAK,EAAEA,aAAa;QAC9D;IACF;IAEA,OAAO,OAAOV;QACZ,MAAM,EAAEW,wBAAwB,EAAE,GAAGX;QACrCY,OACED,0BACA;QAGF,IAAIE,SAAmB,EAAE;QACzB,IAAIC,mBAAmB;QACvB,IAAIC,cAAc;QAClB,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMC,oBACJ,MAAMnB,mCAAmCC;YAC3C,MAAMmB,mBAAmB,MAAMC,kBAC7B,MAAMC,kBAAkBV,2BACxBO,mBACAT;YAGFM,cAAcI,iBAAiB,WAAW,IAAI;YAC9CH,qBAAqBG,iBAAiB,kBAAkB;YACxDF,QAAQE,iBAAiB,KAAK;YAC9BL,mBAAmBK,iBAAiB,GAAG;YAEvC1B,eAAe,qCAAqCsB;YAEpD,MAAMO,qBAAqBf,WAAW,qBAAqB,CACzDY,iBAAiB,OAAO,IAAI,EAAE;YAGhC,IAAI,CAACG,oBACH,MAAM,IAAIC,MAAM;YAGlB,OAAO;gBACLD;gBACAP;gBACAC;gBACAC;gBACAH;YACF;QACF,EAAE,OAAOU,OAAO;YACd,MAAMC,eACJD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;YAClD,IAAIA,iBAAiBG,sBAAsB;gBACzCZ,cAAcS,MAAM,WAAW;gBAC/BR,qBAAqBQ,MAAM,kBAAkB;gBAC7CP,QAAQO,MAAM,KAAK;YACrB;YACAX,SAAS;gBACPY,gBAAgB;aACjB;YACDhC,eAAe,qCAAqCoB,MAAM,CAAC,EAAE;QAC/D;QAEA,OAAO;YACLE;YACAC;YACAC;YACAH;YACAD;QACF;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/section-locate.mjs

@@ -76,7 +76,7 @@ async function AiLocateSection(options) {
             },
             toParseError: (error, response)=>{
                 const parseErrorMessage = error instanceof Error ? `Failed to parse section locate result: ${error.message}` : 'unknown error in section locate';
-                return new AIResponseParseError(parseErrorMessage, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
+                return new AIResponseParseError(parseErrorMessage, response.content, response.usage, response.rawAssistantOutput, response.reasoning_content);
             },
             parseRetryTimes: modelRuntime.config.retryCount,
             parseRetryInterval: modelRuntime.config.retryInterval,
@@ -90,7 +90,7 @@ async function AiLocateSection(options) {
             searchAreaConfig: void 0,
             error: callError.message,
             rawResponse: callError.rawResponse,
-            rawChoiceMessage: callError.rawChoiceMessage,
+            rawAssistantOutput: callError.rawAssistantOutput,
             usage: callError.usage
         };
         const errorMessage = callError instanceof Error ? callError.message : String(callError);
@@ -105,7 +105,7 @@ async function AiLocateSection(options) {
         searchAreaConfig: void 0,
         error: sectionError,
         rawResponse: result.content,
-        rawChoiceMessage: result.rawChoiceMessage,
+        rawAssistantOutput: result.rawAssistantOutput,
         usage: result.usage
     };
     try {
@@ -122,7 +122,7 @@ async function AiLocateSection(options) {
             searchAreaConfig,
             error: sectionError,
             rawResponse: result.content,
-            rawChoiceMessage: result.rawChoiceMessage,
+            rawAssistantOutput: result.rawAssistantOutput,
             usage: result.usage
         };
     } catch (error) {
@@ -132,7 +132,7 @@ async function AiLocateSection(options) {
             searchAreaConfig: void 0,
             error: errorMessage,
             rawResponse: result.content,
-            rawChoiceMessage: result.rawChoiceMessage,
+            rawAssistantOutput: result.rawAssistantOutput,
             usage: result.usage
         };
     }

package/dist/es/ai-model/workflows/grounding/section-locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;AAoBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,uBAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;AAqBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,uBAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/utils.mjs

@@ -1,6 +1,6 @@
 import { userPromptToMultimodalPrompt } from "../../../common.mjs";
 import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
-import { multimodalPromptToChatMessages } from "../../shared/multimodal-prompt.mjs";
+import { multimodalPromptToMessages } from "../../shared/multimodal-prompt.mjs";
 async function prepareLocateModelInput({ locateImage, targetDescription, systemPrompt, userPrompt, imagePreprocess, userMessageContentOrder }) {
     const preparedImage = await prepareModelImage({
         imageBase64: locateImage.imageBase64,
@@ -8,13 +8,10 @@ async function prepareLocateModelInput({ locateImage, targetDescription, systemP
         height: locateImage.height,
         policy: imagePreprocess
     });
-    const referenceImageMessages = await multimodalPromptToChatMessages(userPromptToMultimodalPrompt(targetDescription));
+    const referenceImageMessages = await multimodalPromptToMessages(userPromptToMultimodalPrompt(targetDescription));
     const imageContent = {
-        type: 'image_url',
-        image_url: {
-            url: preparedImage.imageBase64,
-            detail: 'high'
-        }
+        type: 'image',
+        url: preparedImage.imageBase64
     };
     const promptContent = {
         type: 'text',

package/dist/es/ai-model/workflows/grounding/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/utils.mjs","sources":["../../../../../src/ai-model/workflows/grounding/utils.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToMultimodalPrompt } from '@/common';\nimport type {\n  ChatCompletionContentPart,\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport {\n  type ImagePreprocessPolicy,\n  type PreparedModelImage,\n  prepareModelImage,\n} from '../../model-adapter/image-preprocess';\nimport type { LocateUserMessageContentOrder } from '../../model-adapter/types';\nimport type { ModelRuntime } from '../../models';\nimport { multimodalPromptToChatMessages } from '../../shared/multimodal-prompt';\n\nexport type GroundingAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nexport async function prepareLocateModelInput({\n  locateImage,\n  targetDescription,\n  systemPrompt,\n  userPrompt,\n  imagePreprocess,\n  userMessageContentOrder,\n}: {\n  locateImage: {\n    imageBase64: string;\n    width: number;\n    height: number;\n  };\n  targetDescription: TUserPrompt;\n  systemPrompt: string;\n  userPrompt: string;\n  imagePreprocess: ImagePreprocessPolicy;\n  userMessageContentOrder: LocateUserMessageContentOrder;\n}): Promise<{\n  messages: GroundingAIArgs;\n  preparedImage: PreparedModelImage;\n}> {\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateImage.imageBase64,\n    width: locateImage.width,\n    height: locateImage.height,\n    policy: imagePreprocess,\n  });\n  const referenceImageMessages = await multimodalPromptToChatMessages(\n    userPromptToMultimodalPrompt(targetDescription),\n  );\n  const imageContent: ChatCompletionContentPart = {\n    type: 'image_url',\n    image_url: {\n      url: preparedImage.imageBase64,\n      detail: 'high',\n    },\n  };\n  const promptContent: ChatCompletionContentPart = {\n    type: 'text',\n    text: userPrompt,\n  };\n\n  return {\n    preparedImage,\n    messages: [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content:\n          userMessageContentOrder === 'prompt-first'\n            ? [promptContent, imageContent]\n            : [imageContent, promptContent],\n      },\n      ...referenceImageMessages,\n    ],\n  };\n}\n\nexport function formatLocateModelContext(modelRuntime: ModelRuntime): string {\n  const { modelFamily, modelName } = modelRuntime.config;\n  return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;\n}\n"],"names":["prepareLocateModelInput","locateImage","targetDescription","systemPrompt","userPrompt","imagePreprocess","userMessageContentOrder","preparedImage","prepareModelImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","imageContent","promptContent","formatLocateModelContext","modelRuntime","modelFamily","modelName"],"mappings":";;;AAoBO,eAAeA,wBAAwB,EAC5CC,WAAW,EACXC,iBAAiB,EACjBC,YAAY,EACZC,UAAU,EACVC,eAAe,EACfC,uBAAuB,EAYxB;IAIC,MAAMC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,YAAY,WAAW;QACpC,OAAOA,YAAY,KAAK;QACxB,QAAQA,YAAY,MAAM;QAC1B,QAAQI;IACV;IACA,MAAMI,yBAAyB,MAAMC,+BACnCC,6BAA6BT;IAE/B,MAAMU,eAA0C;QAC9C,MAAM;QACN,WAAW;YACT,KAAKL,cAAc,WAAW;YAC9B,QAAQ;QACV;IACF;IACA,MAAMM,gBAA2C;QAC/C,MAAM;QACN,MAAMT;IACR;IAEA,OAAO;QACLG;QACA,UAAU;YACR;gBAAE,MAAM;gBAAU,SAASJ;YAAa;YACxC;gBACE,MAAM;gBACN,SACEG,AAA4B,mBAA5BA,0BACI;oBAACO;oBAAeD;iBAAa,GAC7B;oBAACA;oBAAcC;iBAAc;YACrC;eACGJ;SACJ;IACH;AACF;AAEO,SAASK,yBAAyBC,YAA0B;IACjE,MAAM,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGF,aAAa,MAAM;IACtD,OAAO,CAAC,UAAU,EAAEE,aAAa,QAAQ,aAAa,EAAED,eAAe,SAAS;AAClF"}
+{"version":3,"file":"ai-model/workflows/grounding/utils.mjs","sources":["../../../../../src/ai-model/workflows/grounding/utils.ts"],"sourcesContent":["import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n  MessageContent,\n} from '@/ai-model/service-caller/types';\nimport { type TUserPrompt, userPromptToMultimodalPrompt } from '@/common';\nimport {\n  type ImagePreprocessPolicy,\n  type PreparedModelImage,\n  prepareModelImage,\n} from '../../model-adapter/image-preprocess';\nimport type { LocateUserMessageContentOrder } from '../../model-adapter/types';\nimport type { ModelRuntime } from '../../models';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\n\nexport type GroundingAIArgs = [\n  ConversationSystemMessage,\n  ...ConversationUserMessage[],\n];\n\nexport async function prepareLocateModelInput({\n  locateImage,\n  targetDescription,\n  systemPrompt,\n  userPrompt,\n  imagePreprocess,\n  userMessageContentOrder,\n}: {\n  locateImage: {\n    imageBase64: string;\n    width: number;\n    height: number;\n  };\n  targetDescription: TUserPrompt;\n  systemPrompt: string;\n  userPrompt: string;\n  imagePreprocess: ImagePreprocessPolicy;\n  userMessageContentOrder: LocateUserMessageContentOrder;\n}): Promise<{\n  messages: GroundingAIArgs;\n  preparedImage: PreparedModelImage;\n}> {\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateImage.imageBase64,\n    width: locateImage.width,\n    height: locateImage.height,\n    policy: imagePreprocess,\n  });\n  const referenceImageMessages = await multimodalPromptToMessages(\n    userPromptToMultimodalPrompt(targetDescription),\n  );\n  const imageContent: MessageContent = {\n    type: 'image',\n    url: preparedImage.imageBase64,\n  };\n  const promptContent: MessageContent = {\n    type: 'text',\n    text: userPrompt,\n  };\n\n  return {\n    preparedImage,\n    messages: [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content:\n          userMessageContentOrder === 'prompt-first'\n            ? [promptContent, imageContent]\n            : [imageContent, promptContent],\n      },\n      ...referenceImageMessages,\n    ],\n  };\n}\n\nexport function formatLocateModelContext(modelRuntime: ModelRuntime): string {\n  const { modelFamily, modelName } = modelRuntime.config;\n  return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;\n}\n"],"names":["prepareLocateModelInput","locateImage","targetDescription","systemPrompt","userPrompt","imagePreprocess","userMessageContentOrder","preparedImage","prepareModelImage","referenceImageMessages","multimodalPromptToMessages","userPromptToMultimodalPrompt","imageContent","promptContent","formatLocateModelContext","modelRuntime","modelFamily","modelName"],"mappings":";;;AAoBO,eAAeA,wBAAwB,EAC5CC,WAAW,EACXC,iBAAiB,EACjBC,YAAY,EACZC,UAAU,EACVC,eAAe,EACfC,uBAAuB,EAYxB;IAIC,MAAMC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,YAAY,WAAW;QACpC,OAAOA,YAAY,KAAK;QACxB,QAAQA,YAAY,MAAM;QAC1B,QAAQI;IACV;IACA,MAAMI,yBAAyB,MAAMC,2BACnCC,6BAA6BT;IAE/B,MAAMU,eAA+B;QACnC,MAAM;QACN,KAAKL,cAAc,WAAW;IAChC;IACA,MAAMM,gBAAgC;QACpC,MAAM;QACN,MAAMT;IACR;IAEA,OAAO;QACLG;QACA,UAAU;YACR;gBAAE,MAAM;gBAAU,SAASJ;YAAa;YACxC;gBACE,MAAM;gBACN,SACEG,AAA4B,mBAA5BA,0BACI;oBAACO;oBAAeD;iBAAa,GAC7B;oBAACA;oBAAcC;iBAAc;YACrC;eACGJ;SACJ;IACH;AACF;AAEO,SAASK,yBAAyBC,YAA0B;IACjE,MAAM,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGF,aAAa,MAAM;IACtD,OAAO,CAAC,UAAU,EAAEE,aAAa,QAAQ,aAAa,EAAED,eAAe,SAAS;AAClF"}

package/dist/es/ai-model/workflows/insight/extraction.mjs

@@ -3,7 +3,7 @@ import { renderAIContext } from "../../../agent/prompt-context.mjs";
 import { buildInsightSystemPrompt, extractDataQueryPrompt } from "../../prompt/insight/index.mjs";
 import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
-import { multimodalPromptToChatMessages } from "../../shared/multimodal-prompt.mjs";
+import { multimodalPromptToMessages } from "../../shared/multimodal-prompt.mjs";
 import { parseInsightResponse } from "./insight-response-parser.mjs";
 const debugInsight = getDebug('ai:insight');
 async function AiExtractElementInfo(options) {
@@ -31,11 +31,8 @@ async function AiExtractElementInfo(options) {
                     text: `Frame ${index + 1}/${screenshotSequence.length}`
                 });
                 userContent.push({
-                    type: 'image_url',
-                    image_url: {
-                        url: frame.base64,
-                        detail: 'high'
-                    }
+                    type: 'image',
+                    url: frame.base64
                 });
             });
         } else {
@@ -44,11 +41,8 @@ async function AiExtractElementInfo(options) {
                 text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.'
             });
             userContent.push({
-                type: 'image_url',
-                image_url: {
-                    url: screenshotBase64,
-                    detail: 'high'
-                }
+                type: 'image',
+                url: screenshotBase64
             });
         }
     }
@@ -67,7 +61,7 @@ async function AiExtractElementInfo(options) {
         }
     ];
     if (multimodalPrompt) {
-        const addOns = await multimodalPromptToChatMessages(multimodalPrompt);
+        const addOns = await multimodalPromptToMessages(multimodalPrompt);
         msgs.push(...addOns);
     }
     return callAiAndParseWithRetry({
@@ -76,19 +70,19 @@ async function AiExtractElementInfo(options) {
                 semanticRetryAttempt: retryAttempt
             }),
         parseResponse: (response)=>{
-            const { content: rawResponse, usage, reasoning_content, rawChoiceMessage } = response;
+            const { content: rawResponse, usage, reasoning_content, rawAssistantOutput } = response;
             const parseResult = parseInsightResponse(rawResponse, insightProtocol.dataOutput, modelRuntime.adapter.jsonParser);
             return {
                 parseResult,
                 rawResponse,
-                rawChoiceMessage,
+                rawAssistantOutput,
                 usage,
                 reasoning_content
             };
         },
         toParseError: (parseError, response)=>{
             const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
-            return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawChoiceMessage);
+            return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawAssistantOutput);
         },
         parseRetryTimes: modelRuntime.config.retryCount,
         parseRetryInterval: modelRuntime.config.retryInterval,

package/dist/es/ai-model/workflows/insight/extraction.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/extraction.mjs","sources":["../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["import type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToChatMessages } from '../../shared/multimodal-prompt';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["debugInsight","getDebug","AiExtractElementInfo","options","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToChatMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawChoiceMessage","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;;;;;AA0BA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,qBAAwBC,OAQ7C;IACC,MAAM,EAAEC,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEL;IACF,MAAMM,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,yBAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,gBAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,uBAC5Bb,QAAQ,eAAe,IAAI,IAC3BC,WACAS;IAGF,MAAMI,cAAyD,EAAE;IAEjE,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKL;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,+BAA+BhB;QACpDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaL,QAAQ,WAAW;gBAChC,sBAAsBsB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,GAAGJ;YACJ,MAAMK,cAAcC,qBAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaL,QAAQ,WAAW;QAChC,cAAc,CAACsC;YACbzC,aACE,iDACAyC,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/extraction.mjs","sources":["../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["debugInsight","getDebug","AiExtractElementInfo","options","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;;;;;AAuBA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,qBAAwBC,OAQ7C;IACC,MAAM,EAAEC,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEL;IACF,MAAMM,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,yBAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,gBAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,uBAC5Bb,QAAQ,eAAe,IAAI,IAC3BC,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,2BAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaL,QAAQ,WAAW;gBAChC,sBAAsBsB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,qBAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaL,QAAQ,WAAW;QAChC,cAAc,CAACsC;YACbzC,aACE,iDACAyC,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}

package/dist/es/ai-model/workflows/insight/order-sensitive.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/order-sensitive.mjs","sources":["../../../../../src/ai-model/workflows/insight/order-sensitive.ts"],"sourcesContent":["import type { AIUsageInfo } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { ModelRuntime } from '../../models';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from '../../prompt/order-sensitive-judge';\nimport { callAIWithObjectResponse } from '../../service-caller/index';\n\ntype InsightAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInsight('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInsight","getDebug","AiJudgeOrderSensitive","description","modelRuntime","systemPrompt","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt","msgs","result","callAIWithObjectResponse"],"mappings":";;;AAkBA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,sBACpBC,WAAmB,EACnBC,YAA0B;IAK1B,MAAMC,eAAeC;IACrB,MAAMC,aAAaC,0BAA0BL;IAE7C,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAASE;QACX;KACD;IAEDP,aAAa,yCAAyCG;IAEtD,MAAMO,SAAS,MAAMC,yBACnBF,MACAL,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkBM,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/order-sensitive.mjs","sources":["../../../../../src/ai-model/workflows/insight/order-sensitive.ts"],"sourcesContent":["import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { AIUsageInfo } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ModelRuntime } from '../../models';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from '../../prompt/order-sensitive-judge';\nimport { callAIWithObjectResponse } from '../../service-caller/index';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInsight('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInsight","getDebug","AiJudgeOrderSensitive","description","modelRuntime","systemPrompt","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt","msgs","result","callAIWithObjectResponse"],"mappings":";;;AAeA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,sBACpBC,WAAmB,EACnBC,YAA0B;IAK1B,MAAMC,eAAeC;IACrB,MAAMC,aAAaC,0BAA0BL;IAE7C,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAASE;QACX;KACD;IAEDP,aAAa,yCAAyCG;IAEtD,MAAMO,SAAS,MAAMC,yBACnBF,MACAL,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkBM,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}