@midscene/core 1.9.8 → 1.10.0

package/dist/es/ai-model/llm-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToString } from '@/common';\nimport type {\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans, findAllMidsceneLocatorField } from '../common';\nimport { planningModelFamilyRequiredForLocateMessage } from './errors';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport { AIResponseParseError, callAI } from './service-caller/index';\nimport type { JsonParser, JsonParserSource } from './service-caller/json';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport type { PlanOptions } from './workflows/planning/types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse.\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  jsonParser: JsonParser,\n): RawResponsePlanningAIResponse {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    // Strip any trailing XML tags that LLM might have leaked into the action type\n    // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\nexport async function plan(\n  userInstruction: TUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultAdapter =\n    modelRuntime.config.modelFamily && adapter.locate.kind === 'standard'\n      ? adapter.locate.resultAdapter\n      : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    locatePromptSpec: locateResultAdapter?.promptSpec,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const userInstructionText = userPromptToString(userInstruction);\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = opts.referenceImageMessages ?? [];\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstructionText}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  let {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n    rawChoiceMessage,\n  } = await callAI(msgs, modelRuntime, {\n    abortSignal: opts.abortSignal,\n    // Planning with locate results is localization-sensitive. Adapters decide\n    // whether this should request original image detail.\n    requiresOriginalImageDetail: opts.includeLocateInPlanning,\n  });\n\n  // Parse XML response to JSON object, retry once on parse failure\n  let planFromAI: RawResponsePlanningAIResponse;\n  try {\n    try {\n      planFromAI = parseXMLPlanningResponse(rawResponse, adapter.jsonParser);\n    } catch {\n      const retry = await callAI(msgs, modelRuntime, {\n        abortSignal: opts.abortSignal,\n        // Keep retry requests consistent with the initial planning call.\n        requiresOriginalImageDetail: opts.includeLocateInPlanning,\n      });\n      rawResponse = retry.content;\n      usage = retry.usage;\n      reasoning_content = retry.reasoning_content;\n      rawChoiceMessage = retry.rawChoiceMessage;\n      planFromAI = parseXMLPlanningResponse(rawResponse, adapter.jsonParser);\n    }\n\n    if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n      warnLog(\n        'Planning response included both an action and <complete>; ignoring <complete> output.',\n      );\n      planFromAI.finalizeMessage = undefined;\n      planFromAI.finalizeSuccess = undefined;\n    }\n\n    const actions = planFromAI.action ? [planFromAI.action] : [];\n    let shouldContinuePlanning = true;\n\n    // Check if task is completed via <complete> tag\n    if (planFromAI.finalizeSuccess !== undefined) {\n      debug('task completed via <complete> tag, stop planning');\n      shouldContinuePlanning = false;\n      // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n      if (includeSubGoals) {\n        conversationHistory.markAllSubGoalsFinished();\n      }\n    }\n\n    const returnValue: PlanningAIResponse = {\n      ...planFromAI,\n      actions,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content,\n      yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace),\n      shouldContinuePlanning,\n    };\n\n    assert(planFromAI, \"can't get plans from AI\");\n\n    actions.forEach((action) => {\n      const type = action.type;\n      const actionInActionSpace = opts.actionSpace.find(\n        (action) => action.name === type,\n      );\n\n      debug('actionInActionSpace matched', actionInActionSpace);\n      const locateFields = actionInActionSpace\n        ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n        : [];\n\n      debug('locateFields', locateFields);\n\n      locateFields.forEach((field) => {\n        const locateResult = action.param[field];\n        if (locateResult) {\n          if (!opts.includeLocateInPlanning) {\n            if (typeof locateResult === 'object') {\n              // In prompt-only planning mode, ignore any accidental coordinates from the model.\n              action.param[field] = { prompt: locateResult.prompt };\n            }\n            return;\n          }\n\n          assert(\n            locateResultAdapter,\n            'generic planning locate normalization requires a standard locate adapter',\n          );\n          action.param[field] = {\n            ...locateResult,\n            locatedPixelBbox: locateResultAdapter.adaptPlanningParamToPixelBbox(\n              locateResult,\n              {\n                preparedSize: preparedImage.preparedSize,\n                contentSize: preparedImage.contentSize,\n              },\n            ),\n          };\n        }\n      });\n    });\n\n    // Update sub-goals in conversation history only in planning deep-think mode.\n    if (includeSubGoals) {\n      if (planFromAI.updateSubGoals?.length) {\n        conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n      }\n      if (planFromAI.markFinishedIndexes?.length) {\n        for (const index of planFromAI.markFinishedIndexes) {\n          conversationHistory.markSubGoalFinished(index);\n        }\n      }\n      // Append the planning log to the currently running sub-goal\n      if (planFromAI.log) {\n        conversationHistory.appendSubGoalLog(planFromAI.log);\n      }\n    } else {\n      // Without planning deep-think mode, accumulate logs as historical execution steps.\n      if (planFromAI.log) {\n        conversationHistory.appendHistoricalLog(planFromAI.log);\n      }\n    }\n\n    // Append memory to conversation history if present\n    if (planFromAI.memory) {\n      conversationHistory.appendMemory(planFromAI.memory);\n    }\n\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n\n    return returnValue;\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n      rawChoiceMessage,\n    );\n  }\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","parseXMLPlanningResponse","xmlString","jsonParser","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","e","Error","plan","userInstruction","opts","context","conversationHistory","modelRuntime","adapter","shotSize","screenshotBase64","planningModelFamilyRequiredForLocateMessage","locateResultAdapter","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","preparedImage","prepareModelImage","imagePayload","userInstructionText","userPromptToString","actionContext","referenceImageMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","callAI","planFromAI","retry","actions","shouldContinuePlanning","returnValue","buildYamlFlowFromPlans","assert","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","index","parseError","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;;AAqBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAKK,SAASC,yBACdC,SAAiB,EACjBC,UAAsB;IAEtB,MAAMC,UAAUC,cAAcH,WAAW;IACzC,MAAMI,SAASD,cAAcH,WAAW;IACxC,MAAMK,MAAMF,cAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,cAAcH,WAAW;IACvC,MAAMO,aAAaJ,cAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,cAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,cAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,cAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,qBAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,yBAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQrB,WAAWO,gBAAgB;gBACjC,QAAQ;gBACR,yBACEa,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGR;YACjD;QACF,EAAE,OAAOU,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFH,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAEO,eAAeO,KACpBC,eAA4B,EAC5BC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAMG,eAAeH,KAAK,YAAY;IACtC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpB,MAAM,EAAEE,QAAQ,EAAE,GAAGJ;IACrB,MAAMK,mBAAmBL,QAAQ,UAAU,CAAC,MAAM;IAElD,IAAID,KAAK,uBAAuB,IAAI,CAACG,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIN,MACRU,4CAA4CJ,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMK,sBACJL,aAAa,MAAM,CAAC,WAAW,IAAIC,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,GAClDA,QAAQ,MAAM,CAAC,aAAa,GAC5BlB;IAGN,MAAMuB,kBAAkBT,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMU,eAAe,MAAMC,2BAA2B;QACpD,aAAaX,KAAK,WAAW;QAC7B,kBAAkBQ,qBAAqB;QACvC,yBAAyBR,KAAK,uBAAuB;QACrD,gBAAgB;QAChBS;IACF;IAEA,MAAMG,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,sBAAsBC,mBAAmBjB;IAC/C,MAAMkB,gBAAgBjB,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMkB,yBAAyBlB,KAAK,sBAAsB,IAAI,EAAE;IAChE,MAAMmB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGF,cAAc,kBAAkB,EAAEF,oBAAoB,mBAAmB,CAAC;gBACrF;aACD;QACH;WACGG;KACJ;IAED,IAAIE;IAKJ,MAAMC,wBAAwBZ,kBAC1BP,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMoB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BnB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAE/B,uBAAuB;IAGpC,MAAMoD,eAAerB,oBAAoB,cAAc;IACvD,MAAMsB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIrB,oBAAoB,sBAAsB,EAAE;QAC9CkB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGlB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEsB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAZ,oBAAoB,mCAAmC;IACzD,OACEkB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFZ,oBAAoB,MAAM,CAACkB;IAG3BlB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMuB,aAAavB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAM0B,OAAqC;QACzC;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,IAAI,EACF,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,GAAG,MAAMC,OAAOL,MAAMvB,cAAc;QACnC,aAAaH,KAAK,WAAW;QAG7B,6BAA6BA,KAAK,uBAAuB;IAC3D;IAGA,IAAIgC;IACJ,IAAI;QACF,IAAI;YACFA,aAAa5D,yBAAyBuD,aAAavB,QAAQ,UAAU;QACvE,EAAE,OAAM;YACN,MAAM6B,QAAQ,MAAMF,OAAOL,MAAMvB,cAAc;gBAC7C,aAAaH,KAAK,WAAW;gBAE7B,6BAA6BA,KAAK,uBAAuB;YAC3D;YACA2B,cAAcM,MAAM,OAAO;YAC3BL,QAAQK,MAAM,KAAK;YACnBJ,oBAAoBI,MAAM,iBAAiB;YAC3CH,mBAAmBG,MAAM,gBAAgB;YACzCD,aAAa5D,yBAAyBuD,aAAavB,QAAQ,UAAU;QACvE;QAEA,IAAI4B,WAAW,MAAM,IAAIA,AAA+B9C,WAA/B8C,WAAW,eAAe,EAAgB;YACjE9D,QACE;YAEF8D,WAAW,eAAe,GAAG9C;YAC7B8C,WAAW,eAAe,GAAG9C;QAC/B;QAEA,MAAMgD,UAAUF,WAAW,MAAM,GAAG;YAACA,WAAW,MAAM;SAAC,GAAG,EAAE;QAC5D,IAAIG,yBAAyB;QAG7B,IAAIH,AAA+B9C,WAA/B8C,WAAW,eAAe,EAAgB;YAC5ChE,MAAM;YACNmE,yBAAyB;YAEzB,IAAI1B,iBACFP,oBAAoB,uBAAuB;QAE/C;QAEA,MAAMkC,cAAkC;YACtC,GAAGJ,UAAU;YACbE;YACAP;YACAG;YACAF;YACAC;YACA,UAAUQ,uBAAuBH,SAASlC,KAAK,WAAW;YAC1DmC;QACF;QAEAG,OAAON,YAAY;QAEnBE,QAAQ,OAAO,CAAC,CAACzC;YACf,MAAMC,OAAOD,OAAO,IAAI;YACxB,MAAM8C,sBAAsBvC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACP,SAAWA,OAAO,IAAI,KAAKC;YAG9B1B,MAAM,+BAA+BuE;YACrC,MAAMC,eAAeD,sBACjBE,4BAA4BF,oBAAoB,WAAW,IAC3D,EAAE;YAENvE,MAAM,gBAAgBwE;YAEtBA,aAAa,OAAO,CAAC,CAACE;gBACpB,MAAMC,eAAelD,OAAO,KAAK,CAACiD,MAAM;gBACxC,IAAIC,cAAc;oBAChB,IAAI,CAAC3C,KAAK,uBAAuB,EAAE;wBACjC,IAAI,AAAwB,YAAxB,OAAO2C,cAETlD,OAAO,KAAK,CAACiD,MAAM,GAAG;4BAAE,QAAQC,aAAa,MAAM;wBAAC;wBAEtD;oBACF;oBAEAL,OACE9B,qBACA;oBAEFf,OAAO,KAAK,CAACiD,MAAM,GAAG;wBACpB,GAAGC,YAAY;wBACf,kBAAkBnC,oBAAoB,6BAA6B,CACjEmC,cACA;4BACE,cAAc/B,cAAc,YAAY;4BACxC,aAAaA,cAAc,WAAW;wBACxC;oBAEJ;gBACF;YACF;QACF;QAGA,IAAIH,iBAAiB;YACnB,IAAIuB,WAAW,cAAc,EAAE,QAC7B9B,oBAAoB,aAAa,CAAC8B,WAAW,cAAc;YAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAMY,SAASZ,WAAW,mBAAmB,CAChD9B,oBAAoB,mBAAmB,CAAC0C;YAI5C,IAAIZ,WAAW,GAAG,EAChB9B,oBAAoB,gBAAgB,CAAC8B,WAAW,GAAG;QAEvD,OAEE,IAAIA,WAAW,GAAG,EAChB9B,oBAAoB,mBAAmB,CAAC8B,WAAW,GAAG;QAK1D,IAAIA,WAAW,MAAM,EACnB9B,oBAAoB,YAAY,CAAC8B,WAAW,MAAM;QAGpD9B,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMyB;gBACR;aACD;QACH;QAEA,OAAOS;IACT,EAAE,OAAOS,YAAY;QAEnB,MAAMC,eACJD,sBAAsBhD,QAAQgD,WAAW,OAAO,GAAGE,OAAOF;QAC5D,MAAM,IAAIG,qBACR,CAAC,iBAAiB,EAAEF,cAAc,EAClCnB,aACAC,OACAE;IAEJ;AACF"}
+{"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToString } from '@/common';\nimport type {\n  PlanningAIResponse,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../common';\nimport { planningModelFamilyRequiredForLocateMessage } from './errors';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport { AIResponseParseError, callAI } from './service-caller/index';\nimport type { JsonParser, JsonParserSource } from './service-caller/json';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport { normalizePlanningActionLocateFields } from './workflows/planning/locate-normalization';\nimport type { PlanOptions } from './workflows/planning/types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse.\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  jsonParser: JsonParser,\n): RawResponsePlanningAIResponse {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    // Strip any trailing XML tags that LLM might have leaked into the action type\n    // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\nexport async function plan(\n  userInstruction: TUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultAdapter =\n    modelRuntime.config.modelFamily && adapter.locate.kind === 'standard'\n      ? adapter.locate.resultAdapter\n      : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    locatePromptSpec: locateResultAdapter?.promptSpec,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const userInstructionText = userPromptToString(userInstruction);\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = opts.referenceImageMessages ?? [];\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstructionText}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  let {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n    rawChoiceMessage,\n  } = await callAI(msgs, modelRuntime, {\n    abortSignal: opts.abortSignal,\n    // Planning with locate results is localization-sensitive. Adapters decide\n    // whether this should request original image detail.\n    requiresOriginalImageDetail: opts.includeLocateInPlanning,\n  });\n\n  // Parse XML response to JSON object, retry once on parse failure\n  let planFromAI: RawResponsePlanningAIResponse;\n  try {\n    try {\n      planFromAI = parseXMLPlanningResponse(rawResponse, adapter.jsonParser);\n    } catch {\n      const retry = await callAI(msgs, modelRuntime, {\n        abortSignal: opts.abortSignal,\n        // Keep retry requests consistent with the initial planning call.\n        requiresOriginalImageDetail: opts.includeLocateInPlanning,\n      });\n      rawResponse = retry.content;\n      usage = retry.usage;\n      reasoning_content = retry.reasoning_content;\n      rawChoiceMessage = retry.rawChoiceMessage;\n      planFromAI = parseXMLPlanningResponse(rawResponse, adapter.jsonParser);\n    }\n\n    if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n      warnLog(\n        'Planning response included both an action and <complete>; ignoring <complete> output.',\n      );\n      planFromAI.finalizeMessage = undefined;\n      planFromAI.finalizeSuccess = undefined;\n    }\n\n    const actions = planFromAI.action ? [planFromAI.action] : [];\n    let shouldContinuePlanning = true;\n\n    // Check if task is completed via <complete> tag\n    if (planFromAI.finalizeSuccess !== undefined) {\n      debug('task completed via <complete> tag, stop planning');\n      shouldContinuePlanning = false;\n      // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n      if (includeSubGoals) {\n        conversationHistory.markAllSubGoalsFinished();\n      }\n    }\n\n    const returnValue: PlanningAIResponse = {\n      ...planFromAI,\n      actions,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content,\n      yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace),\n      shouldContinuePlanning,\n    };\n\n    assert(planFromAI, \"can't get plans from AI\");\n\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: opts.actionSpace,\n      includeLocateInPlanning: opts.includeLocateInPlanning,\n      locateResultAdapter,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    });\n\n    // Update sub-goals in conversation history only in planning deep-think mode.\n    if (includeSubGoals) {\n      if (planFromAI.updateSubGoals?.length) {\n        conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n      }\n      if (planFromAI.markFinishedIndexes?.length) {\n        for (const index of planFromAI.markFinishedIndexes) {\n          conversationHistory.markSubGoalFinished(index);\n        }\n      }\n      // Append the planning log to the currently running sub-goal\n      if (planFromAI.log) {\n        conversationHistory.appendSubGoalLog(planFromAI.log);\n      }\n    } else {\n      // Without planning deep-think mode, accumulate logs as historical execution steps.\n      if (planFromAI.log) {\n        conversationHistory.appendHistoricalLog(planFromAI.log);\n      }\n    }\n\n    // Append memory to conversation history if present\n    if (planFromAI.memory) {\n      conversationHistory.appendMemory(planFromAI.memory);\n    }\n\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n\n    return returnValue;\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n      rawChoiceMessage,\n    );\n  }\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","parseXMLPlanningResponse","xmlString","jsonParser","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","e","Error","plan","userInstruction","opts","context","conversationHistory","modelRuntime","adapter","shotSize","screenshotBase64","planningModelFamilyRequiredForLocateMessage","locateResultAdapter","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","preparedImage","prepareModelImage","imagePayload","userInstructionText","userPromptToString","actionContext","referenceImageMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","callAI","planFromAI","retry","actions","shouldContinuePlanning","returnValue","buildYamlFlowFromPlans","assert","normalizePlanningActionLocateFields","index","parseError","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;;;AAsBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAKK,SAASC,yBACdC,SAAiB,EACjBC,UAAsB;IAEtB,MAAMC,UAAUC,cAAcH,WAAW;IACzC,MAAMI,SAASD,cAAcH,WAAW;IACxC,MAAMK,MAAMF,cAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,cAAcH,WAAW;IACvC,MAAMO,aAAaJ,cAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,cAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,cAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,cAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,qBAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,yBAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQrB,WAAWO,gBAAgB;gBACjC,QAAQ;gBACR,yBACEa,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGR;YACjD;QACF,EAAE,OAAOU,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFH,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAEO,eAAeO,KACpBC,eAA4B,EAC5BC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAMG,eAAeH,KAAK,YAAY;IACtC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpB,MAAM,EAAEE,QAAQ,EAAE,GAAGJ;IACrB,MAAMK,mBAAmBL,QAAQ,UAAU,CAAC,MAAM;IAElD,IAAID,KAAK,uBAAuB,IAAI,CAACG,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIN,MACRU,4CAA4CJ,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMK,sBACJL,aAAa,MAAM,CAAC,WAAW,IAAIC,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,GAClDA,QAAQ,MAAM,CAAC,aAAa,GAC5BlB;IAGN,MAAMuB,kBAAkBT,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMU,eAAe,MAAMC,2BAA2B;QACpD,aAAaX,KAAK,WAAW;QAC7B,kBAAkBQ,qBAAqB;QACvC,yBAAyBR,KAAK,uBAAuB;QACrD,gBAAgB;QAChBS;IACF;IAEA,MAAMG,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,sBAAsBC,mBAAmBjB;IAC/C,MAAMkB,gBAAgBjB,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMkB,yBAAyBlB,KAAK,sBAAsB,IAAI,EAAE;IAChE,MAAMmB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGF,cAAc,kBAAkB,EAAEF,oBAAoB,mBAAmB,CAAC;gBACrF;aACD;QACH;WACGG;KACJ;IAED,IAAIE;IAKJ,MAAMC,wBAAwBZ,kBAC1BP,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMoB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BnB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAE/B,uBAAuB;IAGpC,MAAMoD,eAAerB,oBAAoB,cAAc;IACvD,MAAMsB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIrB,oBAAoB,sBAAsB,EAAE;QAC9CkB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGlB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEsB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAZ,oBAAoB,mCAAmC;IACzD,OACEkB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFZ,oBAAoB,MAAM,CAACkB;IAG3BlB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMuB,aAAavB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAM0B,OAAqC;QACzC;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,IAAI,EACF,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,GAAG,MAAMC,OAAOL,MAAMvB,cAAc;QACnC,aAAaH,KAAK,WAAW;QAG7B,6BAA6BA,KAAK,uBAAuB;IAC3D;IAGA,IAAIgC;IACJ,IAAI;QACF,IAAI;YACFA,aAAa5D,yBAAyBuD,aAAavB,QAAQ,UAAU;QACvE,EAAE,OAAM;YACN,MAAM6B,QAAQ,MAAMF,OAAOL,MAAMvB,cAAc;gBAC7C,aAAaH,KAAK,WAAW;gBAE7B,6BAA6BA,KAAK,uBAAuB;YAC3D;YACA2B,cAAcM,MAAM,OAAO;YAC3BL,QAAQK,MAAM,KAAK;YACnBJ,oBAAoBI,MAAM,iBAAiB;YAC3CH,mBAAmBG,MAAM,gBAAgB;YACzCD,aAAa5D,yBAAyBuD,aAAavB,QAAQ,UAAU;QACvE;QAEA,IAAI4B,WAAW,MAAM,IAAIA,AAA+B9C,WAA/B8C,WAAW,eAAe,EAAgB;YACjE9D,QACE;YAEF8D,WAAW,eAAe,GAAG9C;YAC7B8C,WAAW,eAAe,GAAG9C;QAC/B;QAEA,MAAMgD,UAAUF,WAAW,MAAM,GAAG;YAACA,WAAW,MAAM;SAAC,GAAG,EAAE;QAC5D,IAAIG,yBAAyB;QAG7B,IAAIH,AAA+B9C,WAA/B8C,WAAW,eAAe,EAAgB;YAC5ChE,MAAM;YACNmE,yBAAyB;YAEzB,IAAI1B,iBACFP,oBAAoB,uBAAuB;QAE/C;QAEA,MAAMkC,cAAkC;YACtC,GAAGJ,UAAU;YACbE;YACAP;YACAG;YACAF;YACAC;YACA,UAAUQ,uBAAuBH,SAASlC,KAAK,WAAW;YAC1DmC;QACF;QAEAG,OAAON,YAAY;QAEnBO,oCAAoCL,SAAS;YAC3C,aAAalC,KAAK,WAAW;YAC7B,yBAAyBA,KAAK,uBAAuB;YACrDQ;YACA,qBAAqB;gBACnB,cAAcI,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;QACF;QAGA,IAAIH,iBAAiB;YACnB,IAAIuB,WAAW,cAAc,EAAE,QAC7B9B,oBAAoB,aAAa,CAAC8B,WAAW,cAAc;YAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAMQ,SAASR,WAAW,mBAAmB,CAChD9B,oBAAoB,mBAAmB,CAACsC;YAI5C,IAAIR,WAAW,GAAG,EAChB9B,oBAAoB,gBAAgB,CAAC8B,WAAW,GAAG;QAEvD,OAEE,IAAIA,WAAW,GAAG,EAChB9B,oBAAoB,mBAAmB,CAAC8B,WAAW,GAAG;QAK1D,IAAIA,WAAW,MAAM,EACnB9B,oBAAoB,YAAY,CAAC8B,WAAW,MAAM;QAGpD9B,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMyB;gBACR;aACD;QACH;QAEA,OAAOS;IACT,EAAE,OAAOK,YAAY;QAEnB,MAAMC,eACJD,sBAAsB5C,QAAQ4C,WAAW,OAAO,GAAGE,OAAOF;QAC5D,MAAM,IAAIG,qBACR,CAAC,iBAAiB,EAAEF,cAAc,EAClCf,aACAC,OACAE;IAEJ;AACF"}

package/dist/es/ai-model/model-adapter/chat-completion.mjs

@@ -0,0 +1,46 @@
+const defaultImageDetail = (_input)=>void 0;
+const defaultChatCompletionParams = ({ midsceneDefaults, userConfig })=>({
+        config: {
+            temperature: userConfig.temperature ?? midsceneDefaults.temperature
+        }
+    });
+const midsceneChatCompletionDefaults = {
+    temperature: 0
+};
+function defaultExtractContentAndReasoning(message) {
+    if (!message) return {
+        content: '',
+        reasoning_content: ''
+    };
+    const messageReasoning = 'string' == typeof message.reasoning_content ? message.reasoning_content : '';
+    return {
+        content: 'string' == typeof message.content ? message.content : '',
+        reasoning_content: messageReasoning
+    };
+}
+function resolveChatCompletion(chatCompletion) {
+    const buildChatCompletionParams = chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;
+    const resolveImageDetail = chatCompletion?.resolveImageDetail ?? defaultImageDetail;
+    const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];
+    const extractContentAndReasoning = chatCompletion?.extractContentAndReasoning ?? defaultExtractContentAndReasoning;
+    return {
+        unsupportedUserConfig,
+        buildChatCompletionParams: (input)=>{
+            const context = {
+                ...input,
+                userConfig: input.userConfig ?? {},
+                midsceneDefaults: midsceneChatCompletionDefaults
+            };
+            return buildChatCompletionParams(context);
+        },
+        resolveImageDetail: (input)=>resolveImageDetail({
+                ...input,
+                userConfig: input.userConfig ?? {},
+                midsceneDefaults: midsceneChatCompletionDefaults
+            }),
+        extractContentAndReasoning
+    };
+}
+export { defaultExtractContentAndReasoning, resolveChatCompletion };
+
+//# sourceMappingURL=chat-completion.mjs.map

package/dist/es/ai-model/model-adapter/chat-completion.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/chat-completion.mjs","sources":["../../../../src/ai-model/model-adapter/chat-completion.ts"],"sourcesContent":["import type {\n  ChatCompletionAdapter,\n  ChatCompletionCallContext,\n  ChatCompletionContentSource,\n  ContentAndReasoning,\n  MidsceneChatCompletionDefaults,\n  ModelAdapterDefinition,\n} from './types';\n\nconst defaultImageDetail = (_input: unknown) => undefined;\n\nconst defaultChatCompletionParams = ({\n  midsceneDefaults,\n  userConfig,\n}: ChatCompletionCallContext) => ({\n  config: {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n  },\n});\n\nconst midsceneChatCompletionDefaults: MidsceneChatCompletionDefaults = {\n  temperature: 0,\n};\n\nexport function defaultExtractContentAndReasoning(\n  message: ChatCompletionContentSource | undefined,\n): ContentAndReasoning {\n  if (!message) {\n    return {\n      content: '',\n      reasoning_content: '',\n    };\n  }\n\n  const messageReasoning =\n    typeof message.reasoning_content === 'string'\n      ? message.reasoning_content\n      : '';\n\n  return {\n    content: typeof message.content === 'string' ? message.content : '',\n    reasoning_content: messageReasoning,\n  };\n}\n\nexport function resolveChatCompletion(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ChatCompletionAdapter {\n  const buildChatCompletionParams =\n    chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;\n  const resolveImageDetail =\n    chatCompletion?.resolveImageDetail ?? defaultImageDetail;\n  const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];\n  const extractContentAndReasoning =\n    chatCompletion?.extractContentAndReasoning ??\n    defaultExtractContentAndReasoning;\n\n  return {\n    unsupportedUserConfig,\n    buildChatCompletionParams: (input) => {\n      const context = {\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      };\n      return buildChatCompletionParams(context);\n    },\n    resolveImageDetail: (input) =>\n      resolveImageDetail({\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      }),\n    extractContentAndReasoning,\n  };\n}\n"],"names":["defaultImageDetail","_input","undefined","defaultChatCompletionParams","midsceneDefaults","userConfig","midsceneChatCompletionDefaults","defaultExtractContentAndReasoning","message","messageReasoning","resolveChatCompletion","chatCompletion","buildChatCompletionParams","resolveImageDetail","unsupportedUserConfig","extractContentAndReasoning","input","context"],"mappings":"AASA,MAAMA,qBAAqB,CAACC,SAAoBC;AAEhD,MAAMC,8BAA8B,CAAC,EACnCC,gBAAgB,EAChBC,UAAU,EACgB,GAAM;QAChC,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEA,MAAME,iCAAiE;IACrE,aAAa;AACf;AAEO,SAASC,kCACdC,OAAgD;IAEhD,IAAI,CAACA,SACH,OAAO;QACL,SAAS;QACT,mBAAmB;IACrB;IAGF,MAAMC,mBACJ,AAAqC,YAArC,OAAOD,QAAQ,iBAAiB,GAC5BA,QAAQ,iBAAiB,GACzB;IAEN,OAAO;QACL,SAAS,AAA2B,YAA3B,OAAOA,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;QACjE,mBAAmBC;IACrB;AACF;AAEO,SAASC,sBACdC,cAAwD;IAExD,MAAMC,4BACJD,gBAAgB,6BAA6BR;IAC/C,MAAMU,qBACJF,gBAAgB,sBAAsBX;IACxC,MAAMc,wBAAwBH,gBAAgB,yBAAyB,EAAE;IACzE,MAAMI,6BACJJ,gBAAgB,8BAChBJ;IAEF,OAAO;QACLO;QACA,2BAA2B,CAACE;YAC1B,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBV;YACpB;YACA,OAAOM,0BAA0BK;QACnC;QACA,oBAAoB,CAACD,QACnBH,mBAAmB;gBACjB,GAAGG,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBV;YACpB;QACFS;IACF;AACF"}

package/dist/es/ai-model/model-adapter/index.mjs

@@ -0,0 +1,2 @@
+import { ResolvedModelAdapter } from "./resolve.mjs";
+export { ResolvedModelAdapter };

package/dist/es/ai-model/model-adapter/locate.mjs

@@ -0,0 +1,33 @@
+import { createLocateResultAdapter } from "../shared/model-locate-result/factory.mjs";
+import { resolvePlanningTapLocator } from "../workflows/inspect/planning-action-locate.mjs";
+const defaultLocateResultAdapterDefinition = {
+    coordinates: {
+        shape: 'bbox',
+        order: 'xy',
+        normalizedBy: 1000
+    }
+};
+function resolveLocate(locate, resolvedCustomPlanner) {
+    if (locate?.kind === 'custom') {
+        let locateFn = locate.locateFn;
+        if (!locateFn) {
+            const planningTapLocator = locate.planningTapLocator;
+            if (!planningTapLocator) throw new Error('Custom locate definition requires either locateFn or planningTapLocator');
+            if (!resolvedCustomPlanner) throw new Error('Custom planning tap locator requires a custom planning planner definition');
+            locateFn = resolvePlanningTapLocator(planningTapLocator, resolvedCustomPlanner);
+        }
+        return {
+            kind: 'custom',
+            supportsSearchArea: locate.supportsSearchArea ?? false,
+            locateFn
+        };
+    }
+    return {
+        kind: 'standard',
+        supportsSearchArea: locate?.supportsSearchArea ?? true,
+        resultAdapter: createLocateResultAdapter(locate?.resultAdapter ?? defaultLocateResultAdapterDefinition)
+    };
+}
+export { resolveLocate };
+
+//# sourceMappingURL=locate.mjs.map

package/dist/es/ai-model/model-adapter/locate.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/locate.mjs","sources":["../../../../src/ai-model/model-adapter/locate.ts"],"sourcesContent":["import { createLocateResultAdapter } from '../shared/model-locate-result/factory';\nimport type { LocateResultAdapterDefinition } from '../shared/model-locate-result/types';\nimport { resolvePlanningTapLocator } from '../workflows/inspect/planning-action-locate';\nimport type { ResolvedCustomPlanningDefinition } from './custom-planning-types';\nimport type { LocateAdapter, ModelAdapterDefinition } from './types';\n\nconst defaultLocateResultAdapterDefinition: LocateResultAdapterDefinition = {\n  coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n};\n\nexport function resolveLocate(\n  locate: ModelAdapterDefinition['locate'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n): LocateAdapter {\n  if (locate?.kind === 'custom') {\n    let locateFn = locate.locateFn;\n\n    if (!locateFn) {\n      const planningTapLocator = locate.planningTapLocator;\n\n      if (!planningTapLocator) {\n        throw new Error(\n          'Custom locate definition requires either locateFn or planningTapLocator',\n        );\n      }\n\n      if (!resolvedCustomPlanner) {\n        throw new Error(\n          'Custom planning tap locator requires a custom planning planner definition',\n        );\n      }\n      locateFn = resolvePlanningTapLocator(\n        planningTapLocator,\n        resolvedCustomPlanner,\n      );\n    }\n\n    return {\n      kind: 'custom',\n      supportsSearchArea: locate.supportsSearchArea ?? false,\n      locateFn,\n    };\n  }\n\n  return {\n    kind: 'standard',\n    supportsSearchArea: locate?.supportsSearchArea ?? true,\n    resultAdapter: createLocateResultAdapter(\n      locate?.resultAdapter ?? defaultLocateResultAdapterDefinition,\n    ),\n  };\n}\n"],"names":["defaultLocateResultAdapterDefinition","resolveLocate","locate","resolvedCustomPlanner","locateFn","planningTapLocator","Error","resolvePlanningTapLocator","createLocateResultAdapter"],"mappings":";;AAMA,MAAMA,uCAAsE;IAC1E,aAAa;QAAE,OAAO;QAAQ,OAAO;QAAM,cAAc;IAAK;AAChE;AAEO,SAASC,cACdC,MAAwC,EACxCC,qBAAmE;IAEnE,IAAID,QAAQ,SAAS,UAAU;QAC7B,IAAIE,WAAWF,OAAO,QAAQ;QAE9B,IAAI,CAACE,UAAU;YACb,MAAMC,qBAAqBH,OAAO,kBAAkB;YAEpD,IAAI,CAACG,oBACH,MAAM,IAAIC,MACR;YAIJ,IAAI,CAACH,uBACH,MAAM,IAAIG,MACR;YAGJF,WAAWG,0BACTF,oBACAF;QAEJ;QAEA,OAAO;YACL,MAAM;YACN,oBAAoBD,OAAO,kBAAkB,IAAI;YACjDE;QACF;IACF;IAEA,OAAO;QACL,MAAM;QACN,oBAAoBF,QAAQ,sBAAsB;QAClD,eAAeM,0BACbN,QAAQ,iBAAiBF;IAE7B;AACF"}

package/dist/es/ai-model/model-adapter/planning.mjs

@@ -0,0 +1,44 @@
+import { createLocateResultAdapter, resolveLocateResultCoordinates } from "../shared/model-locate-result/index.mjs";
+import { runCustomPlanning } from "../workflows/planning/custom-planning.mjs";
+const defaultReplanningCycleLimit = 20;
+function resolveCustomPlanningDefinition(config) {
+    const { coordinates, ...rest } = config;
+    const coordinateSystem = resolveLocateResultCoordinates(coordinates);
+    const coordinateNormalizer = createLocateResultAdapter({
+        coordinates
+    });
+    return {
+        ...rest,
+        coordinateSystem,
+        coordinateNormalizer
+    };
+}
+function resolvePlanning(planning, resolvedCustomPlanner) {
+    if (planning?.kind === 'custom') {
+        if ('function' == typeof planning.planFn) return {
+            kind: 'custom',
+            cacheEnabled: planning.cacheEnabled ?? false,
+            defaultReplanningCycleLimit: planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
+            supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,
+            planFn: planning.planFn
+        };
+        if (!resolvedCustomPlanner) throw new Error('Custom planning planner definition is not resolved');
+        return {
+            kind: 'custom',
+            cacheEnabled: planning.cacheEnabled ?? false,
+            defaultReplanningCycleLimit: planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
+            supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,
+            coordinateSystem: resolvedCustomPlanner.coordinateSystem,
+            planFn: (userInstruction, options)=>runCustomPlanning(userInstruction, options, resolvedCustomPlanner)
+        };
+    }
+    return {
+        kind: 'standard',
+        cacheEnabled: planning?.cacheEnabled ?? true,
+        defaultReplanningCycleLimit: planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
+        supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true
+    };
+}
+export { resolveCustomPlanningDefinition, resolvePlanning };
+
+//# sourceMappingURL=planning.mjs.map

package/dist/es/ai-model/model-adapter/planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/planning.mjs","sources":["../../../../src/ai-model/model-adapter/planning.ts"],"sourcesContent":["import {\n  createLocateResultAdapter,\n  resolveLocateResultCoordinates,\n} from '../shared/model-locate-result';\nimport { runCustomPlanning } from '../workflows/planning/custom-planning';\nimport type {\n  CustomPlanningDefinition,\n  ResolvedCustomPlanningDefinition,\n} from './custom-planning-types';\nimport type { ModelAdapterDefinition, PlanningAdapter } from './types';\n\nconst defaultReplanningCycleLimit = 20;\n\nexport function resolveCustomPlanningDefinition<TParsed>(\n  config: CustomPlanningDefinition<TParsed>,\n): ResolvedCustomPlanningDefinition<TParsed> {\n  const { coordinates, ...rest } = config;\n  const coordinateSystem = resolveLocateResultCoordinates(coordinates);\n  const coordinateNormalizer = createLocateResultAdapter({ coordinates });\n  return {\n    ...rest,\n    coordinateSystem,\n    coordinateNormalizer,\n  };\n}\n\nexport function resolvePlanning(\n  planning: ModelAdapterDefinition['planning'],\n  resolvedCustomPlanner?: ResolvedCustomPlanningDefinition,\n): PlanningAdapter {\n  if (planning?.kind === 'custom') {\n    if (typeof planning.planFn === 'function') {\n      return {\n        kind: 'custom',\n        cacheEnabled: planning.cacheEnabled ?? false,\n        defaultReplanningCycleLimit:\n          planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n        supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,\n        planFn: planning.planFn,\n      };\n    }\n\n    if (!resolvedCustomPlanner) {\n      throw new Error('Custom planning planner definition is not resolved');\n    }\n\n    return {\n      kind: 'custom',\n      cacheEnabled: planning.cacheEnabled ?? false,\n      defaultReplanningCycleLimit:\n        planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n      supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,\n      coordinateSystem: resolvedCustomPlanner.coordinateSystem,\n      planFn: (userInstruction, options) =>\n        runCustomPlanning(userInstruction, options, resolvedCustomPlanner),\n    };\n  }\n\n  return {\n    kind: 'standard',\n    cacheEnabled: planning?.cacheEnabled ?? true,\n    defaultReplanningCycleLimit:\n      planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n    supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true,\n  };\n}\n"],"names":["defaultReplanningCycleLimit","resolveCustomPlanningDefinition","config","coordinates","rest","coordinateSystem","resolveLocateResultCoordinates","coordinateNormalizer","createLocateResultAdapter","resolvePlanning","planning","resolvedCustomPlanner","Error","userInstruction","options","runCustomPlanning"],"mappings":";;AAWA,MAAMA,8BAA8B;AAE7B,SAASC,gCACdC,MAAyC;IAEzC,MAAM,EAAEC,WAAW,EAAE,GAAGC,MAAM,GAAGF;IACjC,MAAMG,mBAAmBC,+BAA+BH;IACxD,MAAMI,uBAAuBC,0BAA0B;QAAEL;IAAY;IACrE,OAAO;QACL,GAAGC,IAAI;QACPC;QACAE;IACF;AACF;AAEO,SAASE,gBACdC,QAA4C,EAC5CC,qBAAwD;IAExD,IAAID,UAAU,SAAS,UAAU;QAC/B,IAAI,AAA2B,cAA3B,OAAOA,SAAS,MAAM,EACxB,OAAO;YACL,MAAM;YACN,cAAcA,SAAS,YAAY,IAAI;YACvC,6BACEA,SAAS,2BAA2B,IAAIV;YAC1C,0BAA0BU,SAAS,wBAAwB,IAAI;YAC/D,QAAQA,SAAS,MAAM;QACzB;QAGF,IAAI,CAACC,uBACH,MAAM,IAAIC,MAAM;QAGlB,OAAO;YACL,MAAM;YACN,cAAcF,SAAS,YAAY,IAAI;YACvC,6BACEA,SAAS,2BAA2B,IAAIV;YAC1C,0BAA0BU,SAAS,wBAAwB,IAAI;YAC/D,kBAAkBC,sBAAsB,gBAAgB;YACxD,QAAQ,CAACE,iBAAiBC,UACxBC,kBAAkBF,iBAAiBC,SAASH;QAChD;IACF;IAEA,OAAO;QACL,MAAM;QACN,cAAcD,UAAU,gBAAgB;QACxC,6BACEA,UAAU,+BAA+BV;QAC3C,0BAA0BU,UAAU,4BAA4B;IAClE;AACF"}

package/dist/es/ai-model/model-adapter/resolve.mjs

@@ -0,0 +1,43 @@
+import { normalJsonParser } from "../service-caller/json.mjs";
+import { resolveChatCompletion } from "./chat-completion.mjs";
+import { resolveLocate } from "./locate.mjs";
+import { resolveCustomPlanningDefinition, resolvePlanning } from "./planning.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function resolveJsonParser(jsonParser) {
+    if (!jsonParser || 'lenient-json' === jsonParser) return normalJsonParser;
+    if ('function' == typeof jsonParser) return jsonParser;
+    throw new Error(`Unknown json parser preset: ${jsonParser}`);
+}
+function resolveImagePreprocess(imagePreprocess) {
+    return {
+        padBlockSize: imagePreprocess?.padBlockSize
+    };
+}
+class ResolvedModelAdapter {
+    constructor(config, modelFamily){
+        _define_property(this, "jsonParser", void 0);
+        _define_property(this, "chatCompletion", void 0);
+        _define_property(this, "imagePreprocess", void 0);
+        _define_property(this, "planning", void 0);
+        _define_property(this, "locate", void 0);
+        this.jsonParser = resolveJsonParser(config.jsonParser);
+        this.chatCompletion = resolveChatCompletion(config.chatCompletion);
+        this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);
+        const customPlanner = config.planning?.kind === 'custom' ? config.planning.planner : void 0;
+        const resolvedCustomPlanner = customPlanner ? resolveCustomPlanningDefinition(customPlanner) : void 0;
+        this.planning = resolvePlanning(config.planning, resolvedCustomPlanner);
+        this.locate = resolveLocate(config.locate, resolvedCustomPlanner);
+    }
+}
+export { ResolvedModelAdapter };
+
+//# sourceMappingURL=resolve.mjs.map

package/dist/es/ai-model/model-adapter/resolve.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { normalJsonParser } from '../service-caller/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport type {\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n} from './types';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return normalJsonParser;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.planning = resolvePlanning(config.planning, resolvedCustomPlanner);\n    this.locate = resolveLocate(config.locate, resolvedCustomPlanner);\n  }\n}\n"],"names":["resolveJsonParser","jsonParser","normalJsonParser","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolvePlanning","resolveLocate"],"mappings":";;;;;;;;;;;;;;AAcA,SAASA,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAOX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QANjE,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,UAAU,GAAGR,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,eAAe,GAAGH,uBAAuBG,OAAO,eAAe;QACpE,MAAMG,gBACJH,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGI;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,QAAQ,GAAGG,gBAAgBP,OAAO,QAAQ,EAAEK;QACjD,IAAI,CAAC,MAAM,GAAGG,cAAcR,OAAO,MAAM,EAAEK;IAC7C;AACF"}

package/dist/es/ai-model/models/auto-glm/actions.mjs

@@ -1,25 +1,5 @@
 import { getDebug } from "@midscene/shared/logger";
-import { finalizePixelBbox } from "../../shared/model-locate-result/bbox.mjs";
-import { mapLocateResultToPixelBboxByCoordinates } from "../../shared/model-locate-result/pixel-bbox-mapper.mjs";
 const debug = getDebug('auto-glm-actions');
-const AUTO_GLM_COORDINATE_MAX = 1000;
-function autoGLMCoordinateToLocateParam(coordinate, size) {
-    const ctx = {
-        preparedSize: size
-    };
-    const pixelBbox = mapLocateResultToPixelBboxByCoordinates({
-        type: 'point',
-        coordinates: coordinate
-    }, ctx, {
-        shape: 'point',
-        order: 'xy',
-        normalizedBy: AUTO_GLM_COORDINATE_MAX
-    });
-    return {
-        prompt: '',
-        locatedPixelBbox: finalizePixelBbox(pixelBbox, coordinate, ctx)
-    };
-}
 const BACK_BUTTON_NAMES = [
     'AndroidBackButton',
     'HarmonyBackButton'
@@ -33,7 +13,7 @@ function findActionName(actionSpace, knownNames, defaultName) {
     const match = actionSpace.find((a)=>knownNames.includes(a.name));
     return match ? match.name : defaultName;
 }
-function transformAutoGLMAction(action, size, actionSpace) {
+function transformAutoGLMAction(action, { actionSpace, coordinateDistanceToPixels }) {
     try {
         switch(action._metadata){
             case 'finish':
@@ -56,12 +36,14 @@ function transformAutoGLMAction(action, size, actionSpace) {
                             {
                                 const tapAction = doAction;
                                 debug('Transform Tap action:', tapAction);
-                                const locate = autoGLMCoordinateToLocateParam(tapAction.element, size);
                                 return [
                                     {
                                         type: 'Tap',
                                         param: {
-                                            locate
+                                            locate: {
+                                                point: tapAction.element,
+                                                prompt: ''
+                                            }
                                         }
                                     }
                                 ];
@@ -70,12 +52,14 @@ function transformAutoGLMAction(action, size, actionSpace) {
                             {
                                 const doubleTapAction = doAction;
                                 debug('Transform Double Tap action:', doubleTapAction);
-                                const locate = autoGLMCoordinateToLocateParam(doubleTapAction.element, size);
                                 return [
                                     {
                                         type: 'DoubleClick',
                                         param: {
-                                            locate
+                                            locate: {
+                                                point: doubleTapAction.element,
+                                                prompt: ''
+                                            }
                                         }
                                     }
                                 ];
@@ -97,7 +81,6 @@ function transformAutoGLMAction(action, size, actionSpace) {
                             {
                                 const swipeAction = doAction;
                                 debug('Transform Swipe action:', swipeAction);
-                                const locate = autoGLMCoordinateToLocateParam(swipeAction.start, size);
                                 const deltaX = swipeAction.end[0] - swipeAction.start[0];
                                 const deltaY = swipeAction.end[1] - swipeAction.start[1];
                                 let direction;
@@ -105,10 +88,10 @@ function transformAutoGLMAction(action, size, actionSpace) {
                                 const absDeltaX = Math.abs(deltaX);
                                 const absDeltaY = Math.abs(deltaY);
                                 if (absDeltaY > absDeltaX) {
-                                    distance = Math.round(absDeltaY * size.height / AUTO_GLM_COORDINATE_MAX);
+                                    distance = coordinateDistanceToPixels(deltaY, 'y');
                                     direction = deltaY > 0 ? 'up' : 'down';
                                 } else {
-                                    distance = Math.round(absDeltaX * size.width / AUTO_GLM_COORDINATE_MAX);
+                                    distance = coordinateDistanceToPixels(deltaX, 'x');
                                     direction = deltaX > 0 ? 'left' : 'right';
                                 }
                                 debug(`Calculate swipe direction: ${direction}, distance: ${distance}`);
@@ -116,7 +99,10 @@ function transformAutoGLMAction(action, size, actionSpace) {
                                     {
                                         type: 'Scroll',
                                         param: {
-                                            locate,
+                                            locate: {
+                                                point: swipeAction.start,
+                                                prompt: ''
+                                            },
                                             distance,
                                             direction
                                         },
@@ -128,12 +114,14 @@ function transformAutoGLMAction(action, size, actionSpace) {
                             {
                                 const longPressAction = doAction;
                                 debug('Transform Long Press action:', longPressAction);
-                                const locate = autoGLMCoordinateToLocateParam(longPressAction.element, size);
                                 return [
                                     {
                                         type: 'LongPress',
                                         param: {
-                                            locate
+                                            locate: {
+                                                point: longPressAction.element,
+                                                prompt: ''
+                                            }
                                         },
                                         thought: longPressAction.think || ''
                                     }

package/dist/es/ai-model/models/auto-glm/actions.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/auto-glm/actions.mjs","sources":["../../../../../src/ai-model/models/auto-glm/actions.ts"],"sourcesContent":["import type { DeviceAction } from '@/device';\nimport type {\n  PlanningAction,\n  PlanningLocateParamWithLocatedPixelBbox,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { finalizePixelBbox } from '../../shared/model-locate-result/bbox';\nimport { mapLocateResultToPixelBboxByCoordinates } from '../../shared/model-locate-result/pixel-bbox-mapper';\n\nconst debug = getDebug('auto-glm-actions');\n\n/**\n * Auto-GLM coordinate system range: [0, AUTO_GLM_COORDINATE_MAX]\n */\nconst AUTO_GLM_COORDINATE_MAX = 1000;\n\nfunction autoGLMCoordinateToLocateParam(\n  coordinate: [number, number],\n  size: { width: number; height: number },\n): PlanningLocateParamWithLocatedPixelBbox {\n  const ctx = { preparedSize: size };\n  const pixelBbox = mapLocateResultToPixelBboxByCoordinates(\n    { type: 'point', coordinates: coordinate },\n    ctx,\n    { shape: 'point', order: 'xy', normalizedBy: AUTO_GLM_COORDINATE_MAX },\n  );\n\n  return {\n    prompt: '',\n    locatedPixelBbox: finalizePixelBbox(pixelBbox, coordinate, ctx),\n  };\n}\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type ParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nconst BACK_BUTTON_NAMES = ['AndroidBackButton', 'HarmonyBackButton'];\nconst HOME_BUTTON_NAMES = ['AndroidHomeButton', 'HarmonyHomeButton'];\n\n/**\n * Find the action name in actionSpace that matches one of the known names.\n * Falls back to defaultName if no match found or actionSpace is not provided.\n */\nfunction findActionName(\n  actionSpace: DeviceAction[] | undefined,\n  knownNames: string[],\n  defaultName: string,\n): string {\n  if (!actionSpace) return defaultName;\n  const match = actionSpace.find((a) => knownNames.includes(a.name));\n  return match ? match.name : defaultName;\n}\n\nexport function transformAutoGLMAction(\n  action: ParsedAction,\n  size: { width: number; height: number },\n  actionSpace?: DeviceAction[],\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n            const locate = autoGLMCoordinateToLocateParam(\n              tapAction.element,\n              size,\n            );\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n            const locate = autoGLMCoordinateToLocateParam(\n              doubleTapAction.element,\n              size,\n            );\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            const locate = autoGLMCoordinateToLocateParam(\n              swipeAction.start,\n              size,\n            );\n\n            // Calculate horizontal and vertical delta in [0,AUTO_GLM_COORDINATE_MAX] coordinate system\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = Math.round(\n                (absDeltaY * size.height) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = Math.round(\n                (absDeltaX * size.width) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate,\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              },\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n            const locate = autoGLMCoordinateToLocateParam(\n              longPressAction.element,\n              size,\n            );\n\n            return [\n              {\n                type: 'LongPress',\n                param: {\n                  locate,\n                },\n                thought: longPressAction.think || '',\n              },\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  BACK_BUTTON_NAMES,\n                  'AndroidBackButton',\n                ),\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  HOME_BUTTON_NAMES,\n                  'AndroidHomeButton',\n                ),\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: { uri: launchAction.app },\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["debug","getDebug","AUTO_GLM_COORDINATE_MAX","autoGLMCoordinateToLocateParam","coordinate","size","ctx","pixelBbox","mapLocateResultToPixelBboxByCoordinates","finalizePixelBbox","BACK_BUTTON_NAMES","HOME_BUTTON_NAMES","findActionName","actionSpace","knownNames","defaultName","match","a","transformAutoGLMAction","action","finishAction","doAction","tapAction","locate","doubleTapAction","typeAction","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";;;AASA,MAAMA,QAAQC,SAAS;AAKvB,MAAMC,0BAA0B;AAEhC,SAASC,+BACPC,UAA4B,EAC5BC,IAAuC;IAEvC,MAAMC,MAAM;QAAE,cAAcD;IAAK;IACjC,MAAME,YAAYC,wCAChB;QAAE,MAAM;QAAS,aAAaJ;IAAW,GACzCE,KACA;QAAE,OAAO;QAAS,OAAO;QAAM,cAAcJ;IAAwB;IAGvE,OAAO;QACL,QAAQ;QACR,kBAAkBO,kBAAkBF,WAAWH,YAAYE;IAC7D;AACF;AAwGA,MAAMI,oBAAoB;IAAC;IAAqB;CAAoB;AACpE,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AAMpE,SAASC,eACPC,WAAuC,EACvCC,UAAoB,EACpBC,WAAmB;IAEnB,IAAI,CAACF,aAAa,OAAOE;IACzB,MAAMC,QAAQH,YAAY,IAAI,CAAC,CAACI,IAAMH,WAAW,QAAQ,CAACG,EAAE,IAAI;IAChE,OAAOD,QAAQA,MAAM,IAAI,GAAGD;AAC9B;AAEO,SAASG,uBACdC,MAAoB,EACpBd,IAAuC,EACvCQ,WAA4B;IAE5B,IAAI;QACF,OAAQM,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAMC,eAAeD;oBACrBnB,MAAM,4BAA4BoB;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWF;oBAejB,OAASE,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBrB,MAAM,yBAAyBsB;gCAC/B,MAAMC,SAASpB,+BACbmB,UAAU,OAAO,EACjBjB;gCAGF,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLkB;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBH;gCACxBrB,MAAM,gCAAgCwB;gCACtC,MAAMD,SAASpB,+BACbqB,gBAAgB,OAAO,EACvBnB;gCAGF,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLkB;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAME,aAAaJ;gCACnBrB,MAAM,0BAA0ByB;gCAEhC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOA,WAAW,IAAI;wCACxB;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAMC,cAAcL;gCACpBrB,MAAM,2BAA2B0B;gCAEjC,MAAMH,SAASpB,+BACbuB,YAAY,KAAK,EACjBrB;gCAIF,MAAMsB,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWE,KAAK,KAAK,CAClBC,YAAY5B,KAAK,MAAM,GAAIH;oCAE9B2B,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWE,KAAK,KAAK,CAClBD,YAAY1B,KAAK,KAAK,GAAIH;oCAE7B2B,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEA3B,MACE,CAAC,2BAA2B,EAAE6B,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLP;4CAEAO;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMQ,kBAAkBb;gCACxBrB,MAAM,gCAAgCkC;gCACtC,MAAMX,SAASpB,+BACb+B,gBAAgB,OAAO,EACvB7B;gCAGF,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLkB;wCACF;wCACA,SAASW,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAad;gCACnBrB,MAAM,0BAA0BmC;gCAChC,OAAO;oCACL;wCACE,MAAMvB,eACJC,aACAH,mBACA;wCAEF,OAAO,CAAC;wCACR,SAASyB,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAaf;gCACnBrB,MAAM,0BAA0BoC;gCAChC,OAAO;oCACL;wCACE,MAAMxB,eACJC,aACAF,mBACA;wCAEF,OAAO,CAAC;wCACR,SAASyB,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAahB;gCACnBrB,MAAM,0BAA0BqC;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAejB;gCACrBrB,MAAM,4BAA4BsC;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CAAE,KAAKA,aAAa,GAAG;wCAAC;wCAC/B,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGlB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIkB,MACR,CAAC,yBAAyB,EAAGpB,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAOqB,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrExC,MAAM,oBAAoByC;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}
+{"version":3,"file":"ai-model/models/auto-glm/actions.mjs","sources":["../../../../../src/ai-model/models/auto-glm/actions.ts"],"sourcesContent":["import type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { CoordinateDistanceAxis } from '../../shared/model-locate-result';\nimport type {\n  LocatePlanningAction,\n  ScrollPlanningAction,\n} from '../../shared/planning-action';\n\nconst debug = getDebug('auto-glm-actions');\n\ntype CoordinateDistanceToPixels = (\n  delta: number,\n  axis: CoordinateDistanceAxis,\n) => number;\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type AutoGLMParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nconst BACK_BUTTON_NAMES = ['AndroidBackButton', 'HarmonyBackButton'];\nconst HOME_BUTTON_NAMES = ['AndroidHomeButton', 'HarmonyHomeButton'];\n\n/**\n * Find the action name in actionSpace that matches one of the known names.\n * Falls back to defaultName if no match found or actionSpace is not provided.\n */\nfunction findActionName(\n  actionSpace: DeviceAction[] | undefined,\n  knownNames: string[],\n  defaultName: string,\n): string {\n  if (!actionSpace) return defaultName;\n  const match = actionSpace.find((a) => knownNames.includes(a.name));\n  return match ? match.name : defaultName;\n}\n\nexport function transformAutoGLMAction(\n  action: AutoGLMParsedAction,\n  {\n    actionSpace,\n    coordinateDistanceToPixels,\n  }: {\n    actionSpace?: DeviceAction[];\n    coordinateDistanceToPixels: CoordinateDistanceToPixels;\n  },\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate: {\n                    point: tapAction.element,\n                    prompt: '',\n                  },\n                },\n              } satisfies LocatePlanningAction<'Tap'>,\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate: {\n                    point: doubleTapAction.element,\n                    prompt: '',\n                  },\n                },\n              } satisfies LocatePlanningAction<'DoubleClick'>,\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            // Calculate horizontal and vertical delta in the model coordinate system.\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = coordinateDistanceToPixels(deltaY, 'y');\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = coordinateDistanceToPixels(deltaX, 'x');\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate: {\n                    point: swipeAction.start,\n                    prompt: '',\n                  },\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              } satisfies ScrollPlanningAction,\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n\n            return [\n              {\n                type: 'LongPress',\n                param: {\n                  locate: {\n                    point: longPressAction.element,\n                    prompt: '',\n                  },\n                },\n                thought: longPressAction.think || '',\n              } satisfies LocatePlanningAction<'LongPress'>,\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  BACK_BUTTON_NAMES,\n                  'AndroidBackButton',\n                ),\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  HOME_BUTTON_NAMES,\n                  'AndroidHomeButton',\n                ),\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: { uri: launchAction.app },\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["debug","getDebug","BACK_BUTTON_NAMES","HOME_BUTTON_NAMES","findActionName","actionSpace","knownNames","defaultName","match","a","transformAutoGLMAction","action","coordinateDistanceToPixels","finishAction","doAction","tapAction","doubleTapAction","typeAction","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";AASA,MAAMA,QAAQC,SAAS;AA6GvB,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AACpE,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AAMpE,SAASC,eACPC,WAAuC,EACvCC,UAAoB,EACpBC,WAAmB;IAEnB,IAAI,CAACF,aAAa,OAAOE;IACzB,MAAMC,QAAQH,YAAY,IAAI,CAAC,CAACI,IAAMH,WAAW,QAAQ,CAACG,EAAE,IAAI;IAChE,OAAOD,QAAQA,MAAM,IAAI,GAAGD;AAC9B;AAEO,SAASG,uBACdC,MAA2B,EAC3B,EACEN,WAAW,EACXO,0BAA0B,EAI3B;IAED,IAAI;QACF,OAAQD,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAME,eAAeF;oBACrBX,MAAM,4BAA4Ba;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWH;oBAejB,OAASG,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBd,MAAM,yBAAyBe;gCAE/B,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQ;gDACN,OAAOA,UAAU,OAAO;gDACxB,QAAQ;4CACV;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBF;gCACxBd,MAAM,gCAAgCgB;gCAEtC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQ;gDACN,OAAOA,gBAAgB,OAAO;gDAC9B,QAAQ;4CACV;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAaH;gCACnBd,MAAM,0BAA0BiB;gCAEhC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOA,WAAW,IAAI;wCACxB;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAMC,cAAcJ;gCACpBd,MAAM,2BAA2BkB;gCAGjC,MAAMC,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWV,2BAA2BQ,QAAQ;oCAC9CC,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWV,2BAA2BO,QAAQ;oCAC9CE,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEAnB,MACE,CAAC,2BAA2B,EAAEqB,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQ;gDACN,OAAOJ,YAAY,KAAK;gDACxB,QAAQ;4CACV;4CAEAI;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMQ,kBAAkBZ;gCACxBd,MAAM,gCAAgC0B;gCAEtC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQ;gDACN,OAAOA,gBAAgB,OAAO;gDAC9B,QAAQ;4CACV;wCACF;wCACA,SAASA,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAab;gCACnBd,MAAM,0BAA0B2B;gCAChC,OAAO;oCACL;wCACE,MAAMvB,eACJC,aACAH,mBACA;wCAEF,OAAO,CAAC;wCACR,SAASyB,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAad;gCACnBd,MAAM,0BAA0B4B;gCAChC,OAAO;oCACL;wCACE,MAAMxB,eACJC,aACAF,mBACA;wCAEF,OAAO,CAAC;wCACR,SAASyB,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAaf;gCACnBd,MAAM,0BAA0B6B;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAehB;gCACrBd,MAAM,4BAA4B8B;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CAAE,KAAKA,aAAa,GAAG;wCAAC;wCAC/B,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGjB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIiB,MACR,CAAC,yBAAyB,EAAGpB,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAOqB,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrEhC,MAAM,oBAAoBiC;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}

package/dist/es/ai-model/models/auto-glm/adapter.mjs

@@ -1,8 +1,6 @@
-import { autoGlmLocate } from "./locate.mjs";
-import { autoGlmPlanning } from "./planning.mjs";
-import { getAutoGLMChineseLocatePrompt, getAutoGLMChinesePlanPrompt, getAutoGLMMultilingualLocatePrompt, getAutoGLMMultilingualPlanPrompt } from "./prompt.mjs";
-const defaultAutoGlmReplanningCycleLimit = 100;
-function createAutoGlmAdapter({ getPlanPrompt, getLocatePrompt }) {
+import { createAutoGlmPlanningTapLocator } from "./locate.mjs";
+import { createAutoGlmPlanner } from "./planning.mjs";
+function createAutoGlmAdapter(isMultilingual) {
     return {
         chatCompletion: {
             unsupportedUserConfig: [
@@ -29,24 +27,18 @@ function createAutoGlmAdapter({ getPlanPrompt, getLocatePrompt }) {
         planning: {
             kind: 'custom',
             cacheEnabled: false,
-            defaultReplanningCycleLimit: defaultAutoGlmReplanningCycleLimit,
-            planFn: (userInstruction, options)=>autoGlmPlanning(userInstruction, options, getPlanPrompt)
+            defaultReplanningCycleLimit: 100,
+            planner: createAutoGlmPlanner(isMultilingual)
         },
         locate: {
             kind: 'custom',
-            locateFn: (elementDescription, options)=>autoGlmLocate(elementDescription, options, getLocatePrompt)
+            planningTapLocator: createAutoGlmPlanningTapLocator(isMultilingual)
         }
     };
 }
 const autoGlmAdapters = {
-    'auto-glm': createAutoGlmAdapter({
-        getPlanPrompt: getAutoGLMChinesePlanPrompt,
-        getLocatePrompt: getAutoGLMChineseLocatePrompt
-    }),
-    'auto-glm-multilingual': createAutoGlmAdapter({
-        getPlanPrompt: getAutoGLMMultilingualPlanPrompt,
-        getLocatePrompt: getAutoGLMMultilingualLocatePrompt
-    })
+    'auto-glm': createAutoGlmAdapter(false),
+    'auto-glm-multilingual': createAutoGlmAdapter(true)
 };
 export { autoGlmAdapters };
 

package/dist/es/ai-model/models/auto-glm/adapter.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/auto-glm/adapter.mjs","sources":["../../../../../src/ai-model/models/auto-glm/adapter.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type { ModelAdapterDefinition } from '../types';\nimport { autoGlmLocate } from './locate';\nimport { autoGlmPlanning } from './planning';\nimport {\n  getAutoGLMChineseLocatePrompt,\n  getAutoGLMChinesePlanPrompt,\n  getAutoGLMMultilingualLocatePrompt,\n  getAutoGLMMultilingualPlanPrompt,\n} from './prompt';\n\nconst defaultAutoGlmReplanningCycleLimit = 100;\n\nfunction createAutoGlmAdapter({\n  getPlanPrompt,\n  getLocatePrompt,\n}: {\n  getPlanPrompt: () => string;\n  getLocatePrompt: () => string;\n}): ModelAdapterDefinition {\n  return {\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => {\n        const commonOverrideConfig: Record<string, unknown> = {};\n\n        if (userConfig.temperature !== undefined) {\n          commonOverrideConfig.temperature = userConfig.temperature;\n        }\n\n        const modelSpecificConfig = {\n          top_p: 0.85,\n          frequency_penalty: 0.2,\n        };\n\n        return {\n          config: {\n            ...midsceneDefaults,\n            ...commonOverrideConfig,\n            ...modelSpecificConfig,\n          },\n        };\n      },\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: defaultAutoGlmReplanningCycleLimit,\n      planFn: (userInstruction, options) =>\n        autoGlmPlanning(userInstruction, options, getPlanPrompt),\n    },\n    locate: {\n      kind: 'custom',\n      locateFn: (elementDescription, options) =>\n        autoGlmLocate(elementDescription, options, getLocatePrompt),\n    },\n  };\n}\n\nexport const autoGlmAdapters = {\n  'auto-glm': createAutoGlmAdapter({\n    getPlanPrompt: getAutoGLMChinesePlanPrompt,\n    getLocatePrompt: getAutoGLMChineseLocatePrompt,\n  }),\n  'auto-glm-multilingual': createAutoGlmAdapter({\n    getPlanPrompt: getAutoGLMMultilingualPlanPrompt,\n    getLocatePrompt: getAutoGLMMultilingualLocatePrompt,\n  }),\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'auto-glm' | 'auto-glm-multilingual'\n>;\n"],"names":["defaultAutoGlmReplanningCycleLimit","createAutoGlmAdapter","getPlanPrompt","getLocatePrompt","midsceneDefaults","userConfig","commonOverrideConfig","undefined","modelSpecificConfig","userInstruction","options","autoGlmPlanning","elementDescription","autoGlmLocate","autoGlmAdapters","getAutoGLMChinesePlanPrompt","getAutoGLMChineseLocatePrompt","getAutoGLMMultilingualPlanPrompt","getAutoGLMMultilingualLocatePrompt"],"mappings":";;;AAWA,MAAMA,qCAAqC;AAE3C,SAASC,qBAAqB,EAC5BC,aAAa,EACbC,eAAe,EAIhB;IACC,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEC,gBAAgB,EAAEC,UAAU,EAAE;gBAC1D,MAAMC,uBAAgD,CAAC;gBAEvD,IAAID,AAA2BE,WAA3BF,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;gBAG3D,MAAMG,sBAAsB;oBAC1B,OAAO;oBACP,mBAAmB;gBACrB;gBAEA,OAAO;oBACL,QAAQ;wBACN,GAAGJ,gBAAgB;wBACnB,GAAGE,oBAAoB;wBACvB,GAAGE,mBAAmB;oBACxB;gBACF;YACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6BR;YAC7B,QAAQ,CAACS,iBAAiBC,UACxBC,gBAAgBF,iBAAiBC,SAASR;QAC9C;QACA,QAAQ;YACN,MAAM;YACN,UAAU,CAACU,oBAAoBF,UAC7BG,cAAcD,oBAAoBF,SAASP;QAC/C;IACF;AACF;AAEO,MAAMW,kBAAkB;IAC7B,YAAYb,qBAAqB;QAC/B,eAAec;QACf,iBAAiBC;IACnB;IACA,yBAAyBf,qBAAqB;QAC5C,eAAegB;QACf,iBAAiBC;IACnB;AACF"}
+{"version":3,"file":"ai-model/models/auto-glm/adapter.mjs","sources":["../../../../../src/ai-model/models/auto-glm/adapter.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport { createAutoGlmPlanningTapLocator } from './locate';\nimport { createAutoGlmPlanner } from './planning';\n\nfunction createAutoGlmAdapter(isMultilingual: boolean): ModelAdapterDefinition {\n  return {\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => {\n        const commonOverrideConfig: Record<string, unknown> = {};\n\n        if (userConfig.temperature !== undefined) {\n          commonOverrideConfig.temperature = userConfig.temperature;\n        }\n\n        const modelSpecificConfig = {\n          top_p: 0.85,\n          frequency_penalty: 0.2,\n        };\n\n        return {\n          config: {\n            ...midsceneDefaults,\n            ...commonOverrideConfig,\n            ...modelSpecificConfig,\n          },\n        };\n      },\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: 100,\n      planner: createAutoGlmPlanner(isMultilingual),\n    },\n    locate: {\n      kind: 'custom',\n      planningTapLocator: createAutoGlmPlanningTapLocator(isMultilingual),\n    },\n  };\n}\n\nexport const autoGlmAdapters = {\n  'auto-glm': createAutoGlmAdapter(false),\n  'auto-glm-multilingual': createAutoGlmAdapter(true),\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'auto-glm' | 'auto-glm-multilingual'\n>;\n"],"names":["createAutoGlmAdapter","isMultilingual","midsceneDefaults","userConfig","commonOverrideConfig","undefined","modelSpecificConfig","createAutoGlmPlanner","createAutoGlmPlanningTapLocator","autoGlmAdapters"],"mappings":";;AAKA,SAASA,qBAAqBC,cAAuB;IACnD,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEC,gBAAgB,EAAEC,UAAU,EAAE;gBAC1D,MAAMC,uBAAgD,CAAC;gBAEvD,IAAID,AAA2BE,WAA3BF,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;gBAG3D,MAAMG,sBAAsB;oBAC1B,OAAO;oBACP,mBAAmB;gBACrB;gBAEA,OAAO;oBACL,QAAQ;wBACN,GAAGJ,gBAAgB;wBACnB,GAAGE,oBAAoB;wBACvB,GAAGE,mBAAmB;oBACxB;gBACF;YACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6B;YAC7B,SAASC,qBAAqBN;QAChC;QACA,QAAQ;YACN,MAAM;YACN,oBAAoBO,gCAAgCP;QACtD;IACF;AACF;AAEO,MAAMQ,kBAAkB;IAC7B,YAAYT,qBAAqB;IACjC,yBAAyBA,qBAAqB;AAChD"}