npm - @midscene/core - Versions diffs - 1.17.2 → 1.17.3-beta-20261029072240.0 - Mend

@midscene/core 1.17.2 → 1.17.3-beta-20261029072240.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (41) hide show

package/dist/es/agent/task-builder.mjs CHANGED Viewed

@@ -269,7 +269,8 @@ class TaskBuilder {
                     rectFromXpath = await this.interface.rectMatchesCacheFeature({
                         xpaths: [
                             locateParam.xpath
-                        ]
+                        ],
+                        allowShadowHost: true
                     });
                 } catch  {}
                 const elementFromXpath = rectFromXpath ? transformLogicalElementToScreenshot({

package/dist/es/agent/task-builder.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport {\n type TMultimodalPrompt,\n findActionInActionSpaceOrThrow,\n userPromptToMultimodalPrompt,\n userPromptToString,\n validateRequiredLocateFields,\n} from '@/common';\nimport { type AbstractInterface, defineActionSleep } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n AIUsageInfo,\n DetailedLocateParam,\n DeviceAction,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n Rect,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ActionReadiness } from './action-readiness';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n ifLocateParamHasLocatedPixelResult,\n matchElementFromCache,\n matchElementFromPlan,\n transformLogicalElementToScreenshot,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/*\n Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nfunction invalidLocateElementReason(\n element: LocateResultElement,\n): string \| undefined {\n const values = [element.center?.[0], element.center?.[1]];\n if (\n element.center?.length !== 2 \|\|\n values.some((value) => typeof value !== 'number' \|\| !Number.isFinite(value))\n ) {\n return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n }\n return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n param: string \| DetailedLocateParam,\n): DetailedLocateParam {\n if (typeof param === 'string') {\n return { prompt: param };\n }\n\n const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n const deepLocate = rest.deepLocate ?? deepThink;\n\n return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string \| DetailedLocateParam) {\n const locate = normalizeLocateParam(param);\n const locatePlan: PlanningAction<DetailedLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n getDefaultModelRuntime: () => ModelRuntime;\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n actionSpace: DeviceAction[];\n waitAfterAction?: number;\n actionReadiness?: ActionReadiness;\n operationResource?: object;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n multimodalPrompt?: TMultimodalPrompt;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n planningModel: ModelRuntime;\n locateModel: ModelRuntime;\n cacheable?: boolean;\n multimodalPrompt?: TMultimodalPrompt;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n private readonly getDefaultModelRuntime: () => ModelRuntime;\n\n private readonly taskCache?: TaskCache;\n\n private readonly actionSpace: DeviceAction[];\n\n private readonly waitAfterAction?: number;\n private readonly actionReadiness?: ActionReadiness;\n private readonly operationResource: object;\n\n constructor({\n getDefaultModelRuntime,\n interfaceInstance,\n service,\n taskCache,\n actionSpace,\n waitAfterAction,\n actionReadiness,\n operationResource,\n }: TaskBuilderDeps) {\n this.getDefaultModelRuntime = getDefaultModelRuntime;\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n this.actionSpace = actionSpace;\n this.waitAfterAction = waitAfterAction;\n this.actionReadiness = actionReadiness;\n this.operationResource = operationResource ?? interfaceInstance;\n }\n\n public async build(\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n locateModel: ModelRuntime,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n planningModel,\n locateModel,\n cacheable,\n multimodalPrompt: options?.multimodalPrompt,\n deepLocate: options?.deepLocate,\n abortSignal: options?.abortSignal,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> \| void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<DetailedLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n context.abortSignal?.throwIfAborted();\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<DetailedLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = this.actionSpace;\n const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n const param = plan.param;\n\n const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n validateRequiredLocateFields(param, action.paramSchema);\n\n locateFields.forEach((field) => {\n if (param?.[field]) {\n // Always use createLocateTask for all locate params.\n // This ensures cache writing happens even when locatedPixelResult is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasLocatedPixelResult=${ifLocateParamHasLocatedPixelResult(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n planningFeedbackMaxLength: action.planningFeedbackMaxLength,\n executor: async (taskContext) => {\n const abortSignal = context.abortSignal ?? taskContext.signal;\n const timing = taskContext.task.timing;\n\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n validateRequiredLocateFields(param, action.paramSchema);\n\n const { shrunkShotToLogicalRatio } = uiContext;\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in Action task',\n );\n }\n const parsedParam = (() => {\n if (!action.paramSchema) return param;\n try {\n return parseActionParam(param, action.paramSchema, {\n shrunkShotToLogicalRatio,\n });\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n })();\n\n const useDefaultWait = !this.actionReadiness;\n taskContext.task.actionReadiness = useDefaultWait\n ? 'default'\n : 'custom';\n abortSignal?.throwIfAborted();\n\n setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n const delayBeforeRunner = useDefaultWait\n ? (action.delayBeforeRunner ?? 200)\n : 0;\n try {\n const preparation = await Promise.allSettled([\n this.interface.beforeInvokeAction?.(\n action.name,\n param,\n abortSignal,\n ),\n delayBeforeRunner > 0\n ? defineActionSleep(abortSignal).call({\n timeMs: delayBeforeRunner,\n })\n : Promise.resolve(),\n ]);\n const rejected = preparation.find(\n (item) => item.status === 'rejected',\n );\n if (rejected?.status === 'rejected') throw rejected.reason;\n } catch (originalError: any) {\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalError?.message \|\| String(originalError)}`,\n { cause: originalError },\n );\n } finally {\n setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n }\n\n abortSignal?.throwIfAborted();\n setTimingFieldOnce(timing, 'callActionStart');\n debug('calling action', action.name);\n const actionResult = await action.call.call(\n this.interface,\n parsedParam,\n useDefaultWait\n ? { ...taskContext, signal: abortSignal }\n : { ...taskContext, signal: abortSignal, skipDefaultWait: true },\n );\n setTimingFieldOnce(timing, 'callActionEnd');\n debug('called action', action.name, 'result:', actionResult);\n abortSignal?.throwIfAborted();\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n setTimingFieldOnce(timing, 'waitForActionReadyStart');\n try {\n if (useDefaultWait) {\n const delayAfterRunner =\n action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n if (delayAfterRunner > 0)\n await defineActionSleep(abortSignal).call({\n timeMs: delayAfterRunner,\n });\n await this.interface.defaultActionWait?.(\n action.name,\n parsedParam,\n abortSignal,\n );\n } else {\n await this.actionReadiness!.wait(\n {\n id: taskContext.task.taskId,\n name: action.name,\n param: parsedParam,\n },\n abortSignal,\n this.operationResource,\n );\n }\n } finally {\n setTimingFieldOnce(timing, 'waitForActionReadyEnd');\n }\n abortSignal?.throwIfAborted();\n try {\n await this.interface.afterInvokeAction?.(\n action.name,\n parsedParam,\n abortSignal,\n );\n } catch (originalError: any) {\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalError?.message \|\| String(originalError)}`,\n { cause: originalError },\n );\n } finally {\n setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n }\n return { output: actionResult };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<DetailedLocateParam>,\n detailedLocateParam: DetailedLocateParam \| string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, locateModel, deepLocate } = context;\n\n let locateParam = normalizeLocateParam(detailedLocateParam);\n\n // Planning produces text locators; carry the user's reference images into\n // independent grounding calls while preserving explicit locator images.\n if (\n context.multimodalPrompt?.images?.length &&\n locateParam.prompt &&\n !userPromptToMultimodalPrompt(locateParam.prompt)?.images?.length\n ) {\n locateParam = {\n ...locateParam,\n prompt: {\n ...context.multimodalPrompt,\n prompt: userPromptToString(locateParam.prompt),\n },\n };\n }\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n if (deepLocate && !locateParam.deepLocate) {\n locateParam = {\n ...locateParam,\n deepLocate: true,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n param: locateParam,\n thought: plan.thought,\n executor: async (taskContext) => {\n const abortSignal = context.abortSignal ?? taskContext.signal;\n abortSignal?.throwIfAborted();\n const { task } = taskContext;\n let { uiContext } = taskContext;\n const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(\n locateParam,\n )\n ? locateParam\n : undefined;\n\n assert(\n locateParam?.prompt \|\| paramWithLocatedPixelResult,\n `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(\n locateParam,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn(abortSignal);\n abortSignal?.throwIfAborted();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in locate task',\n );\n }\n\n let locateDump: ServiceDump \| undefined;\n let locateResult: LocateResultWithDump \| undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,\n searchAreaRawChoiceMessage:\n dump.taskInfo?.searchAreaRawChoiceMessage,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'locate');\n task.searchArea = dump.taskInfo?.searchArea;\n if (dump.taskInfo?.searchAreaUsage) {\n // The deepLocate search-area call belongs to the same locate task,\n // so it shares the companion usage intent.\n task.searchAreaUsage = withUsageIntent(\n dump.taskInfo.searchAreaUsage,\n 'locate',\n );\n }\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n const planLocatedElement = paramWithLocatedPixelResult\n ? matchElementFromPlan(paramWithLocatedPixelResult)\n : undefined;\n\n // Separate planning normally returns target descriptions, but may still\n // return coordinates. Reuse them even with a dedicated Locate model,\n // preserving the existing direct-plan-hit behavior.\n // when deepLocate is enabled, locatedPixelResult should be used as search\n // area hint, not as a final direct hit\n const elementFromPlan = locateParam.deepLocate\n ? undefined\n : planLocatedElement;\n const isPlanDirectHit = !!elementFromPlan;\n\n // from xpath\n let rectFromXpath: Rect \| undefined;\n if (\n !isPlanDirectHit &&\n locateParam.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [locateParam.xpath],\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n\n const elementFromXpath = rectFromXpath\n ? transformLogicalElementToScreenshot(\n {\n center: [\n rectFromXpath.left + rectFromXpath.width / 2,\n rectFromXpath.top + rectFromXpath.height / 2,\n ],\n description:\n typeof locateParam.prompt === 'string'\n ? locateParam.prompt\n : locateParam.prompt?.prompt \|\| '',\n rect: rectFromXpath,\n },\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = locateParam.prompt;\n const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCacheResult =\n isPlanDirectHit \|\| isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n locateParam.cacheable,\n );\n\n // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n const elementFromCache = elementFromCacheResult\n ? transformLogicalElementToScreenshot(\n elementFromCacheResult,\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isCacheHit = !!elementFromCache;\n\n let elementFromAiLocate: LocateResultElement \| null \| undefined;\n const timing = taskContext.task.timing;\n if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n locateResult = await this.service.locate(\n locateParam,\n {\n context: uiContext,\n planLocatedElement,\n },\n locateModel,\n abortSignal,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n }\n\n const element =\n elementFromPlan \|\|\n elementFromXpath \|\|\n elementFromCache \|\|\n elementFromAiLocate;\n\n if (element) {\n const invalidElementReason = invalidLocateElementReason(element);\n if (invalidElementReason) {\n if (locateDump) {\n throw new ServiceError(invalidElementReason, locateDump);\n }\n throw new Error(invalidElementReason);\n }\n }\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature \| undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read)\n // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n !isCacheHit &&\n (!isPlanDirectHit \|\| !locateCacheAlreadyExists) &&\n locateParam.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForPoint) {\n try {\n // Transform coordinates to logical space for cacheFeatureForPoint\n // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n const pointForCache: [number, number] = [\n Math.round(element.center[0] / shrunkShotToLogicalRatio),\n Math.round(element.center[1] / shrunkShotToLogicalRatio),\n ];\n\n const defaultModel = this.getDefaultModelRuntime();\n const feature = await this.interface.cacheFeatureForPoint(\n pointForCache,\n {\n targetDescription:\n typeof locateParam.prompt === 'string'\n ? locateParam.prompt\n : locateParam.prompt?.prompt,\n modelRuntime: {\n ...defaultModel,\n executionId: locateModel.executionId,\n },\n },\n );\n if (hasNonEmptyCache(feature)) {\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n currentCacheEntry = feature;\n this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForPoint failed: %s', error);\n }\n } else {\n debug('cacheFeatureForPoint is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${locateParam.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${locateParam.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy \| undefined;\n\n if (isPlanDirectHit && paramWithLocatedPixelResult) {\n hitBy = {\n from: 'Plan',\n context: {\n locatedPixelResult:\n paramWithLocatedPixelResult.locatedPixelResult,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: locateParam.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n const promptDisplay = locateParam.promptDisplay;\n const elementForAction = promptDisplay\n ? {\n ...element,\n description: promptDisplay,\n }\n : element;\n\n if (promptDisplay && locateDump?.matchedElement) {\n locateDump.matchedElement = locateDump.matchedElement.map(\n (matchedElement) => ({\n ...matchedElement,\n description: promptDisplay,\n }),\n );\n }\n\n onResult?.(elementForAction);\n\n return {\n output: {\n element: {\n ...elementForAction,\n // backward compatibility for aiLocate, which return value needs a dpr field\n dpr: uiContext.deprecatedDpr,\n },\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","locateModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","validateRequiredLocateFields","field","ifLocateParamHasLocatedPixelResult","locateTask","result","task","taskContext","abortSignal","timing","uiContext","assert","shrunkShotToLogicalRatio","Error","parsedParam","parseActionParam","error","useDefaultWait","setTimingFieldOnce","delayBeforeRunner","preparation","Promise","defineActionSleep","rejected","item","originalError","String","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","userPromptToMultimodalPrompt","userPromptToString","taskLocator","paramWithLocatedPixelResult","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","transformLogicalElementToScreenshot","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","defaultModel","feature","hitBy","promptDisplay","elementForAction","matchedElement","getDefaultModelRuntime","interfaceInstance","service","taskCache","waitAfterAction","actionReadiness","operationResource"],"mappings":";;;;;;;;;;;;;;;;;;;AAwCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QAACD,QAAQ,MAAM,EAAE,CAAC,EAAE;QAAEA,QAAQ,MAAM,EAAE,CAAC,EAAE;KAAC;IACzD,IACEA,QAAQ,MAAM,EAAE,WAAW,KAC3BC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;AAG1E;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AA8BO,MAAMC;IAkCX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,WAAyB,EACzBC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,kBAAkBF,SAAS;YAC3B,YAAYA,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxBM,QAAQ,WAAW,EAAE;YACrB,MAAMK,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnEI,6BAA6B7B,OAAOyB,OAAO,WAAW;QAEtDE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,OAAO,CAAC8B,MAAM,EAAE;gBAGlB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,sBAAsB,EAAEwB,mCAAmC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OACE5C,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;QAEnE;QAEA,MAAMW,OAKF;YACF,MAAM;YACN,SAASX;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,2BAA2BO,OAAO,yBAAyB;YAC3D,UAAU,OAAOU;gBACf,MAAMC,cAAcrB,QAAQ,WAAW,IAAIoB,YAAY,MAAM;gBAC7D,MAAME,SAASF,YAAY,IAAI,CAAC,MAAM;gBAEtC/C,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEmC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAMG,YAAYH,YAAY,SAAS;gBACvCI,OAAOD,WAAW;gBAElBT,6BAA6B7B,OAAOyB,OAAO,WAAW;gBAEtD,MAAM,EAAEe,wBAAwB,EAAE,GAAGF;gBACrC,IAAIE,AAA6BpC,WAA7BoC,0BACF,MAAM,IAAIC,MACR;gBAGJ,MAAMC,cAAe,AAAC;oBACpB,IAAI,CAACjB,OAAO,WAAW,EAAE,OAAOzB;oBAChC,IAAI;wBACF,OAAO2C,iBAAiB3C,OAAOyB,OAAO,WAAW,EAAE;4BACjDe;wBACF;oBACF,EAAE,OAAOI,OAAY;wBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAEmB,MAAM,OAAO,CAAC,cAAc,EAAE9C,KAAK,SAAS,CAACE,QAAQ,EACtG;4BAAE,OAAO4C;wBAAM;oBAEnB;gBACF;gBAEA,MAAMC,iBAAiB,CAAC,IAAI,CAAC,eAAe;gBAC5CV,YAAY,IAAI,CAAC,eAAe,GAAGU,iBAC/B,YACA;gBACJT,aAAa;gBAEbU,mBAAmBT,QAAQ;gBAC3B,MAAMU,oBAAoBF,iBACrBpB,OAAO,iBAAiB,IAAI,MAC7B;gBACJ,IAAI;oBACF,MAAMuB,cAAc,MAAMC,QAAQ,UAAU,CAAC;wBAC3C,IAAI,CAAC,SAAS,CAAC,kBAAkB,GAC/BxB,OAAO,IAAI,EACXzB,OACAoC;wBAEFW,oBAAoB,IAChBG,kBAAkBd,aAAa,IAAI,CAAC;4BAClC,QAAQW;wBACV,KACAE,QAAQ,OAAO;qBACpB;oBACD,MAAME,WAAWH,YAAY,IAAI,CAC/B,CAACI,OAASA,AAAgB,eAAhBA,KAAK,MAAM;oBAEvB,IAAID,UAAU,WAAW,YAAY,MAAMA,SAAS,MAAM;gBAC5D,EAAE,OAAOE,eAAoB;oBAC3B,MAAM,IAAIZ,MACR,CAAC,wCAAwC,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAE4B,eAAe,WAAWC,OAAOD,gBAAgB,EAC5G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRP,mBAAmBT,QAAQ;gBAC7B;gBAEAD,aAAa;gBACbU,mBAAmBT,QAAQ;gBAC3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAM8B,eAAe,MAAM9B,OAAO,IAAI,CAAC,IAAI,CACzC,IAAI,CAAC,SAAS,EACdiB,aACAG,iBACI;oBAAE,GAAGV,WAAW;oBAAE,QAAQC;gBAAY,IACtC;oBAAE,GAAGD,WAAW;oBAAE,QAAQC;oBAAa,iBAAiB;gBAAK;gBAEnEU,mBAAmBT,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW8B;gBAC/CnB,aAAa;gBAEbU,mBAAmBT,QAAQ;gBAC3BS,mBAAmBT,QAAQ;gBAC3B,IAAI;oBACF,IAAIQ,gBAAgB;wBAClB,MAAMW,mBACJ/B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;wBACrD,IAAI+B,mBAAmB,GACrB,MAAMN,kBAAkBd,aAAa,IAAI,CAAC;4BACxC,QAAQoB;wBACV;wBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GACpC/B,OAAO,IAAI,EACXiB,aACAN;oBAEJ,OACE,MAAM,IAAI,CAAC,eAAe,CAAE,IAAI,CAC9B;wBACE,IAAID,YAAY,IAAI,CAAC,MAAM;wBAC3B,MAAMV,OAAO,IAAI;wBACjB,OAAOiB;oBACT,GACAN,aACA,IAAI,CAAC,iBAAiB;gBAG5B,SAAU;oBACRU,mBAAmBT,QAAQ;gBAC7B;gBACAD,aAAa;gBACb,IAAI;oBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GACpCX,OAAO,IAAI,EACXiB,aACAN;gBAEJ,EAAE,OAAOiB,eAAoB;oBAC3B,MAAM,IAAIZ,MACR,CAAC,uCAAuC,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAE4B,eAAe,WAAWC,OAAOD,gBAAgB,EAC3G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRP,mBAAmBT,QAAQ;gBAC7B;gBACA,OAAO;oBAAE,QAAQkB;gBAAa;YAChC;QACF;QAEAxC,QAAQ,KAAK,CAAC,IAAI,CAACmB;IACrB;IAEQ,iBACNhB,IAAyC,EACzCuC,mBAAiD,EACjD1C,OAAyB,EACzB2C,QAAgD,EACd;QAClC,MAAM,EAAE5C,SAAS,EAAEH,WAAW,EAAER,UAAU,EAAE,GAAGY;QAE/C,IAAI4C,cAAc5D,qBAAqB0D;QAIvC,IACE1C,QAAQ,gBAAgB,EAAE,QAAQ,UAClC4C,YAAY,MAAM,IAClB,CAACC,6BAA6BD,YAAY,MAAM,GAAG,QAAQ,QAE3DA,cAAc;YACZ,GAAGA,WAAW;YACd,QAAQ;gBACN,GAAG5C,QAAQ,gBAAgB;gBAC3B,QAAQ8C,mBAAmBF,YAAY,MAAM;YAC/C;QACF;QAGF,IAAI7C,AAAcV,WAAdU,WACF6C,cAAc;YACZ,GAAGA,WAAW;YACd7C;QACF;QAGF,IAAIX,cAAc,CAACwD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMG,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOH;YACP,SAASzC,KAAK,OAAO;YACrB,UAAU,OAAOiB;gBACf,MAAMC,cAAcrB,QAAQ,WAAW,IAAIoB,YAAY,MAAM;gBAC7DC,aAAa;gBACb,MAAM,EAAEF,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEG,SAAS,EAAE,GAAGH;gBACpB,MAAM4B,8BAA8BhC,mCAClC4B,eAEEA,cACAvD;gBAEJmC,OACEoB,aAAa,UAAUI,6BACvB,CAAC,mEAAmE,EAAEjE,KAAK,SAAS,CAClF6D,cACC;gBAGL,IAAI,CAACrB,WAAW;oBACdA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAACF;oBAClDA,aAAa;gBACf;gBAEAG,OAAOD,WAAW;gBAElB,MAAM,EAAEE,wBAAwB,EAAE,GAAGF;gBAErC,IAAIE,AAA6BpC,WAA7BoC,0BACF,MAAM,IAAIC,MACR;gBAIJ,IAAIuB;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACbjC,KAAK,GAAG,GAAG;wBACTiC;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,oBAAoBA,KAAK,QAAQ,EAAE;wBACnC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACAjC,KAAK,KAAK,GAAGkC,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnDjC,KAAK,UAAU,GAAGiC,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjBjC,KAAK,eAAe,GAAGkC,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjBjC,KAAK,iBAAiB,GAAGiC,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,8BACvBO,qBAAqBP,+BACrB3D;gBAOJ,MAAMmE,kBAAkBZ,YAAY,UAAU,GAC1CvD,SACAiE;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDb,YAAY,KAAK,IACjB,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFc,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACd,YAAY,KAAK;yBAAC;oBAC7B;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMe,mBAAmBD,gBACrBE,oCACE;oBACE,QAAQ;wBACNF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;wBAC3CA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;qBAC5C;oBACD,aACE,AAA8B,YAA9B,OAAOd,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE,UAAU;oBACpC,MAAMc;gBACR,GACAjC,4BAEFpC;gBAEJ,MAAMwE,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAclB,YAAY,MAAM;gBACtC,MAAMmB,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJR,mBAAmBI,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAlB,YAAY,SAAS;gBAI7B,MAAMuB,mBAAmBF,yBACrBL,oCACEK,wBACAxC,4BAEFpC;gBAEJ,MAAM+E,aAAa,CAAC,CAACD;gBAErB,IAAIE;gBACJ,MAAM/C,SAASF,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACyC,cAAc,CAACO,cAAc,CAACX,iBACjC,IAAI;oBACF1B,mBAAmBT,QAAQ;oBAC3B4B,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCN,aACA;wBACE,SAASrB;wBACT+B;oBACF,GACA1D,aACAyB;oBAEF8B,UAAUD,aAAa,IAAI;oBAC3BmB,sBAAsBnB,aAAa,OAAO;gBAC5C,EAAE,OAAOrB,OAAO;oBACd,IAAIA,iBAAiByC,cACnBnB,UAAUtB,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRE,mBAAmBT,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJ6E,mBACAG,oBACAQ,oBACAE;gBAEF,IAAI1F,SAAS;oBACX,MAAM4F,uBAAuB7F,2BAA2BC;oBACxD,IAAI4F,sBAAsB;wBACxB,IAAItB,YACF,MAAM,IAAIqB,aAAaC,sBAAsBtB;wBAE/C,MAAM,IAAIvB,MAAM6C;oBAClB;gBACF;gBAGA,MAAMC,2BAA2BjG,iBAC/BwF,mBAAmB,cAAc;gBAGnC,IAAIU;gBAOJ,IACE9F,WACA,IAAI,CAAC,SAAS,IACd,CAACyF,cACA,EAACX,mBAAmB,CAACe,wBAAuB,KAC7C5B,AAA0B,UAA1BA,YAAY,SAAS,EAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,MAAM8B,gBAAkC;wBACtCC,KAAK,KAAK,CAAChG,QAAQ,MAAM,CAAC,EAAE,GAAG8C;wBAC/BkD,KAAK,KAAK,CAAChG,QAAQ,MAAM,CAAC,EAAE,GAAG8C;qBAChC;oBAED,MAAMmD,eAAe,IAAI,CAAC,sBAAsB;oBAChD,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDH,eACA;wBACE,mBACE,AAA8B,YAA9B,OAAO9B,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE;wBAC1B,cAAc;4BACZ,GAAGgC,YAAY;4BACf,aAAahF,YAAY,WAAW;wBACtC;oBACF;oBAEF,IAAIrB,iBAAiBsG,UAAU;wBAC7BxG,MACE,uCACAyF,aACAe;wBAEFJ,oBAAoBI;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACE1F,MACE,yDACAyF;gBAGN,EAAE,OAAOjC,OAAO;oBACdxD,MAAM,mCAAmCwD;gBAC3C;qBAEAxD,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIsE,YACF,MAAM,IAAIqB,aACR,CAAC,oBAAoB,EAAE1B,YAAY,MAAM,EAAE,EAC3CK;oBAGJ,MAAM,IAAIvB,MAAM,CAAC,mBAAmB,EAAEkB,YAAY,MAAM,EAAE;gBAC5D;gBAEA,IAAIkC;gBAEJ,IAAIrB,mBAAmBT,6BACrB8B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,oBACE9B,4BAA4B,kBAAkB;oBAClD;gBACF;qBACK,IAAIa,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOlC,YAAY,KAAK;oBAC1B;gBACF;qBACK,IAAIwB,YACTU,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaS;oBACf;gBACF;gBAGF,MAAMM,gBAAgBnC,YAAY,aAAa;gBAC/C,MAAMoC,mBAAmBD,gBACrB;oBACE,GAAGpG,OAAO;oBACV,aAAaoG;gBACf,IACApG;gBAEJ,IAAIoG,iBAAiB9B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAACgC,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJpC,WAAWqC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKzD,UAAU,aAAa;wBAC9B;oBACF;oBACAuD;gBACF;YACF;QACF;QAEA,OAAO/B;IACT;IAnoBA,YAAY,EACVmC,sBAAsB,EACtBC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACT5E,WAAW,EACX6E,eAAe,EACfC,eAAe,EACfC,iBAAiB,EACD,CAAE;QAtBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QACA,uBAAiB,0BAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QACA,uBAAiB,mBAAjB;QACA,uBAAiB,qBAAjB;QAYE,IAAI,CAAC,sBAAsB,GAAGN;QAC9B,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAG5E;QACnB,IAAI,CAAC,eAAe,GAAG6E;QACvB,IAAI,CAAC,eAAe,GAAGC;QACvB,IAAI,CAAC,iBAAiB,GAAGC,qBAAqBL;IAChD;AAknBF"}
1	+ {"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport {\n type TMultimodalPrompt,\n findActionInActionSpaceOrThrow,\n userPromptToMultimodalPrompt,\n userPromptToString,\n validateRequiredLocateFields,\n} from '@/common';\nimport { type AbstractInterface, defineActionSleep } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n AIUsageInfo,\n DetailedLocateParam,\n DeviceAction,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n Rect,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ActionReadiness } from './action-readiness';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n ifLocateParamHasLocatedPixelResult,\n matchElementFromCache,\n matchElementFromPlan,\n transformLogicalElementToScreenshot,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/*\n Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nfunction invalidLocateElementReason(\n element: LocateResultElement,\n): string \| undefined {\n const values = [element.center?.[0], element.center?.[1]];\n if (\n element.center?.length !== 2 \|\|\n values.some((value) => typeof value !== 'number' \|\| !Number.isFinite(value))\n ) {\n return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n }\n return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n param: string \| DetailedLocateParam,\n): DetailedLocateParam {\n if (typeof param === 'string') {\n return { prompt: param };\n }\n\n const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n const deepLocate = rest.deepLocate ?? deepThink;\n\n return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string \| DetailedLocateParam) {\n const locate = normalizeLocateParam(param);\n const locatePlan: PlanningAction<DetailedLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n getDefaultModelRuntime: () => ModelRuntime;\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n actionSpace: DeviceAction[];\n waitAfterAction?: number;\n actionReadiness?: ActionReadiness;\n operationResource?: object;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n multimodalPrompt?: TMultimodalPrompt;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n planningModel: ModelRuntime;\n locateModel: ModelRuntime;\n cacheable?: boolean;\n multimodalPrompt?: TMultimodalPrompt;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n private readonly getDefaultModelRuntime: () => ModelRuntime;\n\n private readonly taskCache?: TaskCache;\n\n private readonly actionSpace: DeviceAction[];\n\n private readonly waitAfterAction?: number;\n private readonly actionReadiness?: ActionReadiness;\n private readonly operationResource: object;\n\n constructor({\n getDefaultModelRuntime,\n interfaceInstance,\n service,\n taskCache,\n actionSpace,\n waitAfterAction,\n actionReadiness,\n operationResource,\n }: TaskBuilderDeps) {\n this.getDefaultModelRuntime = getDefaultModelRuntime;\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n this.actionSpace = actionSpace;\n this.waitAfterAction = waitAfterAction;\n this.actionReadiness = actionReadiness;\n this.operationResource = operationResource ?? interfaceInstance;\n }\n\n public async build(\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n locateModel: ModelRuntime,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n planningModel,\n locateModel,\n cacheable,\n multimodalPrompt: options?.multimodalPrompt,\n deepLocate: options?.deepLocate,\n abortSignal: options?.abortSignal,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> \| void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<DetailedLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n context.abortSignal?.throwIfAborted();\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<DetailedLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = this.actionSpace;\n const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n const param = plan.param;\n\n const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n validateRequiredLocateFields(param, action.paramSchema);\n\n locateFields.forEach((field) => {\n if (param?.[field]) {\n // Always use createLocateTask for all locate params.\n // This ensures cache writing happens even when locatedPixelResult is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasLocatedPixelResult=${ifLocateParamHasLocatedPixelResult(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n planningFeedbackMaxLength: action.planningFeedbackMaxLength,\n executor: async (taskContext) => {\n const abortSignal = context.abortSignal ?? taskContext.signal;\n const timing = taskContext.task.timing;\n\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n validateRequiredLocateFields(param, action.paramSchema);\n\n const { shrunkShotToLogicalRatio } = uiContext;\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in Action task',\n );\n }\n const parsedParam = (() => {\n if (!action.paramSchema) return param;\n try {\n return parseActionParam(param, action.paramSchema, {\n shrunkShotToLogicalRatio,\n });\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n })();\n\n const useDefaultWait = !this.actionReadiness;\n taskContext.task.actionReadiness = useDefaultWait\n ? 'default'\n : 'custom';\n abortSignal?.throwIfAborted();\n\n setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n const delayBeforeRunner = useDefaultWait\n ? (action.delayBeforeRunner ?? 200)\n : 0;\n try {\n const preparation = await Promise.allSettled([\n this.interface.beforeInvokeAction?.(\n action.name,\n param,\n abortSignal,\n ),\n delayBeforeRunner > 0\n ? defineActionSleep(abortSignal).call({\n timeMs: delayBeforeRunner,\n })\n : Promise.resolve(),\n ]);\n const rejected = preparation.find(\n (item) => item.status === 'rejected',\n );\n if (rejected?.status === 'rejected') throw rejected.reason;\n } catch (originalError: any) {\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalError?.message \|\| String(originalError)}`,\n { cause: originalError },\n );\n } finally {\n setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n }\n\n abortSignal?.throwIfAborted();\n setTimingFieldOnce(timing, 'callActionStart');\n debug('calling action', action.name);\n const actionResult = await action.call.call(\n this.interface,\n parsedParam,\n useDefaultWait\n ? { ...taskContext, signal: abortSignal }\n : { ...taskContext, signal: abortSignal, skipDefaultWait: true },\n );\n setTimingFieldOnce(timing, 'callActionEnd');\n debug('called action', action.name, 'result:', actionResult);\n abortSignal?.throwIfAborted();\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n setTimingFieldOnce(timing, 'waitForActionReadyStart');\n try {\n if (useDefaultWait) {\n const delayAfterRunner =\n action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n if (delayAfterRunner > 0)\n await defineActionSleep(abortSignal).call({\n timeMs: delayAfterRunner,\n });\n await this.interface.defaultActionWait?.(\n action.name,\n parsedParam,\n abortSignal,\n );\n } else {\n await this.actionReadiness!.wait(\n {\n id: taskContext.task.taskId,\n name: action.name,\n param: parsedParam,\n },\n abortSignal,\n this.operationResource,\n );\n }\n } finally {\n setTimingFieldOnce(timing, 'waitForActionReadyEnd');\n }\n abortSignal?.throwIfAborted();\n try {\n await this.interface.afterInvokeAction?.(\n action.name,\n parsedParam,\n abortSignal,\n );\n } catch (originalError: any) {\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalError?.message \|\| String(originalError)}`,\n { cause: originalError },\n );\n } finally {\n setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n }\n return { output: actionResult };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<DetailedLocateParam>,\n detailedLocateParam: DetailedLocateParam \| string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, locateModel, deepLocate } = context;\n\n let locateParam = normalizeLocateParam(detailedLocateParam);\n\n // Planning produces text locators; carry the user's reference images into\n // independent grounding calls while preserving explicit locator images.\n if (\n context.multimodalPrompt?.images?.length &&\n locateParam.prompt &&\n !userPromptToMultimodalPrompt(locateParam.prompt)?.images?.length\n ) {\n locateParam = {\n ...locateParam,\n prompt: {\n ...context.multimodalPrompt,\n prompt: userPromptToString(locateParam.prompt),\n },\n };\n }\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n if (deepLocate && !locateParam.deepLocate) {\n locateParam = {\n ...locateParam,\n deepLocate: true,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n param: locateParam,\n thought: plan.thought,\n executor: async (taskContext) => {\n const abortSignal = context.abortSignal ?? taskContext.signal;\n abortSignal?.throwIfAborted();\n const { task } = taskContext;\n let { uiContext } = taskContext;\n const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(\n locateParam,\n )\n ? locateParam\n : undefined;\n\n assert(\n locateParam?.prompt \|\| paramWithLocatedPixelResult,\n `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(\n locateParam,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn(abortSignal);\n abortSignal?.throwIfAborted();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in locate task',\n );\n }\n\n let locateDump: ServiceDump \| undefined;\n let locateResult: LocateResultWithDump \| undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,\n searchAreaRawChoiceMessage:\n dump.taskInfo?.searchAreaRawChoiceMessage,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'locate');\n task.searchArea = dump.taskInfo?.searchArea;\n if (dump.taskInfo?.searchAreaUsage) {\n // The deepLocate search-area call belongs to the same locate task,\n // so it shares the companion usage intent.\n task.searchAreaUsage = withUsageIntent(\n dump.taskInfo.searchAreaUsage,\n 'locate',\n );\n }\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n const planLocatedElement = paramWithLocatedPixelResult\n ? matchElementFromPlan(paramWithLocatedPixelResult)\n : undefined;\n\n // Separate planning normally returns target descriptions, but may still\n // return coordinates. Reuse them even with a dedicated Locate model,\n // preserving the existing direct-plan-hit behavior.\n // when deepLocate is enabled, locatedPixelResult should be used as search\n // area hint, not as a final direct hit\n const elementFromPlan = locateParam.deepLocate\n ? undefined\n : planLocatedElement;\n const isPlanDirectHit = !!elementFromPlan;\n\n // from xpath\n let rectFromXpath: Rect \| undefined;\n if (\n !isPlanDirectHit &&\n locateParam.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [locateParam.xpath],\n allowShadowHost: true,\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n\n const elementFromXpath = rectFromXpath\n ? transformLogicalElementToScreenshot(\n {\n center: [\n rectFromXpath.left + rectFromXpath.width / 2,\n rectFromXpath.top + rectFromXpath.height / 2,\n ],\n description:\n typeof locateParam.prompt === 'string'\n ? locateParam.prompt\n : locateParam.prompt?.prompt \|\| '',\n rect: rectFromXpath,\n },\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = locateParam.prompt;\n const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCacheResult =\n isPlanDirectHit \|\| isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n locateParam.cacheable,\n );\n\n // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n const elementFromCache = elementFromCacheResult\n ? transformLogicalElementToScreenshot(\n elementFromCacheResult,\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isCacheHit = !!elementFromCache;\n\n let elementFromAiLocate: LocateResultElement \| null \| undefined;\n const timing = taskContext.task.timing;\n if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n locateResult = await this.service.locate(\n locateParam,\n {\n context: uiContext,\n planLocatedElement,\n },\n locateModel,\n abortSignal,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n }\n\n const element =\n elementFromPlan \|\|\n elementFromXpath \|\|\n elementFromCache \|\|\n elementFromAiLocate;\n\n if (element) {\n const invalidElementReason = invalidLocateElementReason(element);\n if (invalidElementReason) {\n if (locateDump) {\n throw new ServiceError(invalidElementReason, locateDump);\n }\n throw new Error(invalidElementReason);\n }\n }\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature \| undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read)\n // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n !isCacheHit &&\n (!isPlanDirectHit \|\| !locateCacheAlreadyExists) &&\n locateParam.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForPoint) {\n try {\n // Transform coordinates to logical space for cacheFeatureForPoint\n // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n const pointForCache: [number, number] = [\n Math.round(element.center[0] / shrunkShotToLogicalRatio),\n Math.round(element.center[1] / shrunkShotToLogicalRatio),\n ];\n\n const defaultModel = this.getDefaultModelRuntime();\n const feature = await this.interface.cacheFeatureForPoint(\n pointForCache,\n {\n targetDescription:\n typeof locateParam.prompt === 'string'\n ? locateParam.prompt\n : locateParam.prompt?.prompt,\n modelRuntime: {\n ...defaultModel,\n executionId: locateModel.executionId,\n },\n },\n );\n if (hasNonEmptyCache(feature)) {\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n currentCacheEntry = feature;\n this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForPoint failed: %s', error);\n }\n } else {\n debug('cacheFeatureForPoint is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${locateParam.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${locateParam.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy \| undefined;\n\n if (isPlanDirectHit && paramWithLocatedPixelResult) {\n hitBy = {\n from: 'Plan',\n context: {\n locatedPixelResult:\n paramWithLocatedPixelResult.locatedPixelResult,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: locateParam.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n const promptDisplay = locateParam.promptDisplay;\n const elementForAction = promptDisplay\n ? {\n ...element,\n description: promptDisplay,\n }\n : element;\n\n if (promptDisplay && locateDump?.matchedElement) {\n locateDump.matchedElement = locateDump.matchedElement.map(\n (matchedElement) => ({\n ...matchedElement,\n description: promptDisplay,\n }),\n );\n }\n\n onResult?.(elementForAction);\n\n return {\n output: {\n element: {\n ...elementForAction,\n // backward compatibility for aiLocate, which return value needs a dpr field\n dpr: uiContext.deprecatedDpr,\n },\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","locateModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","validateRequiredLocateFields","field","ifLocateParamHasLocatedPixelResult","locateTask","result","task","taskContext","abortSignal","timing","uiContext","assert","shrunkShotToLogicalRatio","Error","parsedParam","parseActionParam","error","useDefaultWait","setTimingFieldOnce","delayBeforeRunner","preparation","Promise","defineActionSleep","rejected","item","originalError","String","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","userPromptToMultimodalPrompt","userPromptToString","taskLocator","paramWithLocatedPixelResult","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","transformLogicalElementToScreenshot","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","defaultModel","feature","hitBy","promptDisplay","elementForAction","matchedElement","getDefaultModelRuntime","interfaceInstance","service","taskCache","waitAfterAction","actionReadiness","operationResource"],"mappings":";;;;;;;;;;;;;;;;;;;AAwCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QAACD,QAAQ,MAAM,EAAE,CAAC,EAAE;QAAEA,QAAQ,MAAM,EAAE,CAAC,EAAE;KAAC;IACzD,IACEA,QAAQ,MAAM,EAAE,WAAW,KAC3BC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;AAG1E;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AA8BO,MAAMC;IAkCX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,WAAyB,EACzBC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,kBAAkBF,SAAS;YAC3B,YAAYA,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxBM,QAAQ,WAAW,EAAE;YACrB,MAAMK,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnEI,6BAA6B7B,OAAOyB,OAAO,WAAW;QAEtDE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,OAAO,CAAC8B,MAAM,EAAE;gBAGlB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,sBAAsB,EAAEwB,mCAAmC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OACE5C,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;QAEnE;QAEA,MAAMW,OAKF;YACF,MAAM;YACN,SAASX;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,2BAA2BO,OAAO,yBAAyB;YAC3D,UAAU,OAAOU;gBACf,MAAMC,cAAcrB,QAAQ,WAAW,IAAIoB,YAAY,MAAM;gBAC7D,MAAME,SAASF,YAAY,IAAI,CAAC,MAAM;gBAEtC/C,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEmC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAMG,YAAYH,YAAY,SAAS;gBACvCI,OAAOD,WAAW;gBAElBT,6BAA6B7B,OAAOyB,OAAO,WAAW;gBAEtD,MAAM,EAAEe,wBAAwB,EAAE,GAAGF;gBACrC,IAAIE,AAA6BpC,WAA7BoC,0BACF,MAAM,IAAIC,MACR;gBAGJ,MAAMC,cAAe,AAAC;oBACpB,IAAI,CAACjB,OAAO,WAAW,EAAE,OAAOzB;oBAChC,IAAI;wBACF,OAAO2C,iBAAiB3C,OAAOyB,OAAO,WAAW,EAAE;4BACjDe;wBACF;oBACF,EAAE,OAAOI,OAAY;wBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAEmB,MAAM,OAAO,CAAC,cAAc,EAAE9C,KAAK,SAAS,CAACE,QAAQ,EACtG;4BAAE,OAAO4C;wBAAM;oBAEnB;gBACF;gBAEA,MAAMC,iBAAiB,CAAC,IAAI,CAAC,eAAe;gBAC5CV,YAAY,IAAI,CAAC,eAAe,GAAGU,iBAC/B,YACA;gBACJT,aAAa;gBAEbU,mBAAmBT,QAAQ;gBAC3B,MAAMU,oBAAoBF,iBACrBpB,OAAO,iBAAiB,IAAI,MAC7B;gBACJ,IAAI;oBACF,MAAMuB,cAAc,MAAMC,QAAQ,UAAU,CAAC;wBAC3C,IAAI,CAAC,SAAS,CAAC,kBAAkB,GAC/BxB,OAAO,IAAI,EACXzB,OACAoC;wBAEFW,oBAAoB,IAChBG,kBAAkBd,aAAa,IAAI,CAAC;4BAClC,QAAQW;wBACV,KACAE,QAAQ,OAAO;qBACpB;oBACD,MAAME,WAAWH,YAAY,IAAI,CAC/B,CAACI,OAASA,AAAgB,eAAhBA,KAAK,MAAM;oBAEvB,IAAID,UAAU,WAAW,YAAY,MAAMA,SAAS,MAAM;gBAC5D,EAAE,OAAOE,eAAoB;oBAC3B,MAAM,IAAIZ,MACR,CAAC,wCAAwC,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAE4B,eAAe,WAAWC,OAAOD,gBAAgB,EAC5G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRP,mBAAmBT,QAAQ;gBAC7B;gBAEAD,aAAa;gBACbU,mBAAmBT,QAAQ;gBAC3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAM8B,eAAe,MAAM9B,OAAO,IAAI,CAAC,IAAI,CACzC,IAAI,CAAC,SAAS,EACdiB,aACAG,iBACI;oBAAE,GAAGV,WAAW;oBAAE,QAAQC;gBAAY,IACtC;oBAAE,GAAGD,WAAW;oBAAE,QAAQC;oBAAa,iBAAiB;gBAAK;gBAEnEU,mBAAmBT,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW8B;gBAC/CnB,aAAa;gBAEbU,mBAAmBT,QAAQ;gBAC3BS,mBAAmBT,QAAQ;gBAC3B,IAAI;oBACF,IAAIQ,gBAAgB;wBAClB,MAAMW,mBACJ/B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;wBACrD,IAAI+B,mBAAmB,GACrB,MAAMN,kBAAkBd,aAAa,IAAI,CAAC;4BACxC,QAAQoB;wBACV;wBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GACpC/B,OAAO,IAAI,EACXiB,aACAN;oBAEJ,OACE,MAAM,IAAI,CAAC,eAAe,CAAE,IAAI,CAC9B;wBACE,IAAID,YAAY,IAAI,CAAC,MAAM;wBAC3B,MAAMV,OAAO,IAAI;wBACjB,OAAOiB;oBACT,GACAN,aACA,IAAI,CAAC,iBAAiB;gBAG5B,SAAU;oBACRU,mBAAmBT,QAAQ;gBAC7B;gBACAD,aAAa;gBACb,IAAI;oBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GACpCX,OAAO,IAAI,EACXiB,aACAN;gBAEJ,EAAE,OAAOiB,eAAoB;oBAC3B,MAAM,IAAIZ,MACR,CAAC,uCAAuC,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAE4B,eAAe,WAAWC,OAAOD,gBAAgB,EAC3G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRP,mBAAmBT,QAAQ;gBAC7B;gBACA,OAAO;oBAAE,QAAQkB;gBAAa;YAChC;QACF;QAEAxC,QAAQ,KAAK,CAAC,IAAI,CAACmB;IACrB;IAEQ,iBACNhB,IAAyC,EACzCuC,mBAAiD,EACjD1C,OAAyB,EACzB2C,QAAgD,EACd;QAClC,MAAM,EAAE5C,SAAS,EAAEH,WAAW,EAAER,UAAU,EAAE,GAAGY;QAE/C,IAAI4C,cAAc5D,qBAAqB0D;QAIvC,IACE1C,QAAQ,gBAAgB,EAAE,QAAQ,UAClC4C,YAAY,MAAM,IAClB,CAACC,6BAA6BD,YAAY,MAAM,GAAG,QAAQ,QAE3DA,cAAc;YACZ,GAAGA,WAAW;YACd,QAAQ;gBACN,GAAG5C,QAAQ,gBAAgB;gBAC3B,QAAQ8C,mBAAmBF,YAAY,MAAM;YAC/C;QACF;QAGF,IAAI7C,AAAcV,WAAdU,WACF6C,cAAc;YACZ,GAAGA,WAAW;YACd7C;QACF;QAGF,IAAIX,cAAc,CAACwD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMG,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOH;YACP,SAASzC,KAAK,OAAO;YACrB,UAAU,OAAOiB;gBACf,MAAMC,cAAcrB,QAAQ,WAAW,IAAIoB,YAAY,MAAM;gBAC7DC,aAAa;gBACb,MAAM,EAAEF,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEG,SAAS,EAAE,GAAGH;gBACpB,MAAM4B,8BAA8BhC,mCAClC4B,eAEEA,cACAvD;gBAEJmC,OACEoB,aAAa,UAAUI,6BACvB,CAAC,mEAAmE,EAAEjE,KAAK,SAAS,CAClF6D,cACC;gBAGL,IAAI,CAACrB,WAAW;oBACdA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAACF;oBAClDA,aAAa;gBACf;gBAEAG,OAAOD,WAAW;gBAElB,MAAM,EAAEE,wBAAwB,EAAE,GAAGF;gBAErC,IAAIE,AAA6BpC,WAA7BoC,0BACF,MAAM,IAAIC,MACR;gBAIJ,IAAIuB;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACbjC,KAAK,GAAG,GAAG;wBACTiC;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,oBAAoBA,KAAK,QAAQ,EAAE;wBACnC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACAjC,KAAK,KAAK,GAAGkC,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnDjC,KAAK,UAAU,GAAGiC,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjBjC,KAAK,eAAe,GAAGkC,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjBjC,KAAK,iBAAiB,GAAGiC,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,8BACvBO,qBAAqBP,+BACrB3D;gBAOJ,MAAMmE,kBAAkBZ,YAAY,UAAU,GAC1CvD,SACAiE;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDb,YAAY,KAAK,IACjB,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFc,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACd,YAAY,KAAK;yBAAC;wBAC3B,iBAAiB;oBACnB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMe,mBAAmBD,gBACrBE,oCACE;oBACE,QAAQ;wBACNF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;wBAC3CA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;qBAC5C;oBACD,aACE,AAA8B,YAA9B,OAAOd,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE,UAAU;oBACpC,MAAMc;gBACR,GACAjC,4BAEFpC;gBAEJ,MAAMwE,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAclB,YAAY,MAAM;gBACtC,MAAMmB,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJR,mBAAmBI,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAlB,YAAY,SAAS;gBAI7B,MAAMuB,mBAAmBF,yBACrBL,oCACEK,wBACAxC,4BAEFpC;gBAEJ,MAAM+E,aAAa,CAAC,CAACD;gBAErB,IAAIE;gBACJ,MAAM/C,SAASF,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACyC,cAAc,CAACO,cAAc,CAACX,iBACjC,IAAI;oBACF1B,mBAAmBT,QAAQ;oBAC3B4B,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCN,aACA;wBACE,SAASrB;wBACT+B;oBACF,GACA1D,aACAyB;oBAEF8B,UAAUD,aAAa,IAAI;oBAC3BmB,sBAAsBnB,aAAa,OAAO;gBAC5C,EAAE,OAAOrB,OAAO;oBACd,IAAIA,iBAAiByC,cACnBnB,UAAUtB,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRE,mBAAmBT,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJ6E,mBACAG,oBACAQ,oBACAE;gBAEF,IAAI1F,SAAS;oBACX,MAAM4F,uBAAuB7F,2BAA2BC;oBACxD,IAAI4F,sBAAsB;wBACxB,IAAItB,YACF,MAAM,IAAIqB,aAAaC,sBAAsBtB;wBAE/C,MAAM,IAAIvB,MAAM6C;oBAClB;gBACF;gBAGA,MAAMC,2BAA2BjG,iBAC/BwF,mBAAmB,cAAc;gBAGnC,IAAIU;gBAOJ,IACE9F,WACA,IAAI,CAAC,SAAS,IACd,CAACyF,cACA,EAACX,mBAAmB,CAACe,wBAAuB,KAC7C5B,AAA0B,UAA1BA,YAAY,SAAS,EAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,MAAM8B,gBAAkC;wBACtCC,KAAK,KAAK,CAAChG,QAAQ,MAAM,CAAC,EAAE,GAAG8C;wBAC/BkD,KAAK,KAAK,CAAChG,QAAQ,MAAM,CAAC,EAAE,GAAG8C;qBAChC;oBAED,MAAMmD,eAAe,IAAI,CAAC,sBAAsB;oBAChD,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDH,eACA;wBACE,mBACE,AAA8B,YAA9B,OAAO9B,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE;wBAC1B,cAAc;4BACZ,GAAGgC,YAAY;4BACf,aAAahF,YAAY,WAAW;wBACtC;oBACF;oBAEF,IAAIrB,iBAAiBsG,UAAU;wBAC7BxG,MACE,uCACAyF,aACAe;wBAEFJ,oBAAoBI;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACE1F,MACE,yDACAyF;gBAGN,EAAE,OAAOjC,OAAO;oBACdxD,MAAM,mCAAmCwD;gBAC3C;qBAEAxD,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIsE,YACF,MAAM,IAAIqB,aACR,CAAC,oBAAoB,EAAE1B,YAAY,MAAM,EAAE,EAC3CK;oBAGJ,MAAM,IAAIvB,MAAM,CAAC,mBAAmB,EAAEkB,YAAY,MAAM,EAAE;gBAC5D;gBAEA,IAAIkC;gBAEJ,IAAIrB,mBAAmBT,6BACrB8B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,oBACE9B,4BAA4B,kBAAkB;oBAClD;gBACF;qBACK,IAAIa,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOlC,YAAY,KAAK;oBAC1B;gBACF;qBACK,IAAIwB,YACTU,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaS;oBACf;gBACF;gBAGF,MAAMM,gBAAgBnC,YAAY,aAAa;gBAC/C,MAAMoC,mBAAmBD,gBACrB;oBACE,GAAGpG,OAAO;oBACV,aAAaoG;gBACf,IACApG;gBAEJ,IAAIoG,iBAAiB9B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAACgC,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJpC,WAAWqC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKzD,UAAU,aAAa;wBAC9B;oBACF;oBACAuD;gBACF;YACF;QACF;QAEA,OAAO/B;IACT;IApoBA,YAAY,EACVmC,sBAAsB,EACtBC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACT5E,WAAW,EACX6E,eAAe,EACfC,eAAe,EACfC,iBAAiB,EACD,CAAE;QAtBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QACA,uBAAiB,0BAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QACA,uBAAiB,mBAAjB;QACA,uBAAiB,qBAAjB;QAYE,IAAI,CAAC,sBAAsB,GAAGN;QAC9B,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAG5E;QACnB,IAAI,CAAC,eAAe,GAAG6E;QACvB,IAAI,CAAC,eAAe,GAAGC;QACvB,IAAI,CAAC,iBAAiB,GAAGC,qBAAqBL;IAChD;AAmnBF"}

package/dist/es/agent/utils.mjs CHANGED Viewed

@@ -153,7 +153,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.17.2";
+const getMidsceneVersion = ()=>"1.17.3-beta-20261029072240.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/model-adapter/resolve.mjs CHANGED Viewed

@@ -29,7 +29,7 @@ function resolveImagePreprocess(imagePreprocess) {
 class ResolvedModelAdapter {
     constructor(config, modelFamily){
         _define_property(this, "modelType", void 0);
-        _define_property(this, "supportedTaskTypes", void 0);
+        _define_property(this, "allowedTaskTypes", void 0);
         _define_property(this, "supportedProtocols", void 0);
         _define_property(this, "jsonParser", void 0);
         _define_property(this, "chatCompletion", void 0);
@@ -41,7 +41,7 @@ class ResolvedModelAdapter {
         _define_property(this, "planning", void 0);
         _define_property(this, "locate", void 0);
         this.modelType = config.modelType ?? 'generative';
-        this.supportedTaskTypes = config.supportedTaskTypes;
+        this.allowedTaskTypes = config.allowedTaskTypes;
         this.supportedProtocols = config.supportedProtocols ?? [
             'openai-chat'
         ];

package/dist/es/ai-model/model-adapter/resolve.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { buildDefaultCodexAppServerParams } from './codex-app-server';\nimport { resolveInsight } from './insight';\nimport type { InsightAdapter } from './insight-protocol';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport { resolveResponses } from './responses';\nimport type {\n BuildCodexAppServerParams,\n ChatCompletionAdapter,\n ImagePreprocessPolicy,\n JsonParser,\n LocateAdapter,\n ModelAdapter,\n ModelAdapterDefinition,\n PlanningAdapter,\n ResolveImageDetail,\n ResponsesAdapter,\n} from './types';\n\nconst defaultImageDetail: ResolveImageDetail = ({ imageDetail }) =>\n imageDetail ?? 'high';\n\nfunction resolveJsonParser(\n jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n if (!jsonParser \|\| jsonParser === 'lenient-json') {\n return parseModelResponseJson;\n }\n\n if (typeof jsonParser === 'function') {\n return jsonParser;\n }\n\n throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n return {\n padBlockSize: imagePreprocess?.padBlockSize,\n };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n readonly modelType: ModelAdapter['modelType'];\n readonly ~~supportedTaskTypes~~: ModelAdapter['~~supportedTaskTypes~~'];\n readonly supportedProtocols: ModelAdapter['supportedProtocols'];\n readonly jsonParser: JsonParser;\n readonly chatCompletion: ChatCompletionAdapter;\n readonly resolveImageDetail: ResolveImageDetail;\n readonly responses: ResponsesAdapter;\n readonly buildCodexAppServerParams: BuildCodexAppServerParams;\n readonly imagePreprocess: ImagePreprocessPolicy;\n readonly insight: InsightAdapter;\n readonly planning: PlanningAdapter;\n readonly locate: LocateAdapter;\n\n constructor(config: ModelAdapterDefinition, modelFamily: string) {\n this.modelType = config.modelType ?? 'generative';\n this.~~supportedTaskTypes~~ = config.~~supportedTaskTypes~~;\n this.supportedProtocols = config.supportedProtocols ?? ['openai-chat'];\n this.jsonParser = resolveJsonParser(config.jsonParser);\n this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n this.resolveImageDetail = config.resolveImageDetail ?? defaultImageDetail;\n this.responses = resolveResponses(config.responses);\n this.buildCodexAppServerParams =\n config.buildCodexAppServerParams ?? buildDefaultCodexAppServerParams;\n this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n this.insight = resolveInsight(config.insight, {\n jsonParser: this.jsonParser,\n });\n const customPlanner =\n config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n const resolvedCustomPlanner = customPlanner\n ? resolveCustomPlanningDefinition(customPlanner)\n : undefined;\n this.locate = resolveLocate(config.locate, resolvedCustomPlanner, {\n jsonParser: this.jsonParser,\n });\n this.planning = resolvePlanning(\n config.planning,\n resolvedCustomPlanner,\n { jsonParser: this.jsonParser },\n this.locate.kind === 'standard'\n ? this.locate.element.resultCodec\n : undefined,\n );\n }\n}\n"],"names":["defaultImageDetail","imageDetail","resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","resolveResponses","buildDefaultCodexAppServerParams","resolveInsight","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolveLocate","resolvePlanning"],"mappings":";;;;;;;;;;;;;;;;;AAqBA,MAAMA,qBAAyC,CAAC,EAAEC,WAAW,EAAE,GAC7DA,eAAe;AAEjB,SAASC,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAcX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QAbjE,uBAAS,aAAT;QACA,uBAAS,~~sBAAT~~;QACA,uBAAS,sBAAT;QACA,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,aAAT;QACA,uBAAS,6BAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,WAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,SAAS,GAAGD,OAAO,SAAS,IAAI;QACrC,IAAI,CAAC,~~kBAAkB~~,GAAGA,OAAO,~~kBAAkB~~;~~QACnD~~,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAI;YAAC;SAAc;QACtE,IAAI,CAAC,UAAU,GAAGP,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAIT;QACvD,IAAI,CAAC,SAAS,GAAGY,iBAAiBH,OAAO,SAAS;QAClD,IAAI,CAAC,yBAAyB,GAC5BA,OAAO,yBAAyB,IAAII;QACtC,IAAI,CAAC,eAAe,GAAGP,uBAAuBG,OAAO,eAAe;QACpE,IAAI,CAAC,OAAO,GAAGK,eAAeL,OAAO,OAAO,EAAE;YAC5C,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,MAAMM,gBACJN,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGO;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,MAAM,GAAGG,cAAcV,OAAO,MAAM,EAAEQ,uBAAuB;YAChE,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,IAAI,CAAC,QAAQ,GAAGG,gBACdX,OAAO,QAAQ,EACfQ,uBACA;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC,GAC9B,AAAqB,eAArB,IAAI,CAAC,MAAM,CAAC,IAAI,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,GAC/BD;IAER;AACF"}
1	+ {"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { buildDefaultCodexAppServerParams } from './codex-app-server';\nimport { resolveInsight } from './insight';\nimport type { InsightAdapter } from './insight-protocol';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport { resolveResponses } from './responses';\nimport type {\n BuildCodexAppServerParams,\n ChatCompletionAdapter,\n ImagePreprocessPolicy,\n JsonParser,\n LocateAdapter,\n ModelAdapter,\n ModelAdapterDefinition,\n PlanningAdapter,\n ResolveImageDetail,\n ResponsesAdapter,\n} from './types';\n\nconst defaultImageDetail: ResolveImageDetail = ({ imageDetail }) =>\n imageDetail ?? 'high';\n\nfunction resolveJsonParser(\n jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n if (!jsonParser \|\| jsonParser === 'lenient-json') {\n return parseModelResponseJson;\n }\n\n if (typeof jsonParser === 'function') {\n return jsonParser;\n }\n\n throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n return {\n padBlockSize: imagePreprocess?.padBlockSize,\n };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n readonly modelType: ModelAdapter['modelType'];\n readonly allowedTaskTypes: ModelAdapter['allowedTaskTypes'];\n readonly supportedProtocols: ModelAdapter['supportedProtocols'];\n readonly jsonParser: JsonParser;\n readonly chatCompletion: ChatCompletionAdapter;\n readonly resolveImageDetail: ResolveImageDetail;\n readonly responses: ResponsesAdapter;\n readonly buildCodexAppServerParams: BuildCodexAppServerParams;\n readonly imagePreprocess: ImagePreprocessPolicy;\n readonly insight: InsightAdapter;\n readonly planning: PlanningAdapter;\n readonly locate: LocateAdapter;\n\n constructor(config: ModelAdapterDefinition, modelFamily: string) {\n this.modelType = config.modelType ?? 'generative';\n this.allowedTaskTypes = config.allowedTaskTypes;\n this.supportedProtocols = config.supportedProtocols ?? ['openai-chat'];\n this.jsonParser = resolveJsonParser(config.jsonParser);\n this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n this.resolveImageDetail = config.resolveImageDetail ?? defaultImageDetail;\n this.responses = resolveResponses(config.responses);\n this.buildCodexAppServerParams =\n config.buildCodexAppServerParams ?? buildDefaultCodexAppServerParams;\n this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n this.insight = resolveInsight(config.insight, {\n jsonParser: this.jsonParser,\n });\n const customPlanner =\n config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n const resolvedCustomPlanner = customPlanner\n ? resolveCustomPlanningDefinition(customPlanner)\n : undefined;\n this.locate = resolveLocate(config.locate, resolvedCustomPlanner, {\n jsonParser: this.jsonParser,\n });\n this.planning = resolvePlanning(\n config.planning,\n resolvedCustomPlanner,\n { jsonParser: this.jsonParser },\n this.locate.kind === 'standard'\n ? this.locate.element.resultCodec\n : undefined,\n );\n }\n}\n"],"names":["defaultImageDetail","imageDetail","resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","resolveResponses","buildDefaultCodexAppServerParams","resolveInsight","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolveLocate","resolvePlanning"],"mappings":";;;;;;;;;;;;;;;;;AAqBA,MAAMA,qBAAyC,CAAC,EAAEC,WAAW,EAAE,GAC7DA,eAAe;AAEjB,SAASC,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAcX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QAbjE,uBAAS,aAAT;QACA,uBAAS,oBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,aAAT;QACA,uBAAS,6BAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,WAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,SAAS,GAAGD,OAAO,SAAS,IAAI;QACrC,IAAI,CAAC,gBAAgB,GAAGA,OAAO,gBAAgB;QAC/C,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAI;YAAC;SAAc;QACtE,IAAI,CAAC,UAAU,GAAGP,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAIT;QACvD,IAAI,CAAC,SAAS,GAAGY,iBAAiBH,OAAO,SAAS;QAClD,IAAI,CAAC,yBAAyB,GAC5BA,OAAO,yBAAyB,IAAII;QACtC,IAAI,CAAC,eAAe,GAAGP,uBAAuBG,OAAO,eAAe;QACpE,IAAI,CAAC,OAAO,GAAGK,eAAeL,OAAO,OAAO,EAAE;YAC5C,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,MAAMM,gBACJN,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGO;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,MAAM,GAAGG,cAAcV,OAAO,MAAM,EAAEQ,uBAAuB;YAChE,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,IAAI,CAAC,QAAQ,GAAGG,gBACdX,OAAO,QAAQ,EACfQ,uBACA;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC,GAC9B,AAAqB,eAArB,IAAI,CAAC,MAAM,CAAC,IAAI,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,GAC/BD;IAER;AACF"}

package/dist/es/ai-model/models/ant-ling/adapter.mjs ADDED Viewed

@@ -0,0 +1,51 @@
+const buildAntLingChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    const { reasoningEnabled } = userConfig;
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...void 0 !== userConfig.temperature ? {
+                temperature: userConfig.temperature
+            } : {},
+            ...'none' !== userConfig.responseFormat && input.expectedJsonObjectResponse ? {
+                response_format: {
+                    type: 'json_object'
+                }
+            } : {},
+            ...'default' !== reasoningEnabled ? {
+                thinking: {
+                    type: reasoningEnabled ?? false ? 'enabled' : 'disabled'
+                }
+            } : {}
+        }
+    };
+};
+const antLingAdapters = {
+    'ant-ling': {
+        supportedProtocols: [
+            'openai-chat'
+        ],
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEffort',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: buildAntLingChatCompletionParams
+        },
+        locate: {
+            element: {
+                resultFormat: {
+                    resultKey: 'point',
+                    coordinates: {
+                        shape: 'point',
+                        order: 'xy',
+                        normalizedBy: 1000
+                    }
+                }
+            }
+        }
+    }
+};
+export { antLingAdapters };
+//# sourceMappingURL=adapter.mjs.map

package/dist/es/ai-model/models/ant-ling/adapter.mjs.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"ai-model/models/ant-ling/adapter.mjs","sources":["../../../../../src/ai-model/models/ant-ling/adapter.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n ChatCompletionCallContext,\n ChatCompletionParamsResult,\n ModelAdapterDefinition,\n} from '../../model-adapter/types';\n\nconst buildAntLingChatCompletionParams = (\n input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n const { midsceneDefaults, userConfig } = input;\n const { reasoningEnabled } = userConfig;\n\n return {\n config: {\n ...midsceneDefaults,\n ...(userConfig.temperature !== undefined\n ? { temperature: userConfig.temperature }\n : {}),\n // https://developer.ant-ling.com/en/docs/tutorials/structured-outputs/\n ...(userConfig.responseFormat !== 'none' &&\n input.expectedJsonObjectResponse\n ? { response_format: { type: 'json_object' } }\n : {}),\n ...(reasoningEnabled !== 'default'\n ? {\n thinking: {\n type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n },\n }\n : {}),\n },\n };\n};\n\nexport const antLingAdapters = {\n 'ant-ling': {\n supportedProtocols: ['openai-chat'],\n chatCompletion: {\n unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n buildChatCompletionParams: buildAntLingChatCompletionParams,\n },\n locate: {\n element: {\n resultFormat: {\n resultKey: 'point',\n coordinates: {\n shape: 'point',\n order: 'xy',\n normalizedBy: 1000,\n },\n },\n },\n },\n },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'ant-ling'>;\n"],"names":["buildAntLingChatCompletionParams","input","midsceneDefaults","userConfig","reasoningEnabled","undefined","antLingAdapters"],"mappings":"AAOA,MAAMA,mCAAmC,CACvCC;IAEA,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGF;IACzC,MAAM,EAAEG,gBAAgB,EAAE,GAAGD;IAE7B,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAIC,AAA2BE,WAA3BF,WAAW,WAAW,GACtB;gBAAE,aAAaA,WAAW,WAAW;YAAC,IACtC,CAAC,CAAC;YAEN,GAAIA,AAA8B,WAA9BA,WAAW,cAAc,IAC7BF,MAAM,0BAA0B,GAC5B;gBAAE,iBAAiB;oBAAE,MAAM;gBAAc;YAAE,IAC3C,CAAC,CAAC;YACN,GAAIG,AAAqB,cAArBA,mBACA;gBACE,UAAU;oBACR,MAAOA,oBAAoB,QAAS,YAAY;gBAClD;YACF,IACA,CAAC,CAAC;QACR;IACF;AACF;AAEO,MAAME,kBAAkB;IAC7B,YAAY;QACV,oBAAoB;YAAC;SAAc;QACnC,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BN;QAC7B;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,WAAW;oBACX,aAAa;wBACX,OAAO;wBACP,OAAO;wBACP,cAAc;oBAChB;gBACF;YACF;QACF;IACF;AACF"}

package/dist/es/ai-model/models/jev/adapter.mjs CHANGED Viewed

@@ -4,7 +4,7 @@ import { jevPlan } from "./planning.mjs";
 const jevAdapters = {
     jev: {
         modelType: 'system-one',
-        supportedTaskTypes: [
+        allowedTaskTypes: [
             'locate',
             'boolean',
             'planning'

package/dist/es/ai-model/models/jev/adapter.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/models/jev/adapter.mjs","sources":["../../../../../src/ai-model/models/jev/adapter.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport { jevBoolean } from './boolean';\nimport { jevLocate } from './locate';\nimport { jevPlan } from './planning';\n\nexport const jevAdapters = {\n jev: {\n modelType: 'system-one',\n ~~supportedTaskTypes~~: ['locate', 'boolean', 'planning'],\n supportedProtocols: [],\n locate: { kind: 'custom', locateFn: jevLocate },\n insight: { booleanFn: jevBoolean },\n planning: {\n kind: 'custom',\n planFn: jevPlan,\n cacheEnabled: false,\n supportsActionDeepLocate: false,\n defaultReplanningCycleLimit: 20,\n },\n },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'jev'>;\n"],"names":["jevAdapters","jevLocate","jevBoolean","jevPlan"],"mappings":";;;AAMO,MAAMA,cAAc;IACzB,KAAK;QACH,WAAW;QACX,~~oBAAoB~~;YAAC;YAAU;YAAW;SAAW;~~QACrD~~,oBAAoB,EAAE;QACtB,QAAQ;YAAE,MAAM;YAAU,UAAUC;QAAU;QAC9C,SAAS;YAAE,WAAWC;QAAW;QACjC,UAAU;YACR,MAAM;YACN,QAAQC;YACR,cAAc;YACd,0BAA0B;YAC1B,6BAA6B;QAC/B;IACF;AACF"}
1	+ {"version":3,"file":"ai-model/models/jev/adapter.mjs","sources":["../../../../../src/ai-model/models/jev/adapter.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport { jevBoolean } from './boolean';\nimport { jevLocate } from './locate';\nimport { jevPlan } from './planning';\n\nexport const jevAdapters = {\n jev: {\n modelType: 'system-one',\n allowedTaskTypes: ['locate', 'boolean', 'planning'],\n supportedProtocols: [],\n locate: { kind: 'custom', locateFn: jevLocate },\n insight: { booleanFn: jevBoolean },\n planning: {\n kind: 'custom',\n planFn: jevPlan,\n cacheEnabled: false,\n supportsActionDeepLocate: false,\n defaultReplanningCycleLimit: 20,\n },\n },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'jev'>;\n"],"names":["jevAdapters","jevLocate","jevBoolean","jevPlan"],"mappings":";;;AAMO,MAAMA,cAAc;IACzB,KAAK;QACH,WAAW;QACX,kBAAkB;YAAC;YAAU;YAAW;SAAW;QACnD,oBAAoB,EAAE;QACtB,QAAQ;YAAE,MAAM;YAAU,UAAUC;QAAU;QAC9C,SAAS;YAAE,WAAWC;QAAW;QACjC,UAAU;YACR,MAAM;YACN,QAAQC;YACR,cAAc;YACd,0BAA0B;YAC1B,6BAA6B;QAC/B;IACF;AACF"}

package/dist/es/ai-model/models/model-fallback.mjs CHANGED Viewed

@@ -2,13 +2,13 @@ import { getDebug } from "@midscene/shared/logger";
 import { getModelRuntime } from "./registry.mjs";
 function resolveModelForTaskType(configs, intent, taskType) {
     const requested = getModelRuntime(configs.getModelConfig(intent));
-    if (!requested.adapter.supportedTaskTypes || requested.adapter.supportedTaskTypes.includes(taskType)) return requested;
+    if (!requested.adapter.allowedTaskTypes || requested.adapter.allowedTaskTypes.includes(taskType)) return requested;
     const config = configs.getModelConfig('default');
     const runtime = getModelRuntime({
         ...config,
         intent
     });
-    if (runtime.adapter.supportedTaskTypes && !runtime.adapter.supportedTaskTypes.includes(taskType)) throw new Error(`The default model cannot perform ${taskType}. Configure a general-purpose default model.`);
+    if (runtime.adapter.allowedTaskTypes && !runtime.adapter.allowedTaskTypes.includes(taskType)) throw new Error(`The default model cannot perform ${taskType}. Configure a general-purpose default model.`);
     const reason = `${requested.config.modelFamily} does not support ${taskType}; using the default model`;
     getDebug('ai:model-fallback', {
         console: true

package/dist/es/ai-model/models/model-fallback.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/models/model-fallback.mjs","sources":["../../../../src/ai-model/models/model-fallback.ts"],"sourcesContent":["import type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ModelRuntime, ModelTaskType } from '../model-adapter/types';\nimport { getModelRuntime } from './registry';\n\nexport function resolveModelForTaskType(\n configs: { getModelConfig(intent: TIntent): IModelConfig },\n intent: TIntent,\n taskType: ModelTaskType,\n): ModelRuntime {\n const requested = getModelRuntime(configs.getModelConfig(intent));\n if (\n !requested.adapter.~~supportedTaskTypes~~ \|\|\n requested.adapter.~~supportedTaskTypes~~.includes(taskType)\n ) {\n return requested;\n }\n const config = configs.getModelConfig('default');\n const runtime = getModelRuntime({ ...config, intent });\n if (\n runtime.adapter.~~supportedTaskTypes~~ &&\n !runtime.adapter.~~supportedTaskTypes~~.includes(taskType)\n ) {\n throw new Error(\n `The default model cannot perform ${taskType}. Configure a general-purpose default model.`,\n );\n }\n const reason = `${requested.config.modelFamily} does not support ${taskType}; using the default model`;\n getDebug('ai:model-fallback', { console: true })(reason);\n return {\n ...runtime,\n modelFallback: {\n requestedModel: requested.config.modelName,\n requestedSlot: requested.config.slot,\n taskType,\n reason,\n },\n };\n}\n"],"names":["resolveModelForTaskType","configs","intent","taskType","requested","getModelRuntime","config","runtime","Error","reason","getDebug"],"mappings":";;AAKO,SAASA,wBACdC,OAA0D,EAC1DC,MAAe,EACfC,QAAuB;IAEvB,MAAMC,YAAYC,gBAAgBJ,QAAQ,cAAc,CAACC;IACzD,IACE,CAACE,UAAU,OAAO,CAAC,~~kBAAkB~~,~~IACrCA~~,UAAU,OAAO,CAAC,~~kBAAkB~~,CAAC,QAAQ,CAACD,~~WAE9C~~,OAAOC;IAET,MAAME,SAASL,QAAQ,cAAc,CAAC;IACtC,MAAMM,UAAUF,gBAAgB;QAAE,GAAGC,MAAM;QAAEJ;IAAO;IACpD,IACEK,QAAQ,OAAO,CAAC,~~kBAAkB~~,~~IAClC~~,CAACA,QAAQ,OAAO,CAAC,~~kBAAkB~~,CAAC,QAAQ,CAACJ,~~WAE7C~~,MAAM,IAAIK,MACR,CAAC,iCAAiC,EAAEL,SAAS,4CAA4C,CAAC;IAG9F,MAAMM,SAAS,GAAGL,UAAU,MAAM,CAAC,WAAW,CAAC,kBAAkB,EAAED,SAAS,yBAAyB,CAAC;IACtGO,SAAS,qBAAqB;QAAE,SAAS;IAAK,GAAGD;IACjD,OAAO;QACL,GAAGF,OAAO;QACV,eAAe;YACb,gBAAgBH,UAAU,MAAM,CAAC,SAAS;YAC1C,eAAeA,UAAU,MAAM,CAAC,IAAI;YACpCD;YACAM;QACF;IACF;AACF"}
1	+ {"version":3,"file":"ai-model/models/model-fallback.mjs","sources":["../../../../src/ai-model/models/model-fallback.ts"],"sourcesContent":["import type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ModelRuntime, ModelTaskType } from '../model-adapter/types';\nimport { getModelRuntime } from './registry';\n\nexport function resolveModelForTaskType(\n configs: { getModelConfig(intent: TIntent): IModelConfig },\n intent: TIntent,\n taskType: ModelTaskType,\n): ModelRuntime {\n const requested = getModelRuntime(configs.getModelConfig(intent));\n if (\n !requested.adapter.allowedTaskTypes \|\|\n requested.adapter.allowedTaskTypes.includes(taskType)\n ) {\n return requested;\n }\n const config = configs.getModelConfig('default');\n const runtime = getModelRuntime({ ...config, intent });\n if (\n runtime.adapter.allowedTaskTypes &&\n !runtime.adapter.allowedTaskTypes.includes(taskType)\n ) {\n throw new Error(\n `The default model cannot perform ${taskType}. Configure a general-purpose default model.`,\n );\n }\n const reason = `${requested.config.modelFamily} does not support ${taskType}; using the default model`;\n getDebug('ai:model-fallback', { console: true })(reason);\n return {\n ...runtime,\n modelFallback: {\n requestedModel: requested.config.modelName,\n requestedSlot: requested.config.slot,\n taskType,\n reason,\n },\n };\n}\n"],"names":["resolveModelForTaskType","configs","intent","taskType","requested","getModelRuntime","config","runtime","Error","reason","getDebug"],"mappings":";;AAKO,SAASA,wBACdC,OAA0D,EAC1DC,MAAe,EACfC,QAAuB;IAEvB,MAAMC,YAAYC,gBAAgBJ,QAAQ,cAAc,CAACC;IACzD,IACE,CAACE,UAAU,OAAO,CAAC,gBAAgB,IACnCA,UAAU,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAACD,WAE5C,OAAOC;IAET,MAAME,SAASL,QAAQ,cAAc,CAAC;IACtC,MAAMM,UAAUF,gBAAgB;QAAE,GAAGC,MAAM;QAAEJ;IAAO;IACpD,IACEK,QAAQ,OAAO,CAAC,gBAAgB,IAChC,CAACA,QAAQ,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAACJ,WAE3C,MAAM,IAAIK,MACR,CAAC,iCAAiC,EAAEL,SAAS,4CAA4C,CAAC;IAG9F,MAAMM,SAAS,GAAGL,UAAU,MAAM,CAAC,WAAW,CAAC,kBAAkB,EAAED,SAAS,yBAAyB,CAAC;IACtGO,SAAS,qBAAqB;QAAE,SAAS;IAAK,GAAGD;IACjD,OAAO;QACL,GAAGF,OAAO;QACV,eAAe;YACb,gBAAgBH,UAAU,MAAM,CAAC,SAAS;YAC1C,eAAeA,UAAU,MAAM,CAAC,IAAI;YACpCD;YACAM;QACF;IACF;AACF"}

package/dist/es/ai-model/models/registry.mjs CHANGED Viewed

@@ -1,5 +1,6 @@
 import { getDebug } from "@midscene/shared/logger";
 import { ResolvedModelAdapter } from "../model-adapter/resolve.mjs";
+import { antLingAdapters } from "./ant-ling/adapter.mjs";
 import { autoGlmAdapters } from "./auto-glm/adapter.mjs";
 import { deepSeekAdapters } from "./deepseek/adapter.mjs";
 import { defaultOpenAICompatibleAdapterConfig } from "./default.mjs";
@@ -13,6 +14,7 @@ import { mimoAdapters } from "./mimo.mjs";
 import { qwenAdapters } from "./qwen.mjs";
 import { uiTarsAdapters } from "./ui-tars/adapter.mjs";
 const MODEL_ADAPTER_CONFIGS = {
+    ...antLingAdapters,
     ...jevAdapters,
     ...qwenAdapters,
     ...deepSeekAdapters,

package/dist/es/ai-model/models/registry.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/models/registry.mjs","sources":["../../../../src/ai-model/models/registry.ts"],"sourcesContent":["import type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ResolvedModelAdapter } from '../model-adapter/resolve';\nimport type {\n ModelAdapter,\n ModelAdapterDefinition,\n ModelRuntime,\n} from '../model-adapter/types';\nimport { autoGlmAdapters } from './auto-glm/adapter';\nimport { deepSeekAdapters } from './deepseek/adapter';\nimport { defaultOpenAICompatibleAdapterConfig } from './default';\nimport { doubaoAdapters } from './doubao';\nimport { geminiAdapters } from './gemini';\nimport { glmAdapters } from './glm';\nimport { gptAdapters } from './gpt';\nimport { jevAdapters } from './jev/adapter';\nimport { kimiAdapters } from './kimi';\nimport { mimoAdapters } from './mimo';\nimport { qwenAdapters } from './qwen';\nimport { uiTarsAdapters } from './ui-tars/adapter';\n\nexport const MODEL_ADAPTER_CONFIGS = {\n ...jevAdapters,\n ...qwenAdapters,\n ...deepSeekAdapters,\n ...doubaoAdapters,\n ...geminiAdapters,\n ...uiTarsAdapters,\n ...glmAdapters,\n ...autoGlmAdapters,\n ...gptAdapters,\n ...kimiAdapters,\n ...mimoAdapters,\n} satisfies Record<TModelFamily, ModelAdapterDefinition>;\n\ntype ModelAdapterCacheKey = TModelFamily \| 'default';\n\nconst modelAdapterCache = new Map<ModelAdapterCacheKey, ModelAdapter>();\nconst debugModelAdapter = getDebug('ai:model-adapter');\n\nfunction debugAdapterUnsupportedUserConfig(\n modelFamily: ModelAdapterCacheKey,\n adapter: ModelAdapter,\n): void {\n if (adapter.chatCompletion.unsupportedUserConfig.length > 0) {\n debugModelAdapter(\n `model adapter \"${modelFamily}\" (chat-completion) unsupportedUserConfig: ${JSON.stringify(adapter.chatCompletion.unsupportedUserConfig)}`,\n );\n }\n if (adapter.responses.unsupportedUserConfig.length > 0) {\n debugModelAdapter(\n `model adapter \"${modelFamily}\" (responses) unsupportedUserConfig: ${JSON.stringify(adapter.responses.unsupportedUserConfig)}`,\n );\n }\n}\n\nexport function getModelAdapter(modelFamily?: TModelFamily): ModelAdapter {\n const cacheKey: ModelAdapterCacheKey = modelFamily ?? 'default';\n let adapter = modelAdapterCache.get(cacheKey);\n if (adapter) {\n return adapter;\n }\n\n const config = modelFamily\n ? MODEL_ADAPTER_CONFIGS[modelFamily]\n : defaultOpenAICompatibleAdapterConfig;\n if (!config) {\n throw new Error(\n `No model adapter registered for modelFamily: ${modelFamily}`,\n );\n }\n\n adapter = new ResolvedModelAdapter(config, cacheKey);\n modelAdapterCache.set(cacheKey, adapter);\n debugAdapterUnsupportedUserConfig(cacheKey, adapter);\n\n return adapter;\n}\n\nexport function getModelRuntime(config: IModelConfig): ModelRuntime {\n return {\n config,\n adapter: getModelAdapter(config.modelFamily),\n };\n}\n"],"names":["MODEL_ADAPTER_CONFIGS","jevAdapters","qwenAdapters","deepSeekAdapters","doubaoAdapters","geminiAdapters","uiTarsAdapters","glmAdapters","autoGlmAdapters","gptAdapters","kimiAdapters","mimoAdapters","modelAdapterCache","Map","debugModelAdapter","getDebug","debugAdapterUnsupportedUserConfig","modelFamily","adapter","JSON","getModelAdapter","cacheKey","config","defaultOpenAICompatibleAdapterConfig","Error","ResolvedModelAdapter","getModelRuntime"],"mappings":"~~;;;;;;;;;;;;;;AAqBO~~,MAAMA,wBAAwB;IACnC,GAAGC,WAAW;IACd,GAAGC,YAAY;IACf,GAAGC,gBAAgB;IACnB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,WAAW;IACd,GAAGC,eAAe;IAClB,GAAGC,WAAW;IACd,GAAGC,YAAY;IACf,GAAGC,YAAY;AACjB;AAIA,MAAMC,oBAAoB,IAAIC;AAC9B,MAAMC,oBAAoBC,SAAS;AAEnC,SAASC,kCACPC,WAAiC,EACjCC,OAAqB;IAErB,IAAIA,QAAQ,cAAc,CAAC,qBAAqB,CAAC,MAAM,GAAG,GACxDJ,kBACE,CAAC,eAAe,EAAEG,YAAY,2CAA2C,EAAEE,KAAK,SAAS,CAACD,QAAQ,cAAc,CAAC,qBAAqB,GAAG;IAG7I,IAAIA,QAAQ,SAAS,CAAC,qBAAqB,CAAC,MAAM,GAAG,GACnDJ,kBACE,CAAC,eAAe,EAAEG,YAAY,qCAAqC,EAAEE,KAAK,SAAS,CAACD,QAAQ,SAAS,CAAC,qBAAqB,GAAG;AAGpI;AAEO,SAASE,gBAAgBH,WAA0B;IACxD,MAAMI,WAAiCJ,eAAe;IACtD,IAAIC,UAAUN,kBAAkB,GAAG,CAACS;IACpC,IAAIH,SACF,OAAOA;IAGT,MAAMI,SAASL,~~cACXjB~~,qBAAqB,~~CAACiB~~,YAAY,GAClCM;IACJ,IAAI,CAACD,QACH,MAAM,IAAIE,MACR,CAAC,6CAA6C,EAAEP,aAAa;IAIjEC,UAAU,IAAIO,qBAAqBH,QAAQD;IAC3CT,kBAAkB,GAAG,CAACS,UAAUH;IAChCF,kCAAkCK,UAAUH;IAE5C,OAAOA;AACT;AAEO,SAASQ,gBAAgBJ,MAAoB;IAClD,OAAO;QACLA;QACA,SAASF,gBAAgBE,OAAO,WAAW;IAC7C;AACF"}
1	+ {"version":3,"file":"ai-model/models/registry.mjs","sources":["../../../../src/ai-model/models/registry.ts"],"sourcesContent":["import type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ResolvedModelAdapter } from '../model-adapter/resolve';\nimport type {\n ModelAdapter,\n ModelAdapterDefinition,\n ModelRuntime,\n} from '../model-adapter/types';\nimport { antLingAdapters } from './ant-ling/adapter';\nimport { autoGlmAdapters } from './auto-glm/adapter';\nimport { deepSeekAdapters } from './deepseek/adapter';\nimport { defaultOpenAICompatibleAdapterConfig } from './default';\nimport { doubaoAdapters } from './doubao';\nimport { geminiAdapters } from './gemini';\nimport { glmAdapters } from './glm';\nimport { gptAdapters } from './gpt';\nimport { jevAdapters } from './jev/adapter';\nimport { kimiAdapters } from './kimi';\nimport { mimoAdapters } from './mimo';\nimport { qwenAdapters } from './qwen';\nimport { uiTarsAdapters } from './ui-tars/adapter';\n\nexport const MODEL_ADAPTER_CONFIGS = {\n ...antLingAdapters,\n ...jevAdapters,\n ...qwenAdapters,\n ...deepSeekAdapters,\n ...doubaoAdapters,\n ...geminiAdapters,\n ...uiTarsAdapters,\n ...glmAdapters,\n ...autoGlmAdapters,\n ...gptAdapters,\n ...kimiAdapters,\n ...mimoAdapters,\n} satisfies Record<TModelFamily, ModelAdapterDefinition>;\n\ntype ModelAdapterCacheKey = TModelFamily \| 'default';\n\nconst modelAdapterCache = new Map<ModelAdapterCacheKey, ModelAdapter>();\nconst debugModelAdapter = getDebug('ai:model-adapter');\n\nfunction debugAdapterUnsupportedUserConfig(\n modelFamily: ModelAdapterCacheKey,\n adapter: ModelAdapter,\n): void {\n if (adapter.chatCompletion.unsupportedUserConfig.length > 0) {\n debugModelAdapter(\n `model adapter \"${modelFamily}\" (chat-completion) unsupportedUserConfig: ${JSON.stringify(adapter.chatCompletion.unsupportedUserConfig)}`,\n );\n }\n if (adapter.responses.unsupportedUserConfig.length > 0) {\n debugModelAdapter(\n `model adapter \"${modelFamily}\" (responses) unsupportedUserConfig: ${JSON.stringify(adapter.responses.unsupportedUserConfig)}`,\n );\n }\n}\n\nexport function getModelAdapter(modelFamily?: TModelFamily): ModelAdapter {\n const cacheKey: ModelAdapterCacheKey = modelFamily ?? 'default';\n let adapter = modelAdapterCache.get(cacheKey);\n if (adapter) {\n return adapter;\n }\n\n const config = modelFamily\n ? MODEL_ADAPTER_CONFIGS[modelFamily]\n : defaultOpenAICompatibleAdapterConfig;\n if (!config) {\n throw new Error(\n `No model adapter registered for modelFamily: ${modelFamily}`,\n );\n }\n\n adapter = new ResolvedModelAdapter(config, cacheKey);\n modelAdapterCache.set(cacheKey, adapter);\n debugAdapterUnsupportedUserConfig(cacheKey, adapter);\n\n return adapter;\n}\n\nexport function getModelRuntime(config: IModelConfig): ModelRuntime {\n return {\n config,\n adapter: getModelAdapter(config.modelFamily),\n };\n}\n"],"names":["MODEL_ADAPTER_CONFIGS","antLingAdapters","jevAdapters","qwenAdapters","deepSeekAdapters","doubaoAdapters","geminiAdapters","uiTarsAdapters","glmAdapters","autoGlmAdapters","gptAdapters","kimiAdapters","mimoAdapters","modelAdapterCache","Map","debugModelAdapter","getDebug","debugAdapterUnsupportedUserConfig","modelFamily","adapter","JSON","getModelAdapter","cacheKey","config","defaultOpenAICompatibleAdapterConfig","Error","ResolvedModelAdapter","getModelRuntime"],"mappings":";;;;;;;;;;;;;;;AAsBO,MAAMA,wBAAwB;IACnC,GAAGC,eAAe;IAClB,GAAGC,WAAW;IACd,GAAGC,YAAY;IACf,GAAGC,gBAAgB;IACnB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,WAAW;IACd,GAAGC,eAAe;IAClB,GAAGC,WAAW;IACd,GAAGC,YAAY;IACf,GAAGC,YAAY;AACjB;AAIA,MAAMC,oBAAoB,IAAIC;AAC9B,MAAMC,oBAAoBC,SAAS;AAEnC,SAASC,kCACPC,WAAiC,EACjCC,OAAqB;IAErB,IAAIA,QAAQ,cAAc,CAAC,qBAAqB,CAAC,MAAM,GAAG,GACxDJ,kBACE,CAAC,eAAe,EAAEG,YAAY,2CAA2C,EAAEE,KAAK,SAAS,CAACD,QAAQ,cAAc,CAAC,qBAAqB,GAAG;IAG7I,IAAIA,QAAQ,SAAS,CAAC,qBAAqB,CAAC,MAAM,GAAG,GACnDJ,kBACE,CAAC,eAAe,EAAEG,YAAY,qCAAqC,EAAEE,KAAK,SAAS,CAACD,QAAQ,SAAS,CAAC,qBAAqB,GAAG;AAGpI;AAEO,SAASE,gBAAgBH,WAA0B;IACxD,MAAMI,WAAiCJ,eAAe;IACtD,IAAIC,UAAUN,kBAAkB,GAAG,CAACS;IACpC,IAAIH,SACF,OAAOA;IAGT,MAAMI,SAASL,cACXlB,qBAAqB,CAACkB,YAAY,GAClCM;IACJ,IAAI,CAACD,QACH,MAAM,IAAIE,MACR,CAAC,6CAA6C,EAAEP,aAAa;IAIjEC,UAAU,IAAIO,qBAAqBH,QAAQD;IAC3CT,kBAAkB,GAAG,CAACS,UAAUH;IAChCF,kCAAkCK,UAAUH;IAE5C,OAAOA;AACT;AAEO,SAASQ,gBAAgBJ,MAAoB;IAClD,OAAO;QACLA;QACA,SAASF,gBAAgBE,OAAO,WAAW;IAC7C;AACF"}