npm - @aiscene/core - Versions diffs - 9.0.6 → 9.0.8 - Mend

@aiscene/core 9.0.6 → 9.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (686) hide show

package/dist/es/ai-model/conversation-history.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/conversation-history.mjs","sources":["../../../src/ai-model/conversation-history.ts"],"sourcesContent":["import type { AIMemory, SubGoal } from '@/types';\nimport { normalizeAIMemory } from './prompt/util';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nexport interface ConversationHistoryOptions {\n initialMessages?: ChatCompletionMessageParam[];\n initialMemories?: AIMemory;\n}\n\nexport class ConversationHistory {\n private readonly messages: ChatCompletionMessageParam[] = [];\n private subGoals: SubGoal[] = [];\n private memories: string[] = [];\n private historicalLogs: string[] = [];\n\n public pendingFeedbackMessage: string;\n\n constructor(options?: ConversationHistoryOptions) {\n if (options?.initialMessages?.length) {\n this.seed(options.initialMessages);\n }\n normalizeAIMemory(options?.initialMemories).forEach((memory) => {\n this.appendMemory(memory);\n });\n this.pendingFeedbackMessage = '';\n }\n\n resetPendingFeedbackMessageIfExists() {\n if (this.pendingFeedbackMessage) {\n this.pendingFeedbackMessage = '';\n }\n }\n\n append(message: ChatCompletionMessageParam) {\n this.messages.push(message);\n }\n\n seed(messages: ChatCompletionMessageParam[]) {\n this.reset();\n messages.forEach((message) => {\n this.append(message);\n });\n }\n\n reset() {\n this.messages.length = 0;\n this.memories.length = 0;\n this.subGoals.length = 0;\n this.historicalLogs.length = 0;\n this.pendingFeedbackMessage = '';\n }\n\n /*\n Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n * @returns The snapshot of the conversation history.\n /\n snapshot(maxImages?: number): ChatCompletionMessageParam[] {\n if (maxImages === undefined) {\n return [...this.messages];\n }\n\n const clonedMessages = structuredClone(this.messages);\n let imageCount = 0;\n\n // Traverse from the end to the beginning\n for (let i = clonedMessages.length - 1; i >= 0; i--) {\n const message = clonedMessages[i];\n const content = message.content;\n\n // Only process if content is an array\n if (Array.isArray(content)) {\n for (let j = 0; j < content.length; j++) {\n const item = content[j];\n\n // Check if this is an image\n if (item.type === 'image_url') {\n imageCount++;\n\n // If we've exceeded the limit, replace with text\n if (imageCount > maxImages) {\n content[j] = {\n type: 'text',\n text: '(image ignored due to size optimization)',\n };\n }\n }\n }\n }\n }\n\n return clonedMessages;\n }\n\n get length(): number {\n return this.messages.length;\n }\n\n [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n return this.messages[Symbol.iterator]();\n }\n\n toJSON(): ChatCompletionMessageParam[] {\n return this.snapshot();\n }\n\n // Sub-goal management methods\n\n /\n Set all sub-goals, replacing any existing ones.\n * Automatically marks the first pending goal as running.\n /\n setSubGoals(subGoals: SubGoal[]): void {\n this.subGoals = subGoals.map((goal) => ({ ...goal }));\n this.markFirstPendingAsRunning();\n }\n\n /\n Merge sub-goals from update-plan-content.\n * Preserves existing descriptions when incoming description is empty.\n \n This handles compact XML updates like:\n * <sub-goal index=\"1\" status=\"finished\" />\n /\n mergeSubGoals(subGoals: SubGoal[]): void {\n if (this.subGoals.length === 0) {\n this.setSubGoals(subGoals);\n return;\n }\n\n const existingByIndex = new Map(\n this.subGoals.map((goal) => [goal.index, goal] as const),\n );\n\n const mergedSubGoals = subGoals.map((goal) => {\n const existingGoal = existingByIndex.get(goal.index);\n const hasNonEmptyDescription = goal.description.trim().length > 0;\n\n if (!existingGoal && !hasNonEmptyDescription) {\n return null;\n }\n\n return {\n ...goal,\n description:\n hasNonEmptyDescription \|\| !existingGoal\n ? goal.description\n : existingGoal.description,\n };\n });\n\n const validSubGoals = mergedSubGoals.filter((goal) => goal !== null);\n if (validSubGoals.length === 0) {\n return;\n }\n\n this.setSubGoals(validSubGoals);\n }\n\n /\n Update a single sub-goal by index.\n * Clears logs if status or description actually changes.\n * @returns true if the sub-goal was found and updated, false otherwise\n /\n updateSubGoal(\n index: number,\n updates: Partial<Omit<SubGoal, 'index'>>,\n ): boolean {\n const goal = this.subGoals.find((g) => g.index === index);\n if (!goal) {\n return false;\n }\n\n let changed = false;\n\n if (updates.status !== undefined && updates.status !== goal.status) {\n goal.status = updates.status;\n changed = true;\n }\n if (\n updates.description !== undefined &&\n updates.description !== goal.description\n ) {\n goal.description = updates.description;\n changed = true;\n }\n\n if (changed) {\n goal.logs = [];\n }\n\n return true;\n }\n\n /\n Mark the first pending sub-goal as running.\n * Clears logs since status changes.\n /\n markFirstPendingAsRunning(): void {\n const firstPending = this.subGoals.find((g) => g.status === 'pending');\n if (firstPending) {\n firstPending.status = 'running';\n firstPending.logs = [];\n }\n }\n\n /\n Mark a sub-goal as finished.\n * Automatically marks the next pending goal as running.\n * @returns true if the sub-goal was found and updated, false otherwise\n /\n markSubGoalFinished(index: number): boolean {\n const result = this.updateSubGoal(index, { status: 'finished' });\n if (result) {\n this.markFirstPendingAsRunning();\n }\n return result;\n }\n\n /\n Mark all sub-goals as finished.\n * Clears logs for any goal whose status actually changes.\n /\n markAllSubGoalsFinished(): void {\n for (const goal of this.subGoals) {\n if (goal.status !== 'finished') {\n goal.logs = [];\n }\n goal.status = 'finished';\n }\n }\n\n /\n Append a log entry to the currently running sub-goal.\n * The log describes an action performed while working on the sub-goal.\n /\n appendSubGoalLog(log: string): void {\n if (!log) {\n return;\n }\n const runningGoal = this.subGoals.find((g) => g.status === 'running');\n if (runningGoal) {\n if (!runningGoal.logs) {\n runningGoal.logs = [];\n }\n runningGoal.logs.push(log);\n }\n }\n\n /\n Convert sub-goals to text representation.\n * Includes actions performed (logs) for the current sub-goal.\n /\n subGoalsToText(): string {\n if (this.subGoals.length === 0) {\n return '';\n }\n\n const lines = this.subGoals.map((goal) => {\n return `${goal.index}. ${goal.description} (${goal.status})`;\n });\n\n // Running goal takes priority, otherwise show first pending\n const currentGoal =\n this.subGoals.find((goal) => goal.status === 'running') \|\|\n this.subGoals.find((goal) => goal.status === 'pending');\n\n let currentGoalText = '';\n if (currentGoal) {\n currentGoalText = `\\nCurrent sub-goal is: ${currentGoal.description}`;\n if (currentGoal.logs && currentGoal.logs.length > 0) {\n const logLines = currentGoal.logs.map((log) => `- ${log}`).join('\\n');\n currentGoalText += `\\nActions performed for current sub-goal:\\n${logLines}`;\n }\n }\n\n return `Sub-goals:\\n${lines.join('\\n')}${currentGoalText}`;\n }\n\n // Historical log management methods (used in non-deepThink mode)\n\n /\n Append a log entry to the historical logs list.\n * Used in non-deepThink mode to track executed steps across planning rounds.\n /\n appendHistoricalLog(log: string): void {\n if (log) {\n this.historicalLogs.push(log);\n }\n }\n\n /\n Convert historical logs to text representation.\n * Provides context about previously executed steps to the model.\n /\n historicalLogsToText(): string {\n if (this.historicalLogs.length === 0) {\n return '';\n }\n\n const logLines = this.historicalLogs.map((log) => `- ${log}`).join('\\n');\n return `Here are the steps that have been executed:\\n${logLines}`;\n }\n\n // Memory management methods\n\n /\n Append a memory to the memories list\n /\n appendMemory(memory: string): void {\n if (memory) {\n this.memories.push(memory);\n }\n }\n\n /\n Get all memories\n /\n getMemories(): string[] {\n return [...this.memories];\n }\n\n /\n Convert memories to text representation\n /\n memoriesToText(): string {\n if (this.memories.length === 0) {\n return '';\n }\n\n return `Memories from previous steps:\\n---\\n${this.memories.join('\\n---\\n')}\\n`;\n }\n\n /\n Clear all memories\n /\n clearMemories(): void {\n this.memories.length = 0;\n }\n\n /\n Compress the conversation history if it exceeds the threshold.\n * Removes the oldest messages and replaces them with a single placeholder message.\n * @param threshold - The number of messages that triggers compression.\n * @param keepCount - The number of recent messages to keep after compression.\n * @returns true if compression was performed, false otherwise.\n */\n compressHistory(threshold: number, keepCount: number): boolean {\n if (this.messages.length <= threshold) {\n return false;\n }\n\n const omittedCount = this.messages.length - keepCount;\n const omittedPlaceholder: ChatCompletionMessageParam = {\n role: 'user',\n content: `(${omittedCount} previous conversation messages have been omitted)`,\n };\n\n // Keep only the last `keepCount` messages\n const recentMessages = this.messages.slice(-keepCount);\n\n // Reset and rebuild with placeholder + recent messages\n this.messages.length = 0;\n this.messages.push(omittedPlaceholder);\n for (const msg of recentMessages) {\n this.messages.push(msg);\n }\n\n return true;\n }\n}\n"],"names":["Symbol","ConversationHistory","message","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","content","Array","j","item","subGoals","goal","existingByIndex","Map","mergedSubGoals","existingGoal","hasNonEmptyDescription","validSubGoals","index","updates","g","changed","firstPending","result","log","runningGoal","lines","currentGoal","currentGoalText","logLines","memory","threshold","keepCount","omittedCount","omittedPlaceholder","recentMessages","msg","options","normalizeAIMemory"],"mappings":";;;;;;;;;;;;eAkGGA,OAAO,QAAQ;;AAzFX,MAAMC;IAkBX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG;QAC7B,IAAI,CAAC,sBAAsB,GAAG;IAChC;IAOA,SAASE,SAAkB,EAAgC;QACzD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMP,UAAUI,cAAc,CAACG,EAAE;YACjC,MAAMC,UAAUR,QAAQ,OAAO;YAG/B,IAAIS,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,gBAAdA,KAAK,IAAI,EAAkB;oBAC7BL;oBAGA,IAAIA,aAAaJ,WACfM,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAON;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACN,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAQA,YAAYc,QAAmB,EAAQ;QACrC,IAAI,CAAC,QAAQ,GAAGA,SAAS,GAAG,CAAC,CAACC,OAAU;gBAAE,GAAGA,IAAI;YAAC;QAClD,IAAI,CAAC,yBAAyB;IAChC;IASA,cAAcD,QAAmB,EAAQ;QACvC,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAQ,YAC9B,IAAI,CAAC,WAAW,CAACA;QAInB,MAAME,kBAAkB,IAAIC,IAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACF,OAAS;gBAACA,KAAK,KAAK;gBAAEA;aAAK;QAGhD,MAAMG,iBAAiBJ,SAAS,GAAG,CAAC,CAACC;YACnC,MAAMI,eAAeH,gBAAgB,GAAG,CAACD,KAAK,KAAK;YACnD,MAAMK,yBAAyBL,KAAK,WAAW,CAAC,IAAI,GAAG,MAAM,GAAG;YAEhE,IAAI,CAACI,gBAAgB,CAACC,wBACpB,OAAO;YAGT,OAAO;gBACL,GAAGL,IAAI;gBACP,aACEK,0BAA0B,CAACD,eACvBJ,KAAK,WAAW,GAChBI,aAAa,WAAW;YAChC;QACF;QAEA,MAAME,gBAAgBH,eAAe,MAAM,CAAC,CAACH,OAASA,AAAS,SAATA;QACtD,IAAIM,AAAyB,MAAzBA,cAAc,MAAM,EACtB;QAGF,IAAI,CAAC,WAAW,CAACA;IACnB;IAOA,cACEC,KAAa,EACbC,OAAwC,EAC/B;QACT,MAAMR,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACS,IAAMA,EAAE,KAAK,KAAKF;QACnD,IAAI,CAACP,MACH,OAAO;QAGT,IAAIU,UAAU;QAEd,IAAIF,AAAmBlB,WAAnBkB,QAAQ,MAAM,IAAkBA,QAAQ,MAAM,KAAKR,KAAK,MAAM,EAAE;YAClEA,KAAK,MAAM,GAAGQ,QAAQ,MAAM;YAC5BE,UAAU;QACZ;QACA,IACEF,AAAwBlB,WAAxBkB,QAAQ,WAAW,IACnBA,QAAQ,WAAW,KAAKR,KAAK,WAAW,EACxC;YACAA,KAAK,WAAW,GAAGQ,QAAQ,WAAW;YACtCE,UAAU;QACZ;QAEA,IAAIA,SACFV,KAAK,IAAI,GAAG,EAAE;QAGhB,OAAO;IACT;IAMA,4BAAkC;QAChC,MAAMW,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACF,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACvD,IAAIE,cAAc;YAChBA,aAAa,MAAM,GAAG;YACtBA,aAAa,IAAI,GAAG,EAAE;QACxB;IACF;IAOA,oBAAoBJ,KAAa,EAAW;QAC1C,MAAMK,SAAS,IAAI,CAAC,aAAa,CAACL,OAAO;YAAE,QAAQ;QAAW;QAC9D,IAAIK,QACF,IAAI,CAAC,yBAAyB;QAEhC,OAAOA;IACT;IAMA,0BAAgC;QAC9B,KAAK,MAAMZ,QAAQ,IAAI,CAAC,QAAQ,CAAE;YAChC,IAAIA,AAAgB,eAAhBA,KAAK,MAAM,EACbA,KAAK,IAAI,GAAG,EAAE;YAEhBA,KAAK,MAAM,GAAG;QAChB;IACF;IAMA,iBAAiBa,GAAW,EAAQ;QAClC,IAAI,CAACA,KACH;QAEF,MAAMC,cAAc,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACL,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACtD,IAAIK,aAAa;YACf,IAAI,CAACA,YAAY,IAAI,EACnBA,YAAY,IAAI,GAAG,EAAE;YAEvBA,YAAY,IAAI,CAAC,IAAI,CAACD;QACxB;IACF;IAMA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,MAAME,QAAQ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACf,OACxB,GAAGA,KAAK,KAAK,CAAC,EAAE,EAAEA,KAAK,WAAW,CAAC,EAAE,EAAEA,KAAK,MAAM,CAAC,CAAC,CAAC;QAI9D,MAAMgB,cACJ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAChB,OAASA,AAAgB,cAAhBA,KAAK,MAAM,KACxC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACA,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAE1C,IAAIiB,kBAAkB;QACtB,IAAID,aAAa;YACfC,kBAAkB,CAAC,uBAAuB,EAAED,YAAY,WAAW,EAAE;YACrE,IAAIA,YAAY,IAAI,IAAIA,YAAY,IAAI,CAAC,MAAM,GAAG,GAAG;gBACnD,MAAME,WAAWF,YAAY,IAAI,CAAC,GAAG,CAAC,CAACH,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;gBAChEI,mBAAmB,CAAC,2CAA2C,EAAEC,UAAU;YAC7E;QACF;QAEA,OAAO,CAAC,YAAY,EAAEH,MAAM,IAAI,CAAC,QAAQE,iBAAiB;IAC5D;IAQA,oBAAoBJ,GAAW,EAAQ;QACrC,IAAIA,KACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAACA;IAE7B;IAMA,uBAA+B;QAC7B,IAAI,AAA+B,MAA/B,IAAI,CAAC,cAAc,CAAC,MAAM,EAC5B,OAAO;QAGT,MAAMK,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAACL,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;QACnE,OAAO,CAAC,6CAA6C,EAAEK,UAAU;IACnE;IAOA,aAAaC,MAAc,EAAQ;QACjC,IAAIA,QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IAEvB;IAKA,cAAwB;QACtB,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAKA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACjF;IAKA,gBAAsB;QACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IASA,gBAAgBC,SAAiB,EAAEC,SAAiB,EAAW;QAC7D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAID,WAC1B,OAAO;QAGT,MAAME,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAGD;QAC5C,MAAME,qBAAiD;YACrD,MAAM;YACN,SAAS,CAAC,CAAC,EAAED,aAAa,kDAAkD,CAAC;QAC/E;QAGA,MAAME,iBAAiB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAACH;QAG5C,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACE;QACnB,KAAK,MAAME,OAAOD,eAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACC;QAGrB,OAAO;IACT;IAhWA,YAAYC,OAAoC,CAAE;QAPlD,uBAAiB,YAAyC,EAAE;QAC5D,uBAAQ,YAAsB,EAAE;QAChC,uBAAQ,YAAqB,EAAE;QAC/B,uBAAQ,kBAA2B,EAAE;QAErC,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnCC,kBAAkBD,SAAS,iBAAiB,OAAO,CAAC,CAACP;YACnD,IAAI,CAAC,YAAY,CAACA;QACpB;QACA,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAyVF"}
1	+ {"version":3,"file":"ai-model/conversation-history.mjs","sources":["../../../src/ai-model/conversation-history.ts"],"sourcesContent":["import type { AIMemory, SubGoal } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { normalizeAIMemory } from './prompt/util';\n\nexport interface ConversationHistoryOptions {\n initialMessages?: ChatCompletionMessageParam[];\n initialMemories?: AIMemory;\n}\n\nexport class ConversationHistory {\n private readonly messages: ChatCompletionMessageParam[] = [];\n private subGoals: SubGoal[] = [];\n private memories: string[] = [];\n private historicalLogs: string[] = [];\n\n public pendingFeedbackMessage: string;\n\n constructor(options?: ConversationHistoryOptions) {\n if (options?.initialMessages?.length) {\n this.seed(options.initialMessages);\n }\n normalizeAIMemory(options?.initialMemories).forEach((memory) => {\n this.appendMemory(memory);\n });\n this.pendingFeedbackMessage = '';\n }\n\n resetPendingFeedbackMessageIfExists() {\n if (this.pendingFeedbackMessage) {\n this.pendingFeedbackMessage = '';\n }\n }\n\n append(message: ChatCompletionMessageParam) {\n this.messages.push(message);\n }\n\n seed(messages: ChatCompletionMessageParam[]) {\n this.reset();\n messages.forEach((message) => {\n this.append(message);\n });\n }\n\n reset() {\n this.messages.length = 0;\n this.memories.length = 0;\n this.subGoals.length = 0;\n this.historicalLogs.length = 0;\n this.pendingFeedbackMessage = '';\n }\n\n /*\n Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n * @returns The snapshot of the conversation history.\n /\n snapshot(maxImages?: number): ChatCompletionMessageParam[] {\n if (maxImages === undefined) {\n return [...this.messages];\n }\n\n const clonedMessages = structuredClone(this.messages);\n let imageCount = 0;\n\n // Traverse from the end to the beginning\n for (let i = clonedMessages.length - 1; i >= 0; i--) {\n const message = clonedMessages[i];\n const content = message.content;\n\n // Only process if content is an array\n if (Array.isArray(content)) {\n for (let j = 0; j < content.length; j++) {\n const item = content[j];\n\n // Check if this is an image\n if (item.type === 'image_url') {\n imageCount++;\n\n // If we've exceeded the limit, replace with text\n if (imageCount > maxImages) {\n content[j] = {\n type: 'text',\n text: '(image ignored due to size optimization)',\n };\n }\n }\n }\n }\n }\n\n return clonedMessages;\n }\n\n get length(): number {\n return this.messages.length;\n }\n\n [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n return this.messages[Symbol.iterator]();\n }\n\n toJSON(): ChatCompletionMessageParam[] {\n return this.snapshot();\n }\n\n // Sub-goal management methods\n\n /\n Set all sub-goals, replacing any existing ones.\n * Automatically marks the first pending goal as running.\n /\n setSubGoals(subGoals: SubGoal[]): void {\n this.subGoals = subGoals.map((goal) => ({ ...goal }));\n this.markFirstPendingAsRunning();\n }\n\n /\n Merge sub-goals from update-plan-content.\n * Preserves existing descriptions when incoming description is empty.\n \n This handles compact XML updates like:\n * <sub-goal index=\"1\" status=\"finished\" />\n /\n mergeSubGoals(subGoals: SubGoal[]): void {\n if (this.subGoals.length === 0) {\n this.setSubGoals(subGoals);\n return;\n }\n\n const existingByIndex = new Map(\n this.subGoals.map((goal) => [goal.index, goal] as const),\n );\n\n const mergedSubGoals = subGoals.map((goal) => {\n const existingGoal = existingByIndex.get(goal.index);\n const hasNonEmptyDescription = goal.description.trim().length > 0;\n\n if (!existingGoal && !hasNonEmptyDescription) {\n return null;\n }\n\n return {\n ...goal,\n description:\n hasNonEmptyDescription \|\| !existingGoal\n ? goal.description\n : existingGoal.description,\n };\n });\n\n const validSubGoals = mergedSubGoals.filter((goal) => goal !== null);\n if (validSubGoals.length === 0) {\n return;\n }\n\n this.setSubGoals(validSubGoals);\n }\n\n /\n Update a single sub-goal by index.\n * Clears logs if status or description actually changes.\n * @returns true if the sub-goal was found and updated, false otherwise\n /\n updateSubGoal(\n index: number,\n updates: Partial<Omit<SubGoal, 'index'>>,\n ): boolean {\n const goal = this.subGoals.find((g) => g.index === index);\n if (!goal) {\n return false;\n }\n\n let changed = false;\n\n if (updates.status !== undefined && updates.status !== goal.status) {\n goal.status = updates.status;\n changed = true;\n }\n if (\n updates.description !== undefined &&\n updates.description !== goal.description\n ) {\n goal.description = updates.description;\n changed = true;\n }\n\n if (changed) {\n goal.logs = [];\n }\n\n return true;\n }\n\n /\n Mark the first pending sub-goal as running.\n * Clears logs since status changes.\n /\n markFirstPendingAsRunning(): void {\n const firstPending = this.subGoals.find((g) => g.status === 'pending');\n if (firstPending) {\n firstPending.status = 'running';\n firstPending.logs = [];\n }\n }\n\n /\n Mark a sub-goal as finished.\n * Automatically marks the next pending goal as running.\n * @returns true if the sub-goal was found and updated, false otherwise\n /\n markSubGoalFinished(index: number): boolean {\n const result = this.updateSubGoal(index, { status: 'finished' });\n if (result) {\n this.markFirstPendingAsRunning();\n }\n return result;\n }\n\n /\n Mark all sub-goals as finished.\n * Clears logs for any goal whose status actually changes.\n /\n markAllSubGoalsFinished(): void {\n for (const goal of this.subGoals) {\n if (goal.status !== 'finished') {\n goal.logs = [];\n }\n goal.status = 'finished';\n }\n }\n\n /\n Append a log entry to the currently running sub-goal.\n * The log describes an action performed while working on the sub-goal.\n /\n appendSubGoalLog(log: string): void {\n if (!log) {\n return;\n }\n const runningGoal = this.subGoals.find((g) => g.status === 'running');\n if (runningGoal) {\n if (!runningGoal.logs) {\n runningGoal.logs = [];\n }\n runningGoal.logs.push(log);\n }\n }\n\n /\n Convert sub-goals to text representation.\n * Includes actions performed (logs) for the current sub-goal.\n /\n subGoalsToText(): string {\n if (this.subGoals.length === 0) {\n return '';\n }\n\n const lines = this.subGoals.map((goal) => {\n return `${goal.index}. ${goal.description} (${goal.status})`;\n });\n\n // Running goal takes priority, otherwise show first pending\n const currentGoal =\n this.subGoals.find((goal) => goal.status === 'running') \|\|\n this.subGoals.find((goal) => goal.status === 'pending');\n\n let currentGoalText = '';\n if (currentGoal) {\n currentGoalText = `\\nCurrent sub-goal is: ${currentGoal.description}`;\n if (currentGoal.logs && currentGoal.logs.length > 0) {\n const logLines = currentGoal.logs.map((log) => `- ${log}`).join('\\n');\n currentGoalText += `\\nActions performed for current sub-goal:\\n${logLines}`;\n }\n }\n\n return `Sub-goals:\\n${lines.join('\\n')}${currentGoalText}`;\n }\n\n // Historical log management methods (used in non-deepThink mode)\n\n /\n Append a log entry to the historical logs list.\n * Used in non-deepThink mode to track executed steps across planning rounds.\n /\n appendHistoricalLog(log: string): void {\n if (log) {\n this.historicalLogs.push(log);\n }\n }\n\n /\n Convert historical logs to text representation.\n * Provides context about previously executed steps to the model.\n /\n historicalLogsToText(): string {\n if (this.historicalLogs.length === 0) {\n return '';\n }\n\n const logLines = this.historicalLogs.map((log) => `- ${log}`).join('\\n');\n return `Here are the steps that have been executed:\\n${logLines}`;\n }\n\n // Memory management methods\n\n /\n Append a memory to the memories list\n /\n appendMemory(memory: string): void {\n if (memory) {\n this.memories.push(memory);\n }\n }\n\n /\n Get all memories\n /\n getMemories(): string[] {\n return [...this.memories];\n }\n\n /\n Convert memories to text representation\n /\n memoriesToText(): string {\n if (this.memories.length === 0) {\n return '';\n }\n\n return `Memories from previous steps:\\n---\\n${this.memories.join('\\n---\\n')}\\n`;\n }\n\n /\n Clear all memories\n /\n clearMemories(): void {\n this.memories.length = 0;\n }\n\n /\n Compress the conversation history if it exceeds the threshold.\n * Removes the oldest messages and replaces them with a single placeholder message.\n * @param threshold - The number of messages that triggers compression.\n * @param keepCount - The number of recent messages to keep after compression.\n * @returns true if compression was performed, false otherwise.\n */\n compressHistory(threshold: number, keepCount: number): boolean {\n if (this.messages.length <= threshold) {\n return false;\n }\n\n const omittedCount = this.messages.length - keepCount;\n const omittedPlaceholder: ChatCompletionMessageParam = {\n role: 'user',\n content: `(${omittedCount} previous conversation messages have been omitted)`,\n };\n\n // Keep only the last `keepCount` messages\n const recentMessages = this.messages.slice(-keepCount);\n\n // Reset and rebuild with placeholder + recent messages\n this.messages.length = 0;\n this.messages.push(omittedPlaceholder);\n for (const msg of recentMessages) {\n this.messages.push(msg);\n }\n\n return true;\n }\n}\n"],"names":["Symbol","ConversationHistory","message","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","content","Array","j","item","subGoals","goal","existingByIndex","Map","mergedSubGoals","existingGoal","hasNonEmptyDescription","validSubGoals","index","updates","g","changed","firstPending","result","log","runningGoal","lines","currentGoal","currentGoalText","logLines","memory","threshold","keepCount","omittedCount","omittedPlaceholder","recentMessages","msg","options","normalizeAIMemory"],"mappings":";;;;;;;;;;;;eAkGGA,OAAO,QAAQ;;AAzFX,MAAMC;IAkBX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG;QAC7B,IAAI,CAAC,sBAAsB,GAAG;IAChC;IAOA,SAASE,SAAkB,EAAgC;QACzD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMP,UAAUI,cAAc,CAACG,EAAE;YACjC,MAAMC,UAAUR,QAAQ,OAAO;YAG/B,IAAIS,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,gBAAdA,KAAK,IAAI,EAAkB;oBAC7BL;oBAGA,IAAIA,aAAaJ,WACfM,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAON;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACN,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAQA,YAAYc,QAAmB,EAAQ;QACrC,IAAI,CAAC,QAAQ,GAAGA,SAAS,GAAG,CAAC,CAACC,OAAU;gBAAE,GAAGA,IAAI;YAAC;QAClD,IAAI,CAAC,yBAAyB;IAChC;IASA,cAAcD,QAAmB,EAAQ;QACvC,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAQ,YAC9B,IAAI,CAAC,WAAW,CAACA;QAInB,MAAME,kBAAkB,IAAIC,IAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACF,OAAS;gBAACA,KAAK,KAAK;gBAAEA;aAAK;QAGhD,MAAMG,iBAAiBJ,SAAS,GAAG,CAAC,CAACC;YACnC,MAAMI,eAAeH,gBAAgB,GAAG,CAACD,KAAK,KAAK;YACnD,MAAMK,yBAAyBL,KAAK,WAAW,CAAC,IAAI,GAAG,MAAM,GAAG;YAEhE,IAAI,CAACI,gBAAgB,CAACC,wBACpB,OAAO;YAGT,OAAO;gBACL,GAAGL,IAAI;gBACP,aACEK,0BAA0B,CAACD,eACvBJ,KAAK,WAAW,GAChBI,aAAa,WAAW;YAChC;QACF;QAEA,MAAME,gBAAgBH,eAAe,MAAM,CAAC,CAACH,OAASA,AAAS,SAATA;QACtD,IAAIM,AAAyB,MAAzBA,cAAc,MAAM,EACtB;QAGF,IAAI,CAAC,WAAW,CAACA;IACnB;IAOA,cACEC,KAAa,EACbC,OAAwC,EAC/B;QACT,MAAMR,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACS,IAAMA,EAAE,KAAK,KAAKF;QACnD,IAAI,CAACP,MACH,OAAO;QAGT,IAAIU,UAAU;QAEd,IAAIF,AAAmBlB,WAAnBkB,QAAQ,MAAM,IAAkBA,QAAQ,MAAM,KAAKR,KAAK,MAAM,EAAE;YAClEA,KAAK,MAAM,GAAGQ,QAAQ,MAAM;YAC5BE,UAAU;QACZ;QACA,IACEF,AAAwBlB,WAAxBkB,QAAQ,WAAW,IACnBA,QAAQ,WAAW,KAAKR,KAAK,WAAW,EACxC;YACAA,KAAK,WAAW,GAAGQ,QAAQ,WAAW;YACtCE,UAAU;QACZ;QAEA,IAAIA,SACFV,KAAK,IAAI,GAAG,EAAE;QAGhB,OAAO;IACT;IAMA,4BAAkC;QAChC,MAAMW,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACF,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACvD,IAAIE,cAAc;YAChBA,aAAa,MAAM,GAAG;YACtBA,aAAa,IAAI,GAAG,EAAE;QACxB;IACF;IAOA,oBAAoBJ,KAAa,EAAW;QAC1C,MAAMK,SAAS,IAAI,CAAC,aAAa,CAACL,OAAO;YAAE,QAAQ;QAAW;QAC9D,IAAIK,QACF,IAAI,CAAC,yBAAyB;QAEhC,OAAOA;IACT;IAMA,0BAAgC;QAC9B,KAAK,MAAMZ,QAAQ,IAAI,CAAC,QAAQ,CAAE;YAChC,IAAIA,AAAgB,eAAhBA,KAAK,MAAM,EACbA,KAAK,IAAI,GAAG,EAAE;YAEhBA,KAAK,MAAM,GAAG;QAChB;IACF;IAMA,iBAAiBa,GAAW,EAAQ;QAClC,IAAI,CAACA,KACH;QAEF,MAAMC,cAAc,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACL,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACtD,IAAIK,aAAa;YACf,IAAI,CAACA,YAAY,IAAI,EACnBA,YAAY,IAAI,GAAG,EAAE;YAEvBA,YAAY,IAAI,CAAC,IAAI,CAACD;QACxB;IACF;IAMA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,MAAME,QAAQ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACf,OACxB,GAAGA,KAAK,KAAK,CAAC,EAAE,EAAEA,KAAK,WAAW,CAAC,EAAE,EAAEA,KAAK,MAAM,CAAC,CAAC,CAAC;QAI9D,MAAMgB,cACJ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAChB,OAASA,AAAgB,cAAhBA,KAAK,MAAM,KACxC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACA,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAE1C,IAAIiB,kBAAkB;QACtB,IAAID,aAAa;YACfC,kBAAkB,CAAC,uBAAuB,EAAED,YAAY,WAAW,EAAE;YACrE,IAAIA,YAAY,IAAI,IAAIA,YAAY,IAAI,CAAC,MAAM,GAAG,GAAG;gBACnD,MAAME,WAAWF,YAAY,IAAI,CAAC,GAAG,CAAC,CAACH,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;gBAChEI,mBAAmB,CAAC,2CAA2C,EAAEC,UAAU;YAC7E;QACF;QAEA,OAAO,CAAC,YAAY,EAAEH,MAAM,IAAI,CAAC,QAAQE,iBAAiB;IAC5D;IAQA,oBAAoBJ,GAAW,EAAQ;QACrC,IAAIA,KACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAACA;IAE7B;IAMA,uBAA+B;QAC7B,IAAI,AAA+B,MAA/B,IAAI,CAAC,cAAc,CAAC,MAAM,EAC5B,OAAO;QAGT,MAAMK,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAACL,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;QACnE,OAAO,CAAC,6CAA6C,EAAEK,UAAU;IACnE;IAOA,aAAaC,MAAc,EAAQ;QACjC,IAAIA,QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IAEvB;IAKA,cAAwB;QACtB,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAKA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACjF;IAKA,gBAAsB;QACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IASA,gBAAgBC,SAAiB,EAAEC,SAAiB,EAAW;QAC7D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAID,WAC1B,OAAO;QAGT,MAAME,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAGD;QAC5C,MAAME,qBAAiD;YACrD,MAAM;YACN,SAAS,CAAC,CAAC,EAAED,aAAa,kDAAkD,CAAC;QAC/E;QAGA,MAAME,iBAAiB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAACH;QAG5C,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACE;QACnB,KAAK,MAAME,OAAOD,eAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACC;QAGrB,OAAO;IACT;IAhWA,YAAYC,OAAoC,CAAE;QAPlD,uBAAiB,YAAyC,EAAE;QAC5D,uBAAQ,YAAsB,EAAE;QAChC,uBAAQ,YAAqB,EAAE;QAC/B,uBAAQ,kBAA2B,EAAE;QAErC,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnCC,kBAAkBD,SAAS,iBAAiB,OAAO,CAAC,CAACP;YACnD,IAAI,CAAC,YAAY,CAACA;QACpB;QACA,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAyVF"}

package/dist/es/ai-model/index.mjs CHANGED Viewed

@@ -4,9 +4,10 @@ import { systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { generatePlaywrightTest, generatePlaywrightTestStream } from "./prompt/playwright-generator.mjs";
 import { generateYamlTest, generateYamlTestStream } from "./prompt/yaml-generator.mjs";
 import { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection } from "./inspect.mjs";
+import { getModelRuntime } from "./models/index.mjs";
 import { plan } from "./llm-planning.mjs";
 import { autoGLMPlanning } from "./auto-glm/planning.mjs";
 import { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, parseActionParam } from "../common.mjs";
 import { uiTarsPlanning } from "./ui-tars-planning.mjs";
 import { ConversationHistory } from "./conversation-history.mjs";
-export { AIResponseParseError, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, autoGLMPlanning, callAI, callAIWithObjectResponse, callAIWithStringResponse, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, parseActionParam, plan, runConnectivityTest, systemPromptToLocateElement, uiTarsPlanning };
+export { AIResponseParseError, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, autoGLMPlanning, callAI, callAIWithObjectResponse, callAIWithStringResponse, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, getModelRuntime, parseActionParam, plan, runConnectivityTest, systemPromptToLocateElement, uiTarsPlanning };

package/dist/es/ai-model/inspect.mjs CHANGED Viewed

@@ -9,8 +9,8 @@ import { isAutoGLM } from "./auto-glm/util.mjs";
 import { extractDataQueryPrompt, parseXMLExtractionResponse, systemPromptToExtract } from "./prompt/extraction.mjs";
 import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
-import { appendAIMemoryToText } from "./prompt/util.mjs";
 import { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive } from "./prompt/order-sensitive-judge.mjs";
+import { appendAIMemoryToText } from "./prompt/util.mjs";
 import { AIResponseParseError, callAI, callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
 const debugInspect = getDebug('ai:inspect');
 const debugSection = getDebug('ai:section');

package/dist/es/ai-model/inspect.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n AIDataExtractionResponse,\n AIElementResponse,\n AISectionLocatorResponse,\n AIUsageInfo,\n Rect,\n ServiceExtractOption,\n UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@aiscene/shared/env';\nimport {\n generateElementByPoint,\n generateElementByRect,\n} from '@aiscene/shared/extractor/dom-util';\nimport {\n cropByRect,\n paddingToMatchBlockByBase64,\n preProcessImageUrl,\n scaleImage,\n} from '@aiscene/shared/img';\nimport { getDebug } from '@aiscene/shared/logger';\nimport type { LocateResultElement } from '@aiscene/shared/types';\nimport { assert } from '@aiscene/shared/utils';\nimport type {\n ChatCompletionSystemMessageParam,\n ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { adaptBboxToRect, expandSearchArea, mergeRects } from '../common';\nimport { parseAutoGLMLocateResponse } from './auto-glm/parser';\nimport { getAutoGLMLocatePrompt } from './auto-glm/prompt';\nimport { isAutoGLM } from './auto-glm/util';\nimport {\n extractDataQueryPrompt,\n parseXMLExtractionResponse,\n systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n findElementPrompt,\n systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n sectionLocatorInstruction,\n systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport { appendAIMemoryToText } from './prompt/util';\nimport {\n orderSensitiveJudgePrompt,\n systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n AIResponseParseError,\n callAI,\n callAIWithObjectResponse,\n callAIWithStringResponse,\n} from './service-caller/index';\n\nexport type AIArgs = [\n ChatCompletionSystemMessageParam,\n ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport async function buildSearchAreaConfig(options: {\n context: UIContext;\n baseRect: Rect;\n modelFamily: IModelConfig['modelFamily'];\n}): Promise<{ rect: Rect; imageBase64: string; scale: number }> {\n const { context, baseRect, modelFamily } = options;\n const scaleRatio = 2;\n const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n const croppedResult = await cropByRect(\n context.screenshot.base64,\n sectionRect,\n modelFamily === 'qwen2.5-vl',\n );\n\n const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n sectionRect.width = scaledResult.width;\n sectionRect.height = scaledResult.height;\n return {\n rect: sectionRect,\n imageBase64: scaledResult.imageBase64,\n scale: scaleRatio,\n };\n}\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n if (typeof prompt === 'string') {\n return prompt;\n } else {\n return prompt.prompt;\n }\n};\n\nconst promptsToChatParam = async (\n multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n const msgs: ChatCompletionUserMessageParam[] = [];\n if (multimodalPrompt?.images?.length) {\n msgs.push({\n role: 'user',\n content: [\n {\n type: 'text',\n text: 'Next, I will provide all the reference images.',\n },\n ],\n });\n\n for (const item of multimodalPrompt.images) {\n const base64 = await preProcessImageUrl(\n item.url,\n !!multimodalPrompt.convertHttpImage2Base64,\n );\n\n msgs.push({\n role: 'user',\n content: [\n {\n type: 'text',\n text: `this is the reference image named '${item.name}':`,\n },\n ],\n });\n\n msgs.push({\n role: 'user',\n content: [\n {\n type: 'image_url',\n image_url: {\n url: base64,\n detail: 'high',\n },\n },\n ],\n });\n }\n }\n return msgs;\n};\n\nexport async function AiLocateElement(options: {\n context: UIContext;\n targetElementDescription: TUserPrompt;\n searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n modelConfig: IModelConfig;\n memories?: readonly string[];\n abortSignal?: AbortSignal;\n}): Promise<{\n parseResult: {\n elements: LocateResultElement[];\n errors?: string[];\n };\n rect?: Rect;\n rawResponse: string;\n usage?: AIUsageInfo;\n reasoning_content?: string;\n}> {\n const { context, targetElementDescription, modelConfig } = options;\n const { modelFamily } = modelConfig;\n const screenshotBase64 = context.screenshot.base64;\n\n assert(\n targetElementDescription,\n 'cannot find the target element description',\n );\n const targetElementDescriptionText = extraTextFromUserPrompt(\n targetElementDescription,\n );\n const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n const systemPrompt = isAutoGLM(modelFamily)\n ? getAutoGLMLocatePrompt(modelFamily)\n : systemPromptToLocateElement(modelFamily);\n\n let imagePayload = screenshotBase64;\n let imageWidth = context.shotSize.width;\n let imageHeight = context.shotSize.height;\n let originalImageWidth = imageWidth;\n let originalImageHeight = imageHeight;\n\n if (options.searchConfig) {\n assert(\n options.searchConfig.rect,\n 'searchArea is provided but its rect cannot be found. Failed to locate element',\n );\n assert(\n options.searchConfig.imageBase64,\n 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n );\n\n imagePayload = options.searchConfig.imageBase64;\n imageWidth = options.searchConfig.rect?.width;\n imageHeight = options.searchConfig.rect?.height;\n originalImageWidth = imageWidth;\n originalImageHeight = imageHeight;\n } else if (modelFamily === 'qwen2.5-vl') {\n const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n imageWidth = paddedResult.width;\n imageHeight = paddedResult.height;\n imagePayload = paddedResult.imageBase64;\n }\n\n const msgs: AIArgs = [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: [\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n {\n type: 'text',\n text: appendAIMemoryToText(\n isAutoGLM(modelFamily)\n ? `Tap: ${userInstructionPrompt}`\n : userInstructionPrompt,\n options.memories,\n ),\n },\n ],\n },\n ];\n\n if (typeof targetElementDescription !== 'string') {\n const addOns = await promptsToChatParam({\n images: targetElementDescription.images,\n convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n });\n msgs.push(...addOns);\n }\n\n if (isAutoGLM(modelFamily)) {\n const { content: rawResponseContent, usage } =\n await callAIWithStringResponse(msgs, modelConfig, {\n abortSignal: options.abortSignal,\n });\n\n debugInspect('auto-glm rawResponse:', rawResponseContent);\n\n const parsed = parseAutoGLMLocateResponse(rawResponseContent);\n\n debugInspect('auto-glm thinking:', parsed.think);\n debugInspect('auto-glm coordinates:', parsed.coordinates);\n\n let resRect: Rect \| undefined;\n let matchedElements: LocateResultElement[] = [];\n let errors: string[] = [];\n\n if (parsed.error \|\| !parsed.coordinates) {\n errors = [parsed.error \|\| 'Failed to parse auto-glm response'];\n debugInspect('auto-glm parse error:', errors[0]);\n } else {\n const { x, y } = parsed.coordinates;\n\n debugInspect('auto-glm coordinates [0-999]:', { x, y });\n\n // Convert auto-glm coordinates [0,999] to pixel bbox\n // Map from [0,999] to pixel coordinates\n const pixelX = Math.round((x * imageWidth) / 1000);\n const pixelY = Math.round((y * imageHeight) / 1000);\n\n debugInspect('auto-glm pixel coordinates:', { pixelX, pixelY });\n\n // Apply offset if searching in a cropped area\n let finalX = pixelX;\n let finalY = pixelY;\n if (options.searchConfig?.rect) {\n finalX += options.searchConfig.rect.left;\n finalY += options.searchConfig.rect.top;\n }\n\n const element: LocateResultElement = generateElementByPoint(\n [finalX, finalY],\n targetElementDescriptionText as string,\n );\n\n resRect = element.rect;\n debugInspect('auto-glm resRect:', resRect);\n\n if (element) {\n matchedElements = [element];\n }\n }\n\n return {\n rect: resRect,\n parseResult: {\n elements: matchedElements,\n errors,\n },\n rawResponse: rawResponseContent,\n usage,\n reasoning_content: parsed.think,\n };\n }\n\n let res: Awaited<\n ReturnType<\n typeof callAIWithObjectResponse<AIElementResponse \| [number, number]>\n >\n >;\n try {\n res = await callAIWithObjectResponse<AIElementResponse \| [number, number]>(\n msgs,\n modelConfig,\n {\n abortSignal: options.abortSignal,\n },\n );\n } catch (callError) {\n // Return error with usage and rawResponse if available\n const errorMessage =\n callError instanceof Error ? callError.message : String(callError);\n const rawResponse =\n callError instanceof AIResponseParseError\n ? callError.rawResponse\n : errorMessage;\n const usage =\n callError instanceof AIResponseParseError ? callError.usage : undefined;\n return {\n rect: undefined,\n parseResult: {\n elements: [],\n errors: [`AI call error: ${errorMessage}`],\n },\n rawResponse,\n usage,\n reasoning_content: undefined,\n };\n }\n\n const rawResponse = JSON.stringify(res.content);\n\n let resRect: Rect \| undefined;\n let matchedElements: LocateResultElement[] = [];\n let errors: string[] \| undefined =\n 'errors' in res.content ? res.content.errors : [];\n try {\n if (\n 'bbox' in res.content &&\n Array.isArray(res.content.bbox) &&\n res.content.bbox.length >= 1\n ) {\n resRect = adaptBboxToRect(\n res.content.bbox,\n imageWidth,\n imageHeight,\n options.searchConfig?.rect?.left,\n options.searchConfig?.rect?.top,\n originalImageWidth,\n originalImageHeight,\n modelFamily,\n options.searchConfig?.scale,\n );\n\n debugInspect('resRect', resRect);\n\n const element: LocateResultElement = generateElementByRect(\n resRect,\n targetElementDescriptionText as string,\n );\n errors = [];\n\n if (element) {\n matchedElements = [element];\n }\n }\n } catch (e) {\n const msg =\n e instanceof Error\n ? `Failed to parse bbox: ${e.message}`\n : 'unknown error in locate';\n if (!errors \|\| errors?.length === 0) {\n errors = [msg];\n } else {\n errors.push(`(${msg})`);\n }\n }\n\n return {\n rect: resRect,\n parseResult: {\n elements: matchedElements as LocateResultElement[],\n errors: errors as string[],\n },\n rawResponse,\n usage: res.usage,\n reasoning_content: res.reasoning_content,\n };\n}\n\nexport async function AiLocateSection(options: {\n context: UIContext;\n sectionDescription: TUserPrompt;\n modelConfig: IModelConfig;\n memories?: readonly string[];\n abortSignal?: AbortSignal;\n}): Promise<{\n rect?: Rect;\n imageBase64?: string;\n scale?: number;\n error?: string;\n rawResponse: string;\n usage?: AIUsageInfo;\n}> {\n const { context, sectionDescription, modelConfig } = options;\n const { modelFamily } = modelConfig;\n const screenshotBase64 = context.screenshot.base64;\n\n const systemPrompt = systemPromptToLocateSection(modelFamily);\n const sectionLocatorInstructionText = sectionLocatorInstruction(\n extraTextFromUserPrompt(sectionDescription),\n );\n const msgs: AIArgs = [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: [\n {\n type: 'image_url',\n image_url: {\n url: screenshotBase64,\n detail: 'high',\n },\n },\n {\n type: 'text',\n text: appendAIMemoryToText(\n sectionLocatorInstructionText,\n options.memories,\n ),\n },\n ],\n },\n ];\n\n if (typeof sectionDescription !== 'string') {\n const addOns = await promptsToChatParam({\n images: sectionDescription.images,\n convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n });\n msgs.push(...addOns);\n }\n\n let result: Awaited<\n ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n >;\n try {\n result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n msgs,\n modelConfig,\n {\n abortSignal: options.abortSignal,\n },\n );\n } catch (callError) {\n // Return error with usage and rawResponse if available\n const errorMessage =\n callError instanceof Error ? callError.message : String(callError);\n const rawResponse =\n callError instanceof AIResponseParseError\n ? callError.rawResponse\n : errorMessage;\n const usage =\n callError instanceof AIResponseParseError ? callError.usage : undefined;\n return {\n rect: undefined,\n imageBase64: undefined,\n error: `AI call error: ${errorMessage}`,\n rawResponse,\n usage,\n };\n }\n\n let searchAreaConfig:\n \| Awaited<ReturnType<typeof buildSearchAreaConfig>>\n \| undefined;\n const sectionBbox = result.content.bbox;\n if (sectionBbox) {\n const targetRect = adaptBboxToRect(\n sectionBbox,\n context.shotSize.width,\n context.shotSize.height,\n 0,\n 0,\n context.shotSize.width,\n context.shotSize.height,\n modelFamily,\n );\n debugSection('original targetRect %j', targetRect);\n\n const referenceBboxList = result.content.references_bbox \|\| [];\n debugSection('referenceBboxList %j', referenceBboxList);\n\n const referenceRects = referenceBboxList\n .filter((bbox) => Array.isArray(bbox))\n .map((bbox) => {\n return adaptBboxToRect(\n bbox,\n context.shotSize.width,\n context.shotSize.height,\n 0,\n 0,\n context.shotSize.width,\n context.shotSize.height,\n modelFamily,\n );\n });\n debugSection('referenceRects %j', referenceRects);\n\n // merge the sectionRect and referenceRects\n const mergedRect = mergeRects([targetRect, ...referenceRects]);\n debugSection('mergedRect %j', mergedRect);\n\n const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n const originalWidth = expandedRect.width;\n const originalHeight = expandedRect.height;\n debugSection('expanded sectionRect %j', expandedRect);\n\n searchAreaConfig = await buildSearchAreaConfig({\n context,\n baseRect: mergedRect,\n modelFamily,\n });\n\n debugSection(\n 'scaled sectionRect from %dx%d to %dx%d (scale=%d)',\n originalWidth,\n originalHeight,\n searchAreaConfig.rect.width,\n searchAreaConfig.rect.height,\n searchAreaConfig.scale,\n );\n }\n\n return {\n rect: searchAreaConfig?.rect,\n imageBase64: searchAreaConfig?.imageBase64,\n scale: searchAreaConfig?.scale,\n error: result.content.error,\n rawResponse: JSON.stringify(result.content),\n usage: result.usage,\n };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n dataQuery: string \| Record<string, string>;\n multimodalPrompt?: TMultimodalPrompt;\n context: UIContext;\n pageDescription?: string;\n extractOption?: ServiceExtractOption;\n modelConfig: IModelConfig;\n memories?: readonly string[];\n}) {\n const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n options;\n const systemPrompt = systemPromptToExtract();\n const screenshotBase64 = context.screenshot.base64;\n\n const extractDataPromptText = extractDataQueryPrompt(\n options.pageDescription \|\| '',\n dataQuery,\n );\n\n const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n if (extractOption?.screenshotIncluded !== false) {\n userContent.push({\n type: 'image_url',\n image_url: {\n url: screenshotBase64,\n detail: 'high',\n },\n });\n }\n\n userContent.push({\n type: 'text',\n text: appendAIMemoryToText(extractDataPromptText, options.memories),\n });\n\n const msgs: AIArgs = [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: userContent,\n },\n ];\n\n if (multimodalPrompt) {\n const addOns = await promptsToChatParam({\n images: multimodalPrompt.images,\n convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n });\n msgs.push(...addOns);\n }\n\n const {\n content: rawResponse,\n usage,\n reasoning_content,\n } = await callAI(msgs, modelConfig);\n\n // Parse XML response to JSON object\n let parseResult: AIDataExtractionResponse<T>;\n try {\n parseResult = parseXMLExtractionResponse<T>(rawResponse);\n } catch (parseError) {\n // Throw AIResponseParseError with usage and rawResponse preserved\n const errorMessage =\n parseError instanceof Error ? parseError.message : String(parseError);\n throw new AIResponseParseError(\n `XML parse error: ${errorMessage}`,\n rawResponse,\n usage,\n );\n }\n\n return {\n parseResult,\n rawResponse,\n usage,\n reasoning_content,\n };\n}\n\nexport async function AiJudgeOrderSensitive(\n description: string,\n callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n modelConfig: IModelConfig,\n): Promise<{\n isOrderSensitive: boolean;\n usage?: AIUsageInfo;\n}> {\n const systemPrompt = systemPromptToJudgeOrderSensitive();\n const userPrompt = orderSensitiveJudgePrompt(description);\n\n const msgs: AIArgs = [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: userPrompt,\n },\n ];\n\n debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n const result = await callAIFn(msgs, modelConfig);\n\n return {\n isOrderSensitive: result.content.isOrderSensitive ?? false,\n usage: result.usage,\n };\n}\n"],"names":["debugInspect","getDebug","debugSection","buildSearchAreaConfig","options","context","baseRect","modelFamily","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","targetElementDescription","modelConfig","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","isAutoGLM","getAutoGLMLocatePrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","paddedResult","paddingToMatchBlockByBase64","appendAIMemoryToText","addOns","rawResponseContent","usage","callAIWithStringResponse","parsed","parseAutoGLMLocateResponse","resRect","matchedElements","errors","x","y","pixelX","Math","pixelY","finalX","finalY","element","generateElementByPoint","res","callAIWithObjectResponse","callError","errorMessage","Error","String","rawResponse","AIResponseParseError","undefined","JSON","Array","adaptBboxToRect","generateElementByRect","e","msg","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","searchAreaConfig","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandedRect","originalWidth","originalHeight","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","reasoning_content","callAI","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","callAIFn","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;;AA8DA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAEvB,eAAeE,sBAAsBC,OAI3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGH;IAC3C,MAAMI,aAAa;IACnB,MAAMC,cAAcC,iBAAiBJ,UAAUD,QAAQ,QAAQ;IAE/D,MAAMM,gBAAgB,MAAMC,WAC1BP,QAAQ,UAAU,CAAC,MAAM,EACzBI,aACAF,AAAgB,iBAAhBA;IAGF,MAAMM,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjEC,YAAY,KAAK,GAAGI,aAAa,KAAK;IACtCJ,YAAY,MAAM,GAAGI,aAAa,MAAM;IACxC,OAAO;QACL,MAAMJ;QACN,aAAaI,aAAa,WAAW;QACrC,OAAOL;IACT;AACF;AAEA,MAAMO,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBnB,OAOrC;IAUC,MAAM,EAAEC,OAAO,EAAEmB,wBAAwB,EAAEC,WAAW,EAAE,GAAGrB;IAC3D,MAAM,EAAEG,WAAW,EAAE,GAAGkB;IACxB,MAAMC,mBAAmBrB,QAAQ,UAAU,CAAC,MAAM;IAElDsB,OACEH,0BACA;IAEF,MAAMI,+BAA+Bb,wBACnCS;IAEF,MAAMK,wBAAwBC,kBAAkBF;IAChD,MAAMG,eAAeC,UAAUzB,eAC3B0B,uBAAuB1B,eACvB2B,4BAA4B3B;IAEhC,IAAI4B,eAAeT;IACnB,IAAIU,aAAa/B,QAAQ,QAAQ,CAAC,KAAK;IACvC,IAAIgC,cAAchC,QAAQ,QAAQ,CAAC,MAAM;IACzC,IAAIiC,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIjC,QAAQ,YAAY,EAAE;QACxBuB,OACEvB,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFuB,OACEvB,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGF+B,eAAe/B,QAAQ,YAAY,CAAC,WAAW;QAC/CgC,aAAahC,QAAQ,YAAY,CAAC,IAAI,EAAE;QACxCiC,cAAcjC,QAAQ,YAAY,CAAC,IAAI,EAAE;QACzCkC,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAI9B,AAAgB,iBAAhBA,aAA8B;QACvC,MAAMiC,eAAe,MAAMC,4BAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAMrB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASY;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMO,qBACJV,UAAUzB,eACN,CAAC,KAAK,EAAEsB,uBAAuB,GAC/BA,uBACJzB,QAAQ,QAAQ;gBAEpB;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOoB,0BAAuC;QAChD,MAAMmB,SAAS,MAAM1B,mBAAmB;YACtC,QAAQO,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAL,KAAK,IAAI,IAAIwB;IACf;IAEA,IAAIX,UAAUzB,cAAc;QAC1B,MAAM,EAAE,SAASqC,kBAAkB,EAAEC,KAAK,EAAE,GAC1C,MAAMC,yBAAyB3B,MAAMM,aAAa;YAChD,aAAarB,QAAQ,WAAW;QAClC;QAEFJ,aAAa,yBAAyB4C;QAEtC,MAAMG,SAASC,2BAA2BJ;QAE1C5C,aAAa,sBAAsB+C,OAAO,KAAK;QAC/C/C,aAAa,yBAAyB+C,OAAO,WAAW;QAExD,IAAIE;QACJ,IAAIC,kBAAyC,EAAE;QAC/C,IAAIC,SAAmB,EAAE;QAEzB,IAAIJ,OAAO,KAAK,IAAI,CAACA,OAAO,WAAW,EAAE;YACvCI,SAAS;gBAACJ,OAAO,KAAK,IAAI;aAAoC;YAC9D/C,aAAa,yBAAyBmD,MAAM,CAAC,EAAE;QACjD,OAAO;YACL,MAAM,EAAEC,CAAC,EAAEC,CAAC,EAAE,GAAGN,OAAO,WAAW;YAEnC/C,aAAa,iCAAiC;gBAAEoD;gBAAGC;YAAE;YAIrD,MAAMC,SAASC,KAAK,KAAK,CAAEH,IAAIhB,aAAc;YAC7C,MAAMoB,SAASD,KAAK,KAAK,CAAEF,IAAIhB,cAAe;YAE9CrC,aAAa,+BAA+B;gBAAEsD;gBAAQE;YAAO;YAG7D,IAAIC,SAASH;YACb,IAAII,SAASF;YACb,IAAIpD,QAAQ,YAAY,EAAE,MAAM;gBAC9BqD,UAAUrD,QAAQ,YAAY,CAAC,IAAI,CAAC,IAAI;gBACxCsD,UAAUtD,QAAQ,YAAY,CAAC,IAAI,CAAC,GAAG;YACzC;YAEA,MAAMuD,UAA+BC,uBACnC;gBAACH;gBAAQC;aAAO,EAChB9B;YAGFqB,UAAUU,QAAQ,IAAI;YACtB3D,aAAa,qBAAqBiD;YAElC,IAAIU,SACFT,kBAAkB;gBAACS;aAAQ;QAE/B;QAEA,OAAO;YACL,MAAMV;YACN,aAAa;gBACX,UAAUC;gBACVC;YACF;YACA,aAAaP;YACbC;YACA,mBAAmBE,OAAO,KAAK;QACjC;IACF;IAEA,IAAIc;IAKJ,IAAI;QACFA,MAAM,MAAMC,yBACV3C,MACAM,aACA;YACE,aAAarB,QAAQ,WAAW;QAClC;IAEJ,EAAE,OAAO2D,WAAW;QAElB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMnB,QACJkB,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAGM;QAChE,OAAO;YACL,MAAMA;YACN,aAAa;gBACX,UAAU,EAAE;gBACZ,QAAQ;oBAAC,CAAC,eAAe,EAAEL,cAAc;iBAAC;YAC5C;YACAG;YACAtB;YACA,mBAAmBwB;QACrB;IACF;IAEA,MAAMF,cAAcG,KAAK,SAAS,CAACT,IAAI,OAAO;IAE9C,IAAIZ;IACJ,IAAIC,kBAAyC,EAAE;IAC/C,IAAIC,SACF,YAAYU,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBU,MAAM,OAAO,CAACV,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;YACAZ,UAAUuB,gBACRX,IAAI,OAAO,CAAC,IAAI,EAChBzB,YACAC,aACAjC,QAAQ,YAAY,EAAE,MAAM,MAC5BA,QAAQ,YAAY,EAAE,MAAM,KAC5BkC,oBACAC,qBACAhC,aACAH,QAAQ,YAAY,EAAE;YAGxBJ,aAAa,WAAWiD;YAExB,MAAMU,UAA+Bc,sBACnCxB,SACArB;YAEFuB,SAAS,EAAE;YAEX,IAAIQ,SACFT,kBAAkB;gBAACS;aAAQ;QAE/B;IACF,EAAE,OAAOe,GAAG;QACV,MAAMC,MACJD,aAAaT,QACT,CAAC,sBAAsB,EAAES,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACvB,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEwB,IAAI,CAAC,CAAC;aAFtBxB,SAAS;YAACwB;SAAI;IAIlB;IAEA,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAgB;QACA,OAAON,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAee,gBAAgBxE,OAMrC;IAQC,MAAM,EAAEC,OAAO,EAAEwE,kBAAkB,EAAEpD,WAAW,EAAE,GAAGrB;IACrD,MAAM,EAAEG,WAAW,EAAE,GAAGkB;IACxB,MAAMC,mBAAmBrB,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM0B,eAAe+C,4BAA4BvE;IACjD,MAAMwE,gCAAgCC,0BACpCjE,wBAAwB8D;IAE1B,MAAM1D,OAAe;QACnB;YAAE,MAAM;YAAU,SAASY;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMgB,qBACJqC,+BACA3E,QAAQ,QAAQ;gBAEpB;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOyE,oBAAiC;QAC1C,MAAMlC,SAAS,MAAM1B,mBAAmB;YACtC,QAAQ4D,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA1D,KAAK,IAAI,IAAIwB;IACf;IAEA,IAAIsC;IAGJ,IAAI;QACFA,SAAS,MAAMnB,yBACb3C,MACAM,aACA;YACE,aAAarB,QAAQ,WAAW;QAClC;IAEJ,EAAE,OAAO2D,WAAW;QAElB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMnB,QACJkB,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAGM;QAChE,OAAO;YACL,MAAMA;YACN,aAAaA;YACb,OAAO,CAAC,eAAe,EAAEL,cAAc;YACvCG;YACAtB;QACF;IACF;IAEA,IAAIqC;IAGJ,MAAMC,cAAcF,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIE,aAAa;QACf,MAAMC,aAAaZ,gBACjBW,aACA9E,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvB,GACA,GACAA,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvBE;QAEFL,aAAa,0BAA0BkF;QAEvC,MAAMC,oBAAoBJ,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9D/E,aAAa,wBAAwBmF;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAShB,MAAM,OAAO,CAACgB,OAC/B,GAAG,CAAC,CAACA,OACGf,gBACLe,MACAlF,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvB,GACA,GACAA,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvBE;QAGNL,aAAa,qBAAqBoF;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DpF,aAAa,iBAAiBsF;QAE9B,MAAME,eAAehF,iBAAiB8E,YAAYnF,QAAQ,QAAQ;QAClE,MAAMsF,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1CxF,aAAa,2BAA2BwF;QAExCR,mBAAmB,MAAM/E,sBAAsB;YAC7CE;YACA,UAAUmF;YACVjF;QACF;QAEAL,aACE,qDACAyF,eACAC,gBACAV,iBAAiB,IAAI,CAAC,KAAK,EAC3BA,iBAAiB,IAAI,CAAC,MAAM,EAC5BA,iBAAiB,KAAK;IAE1B;IAEA,OAAO;QACL,MAAMA,kBAAkB;QACxB,aAAaA,kBAAkB;QAC/B,OAAOA,kBAAkB;QACzB,OAAOD,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAaX,KAAK,SAAS,CAACW,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeY,qBAAwBzF,OAQ7C;IACC,MAAM,EAAE0F,SAAS,EAAEzF,OAAO,EAAE0F,aAAa,EAAE7E,gBAAgB,EAAEO,WAAW,EAAE,GACxErB;IACF,MAAM2B,eAAeiE;IACrB,MAAMtE,mBAAmBrB,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM4F,wBAAwBC,uBAC5B9F,QAAQ,eAAe,IAAI,IAC3B0F;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKzE;YACL,QAAQ;QACV;IACF;IAGFyE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMzD,qBAAqBuD,uBAAuB7F,QAAQ,QAAQ;IACpE;IAEA,MAAMe,OAAe;QACnB;YAAE,MAAM;YAAU,SAASY;QAAa;QACxC;YACE,MAAM;YACN,SAASoE;QACX;KACD;IAED,IAAIjF,kBAAkB;QACpB,MAAMyB,SAAS,MAAM1B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAIwB;IACf;IAEA,MAAM,EACJ,SAASwB,WAAW,EACpBtB,KAAK,EACLuD,iBAAiB,EAClB,GAAG,MAAMC,OAAOlF,MAAMM;IAGvB,IAAI6E;IACJ,IAAI;QACFA,cAAcC,2BAA8BpC;IAC9C,EAAE,OAAOqC,YAAY;QAEnB,MAAMxC,eACJwC,sBAAsBvC,QAAQuC,WAAW,OAAO,GAAGtC,OAAOsC;QAC5D,MAAM,IAAIpC,qBACR,CAAC,iBAAiB,EAAEJ,cAAc,EAClCG,aACAtB;IAEJ;IAEA,OAAO;QACLyD;QACAnC;QACAtB;QACAuD;IACF;AACF;AAEO,eAAeK,sBACpBC,WAAmB,EACnBC,QAAwE,EACxElF,WAAyB;IAKzB,MAAMM,eAAe6E;IACrB,MAAMC,aAAaC,0BAA0BJ;IAE7C,MAAMvF,OAAe;QACnB;YAAE,MAAM;YAAU,SAASY;QAAa;QACxC;YACE,MAAM;YACN,SAAS8E;QACX;KACD;IAED7G,aAAa,yCAAyC0G;IAEtD,MAAMzB,SAAS,MAAM0B,SAASxF,MAAMM;IAEpC,OAAO;QACL,kBAAkBwD,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
1	+ {"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n AIDataExtractionResponse,\n AIElementResponse,\n AISectionLocatorResponse,\n AIUsageInfo,\n Rect,\n ServiceExtractOption,\n UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@aiscene/shared/env';\nimport {\n generateElementByPoint,\n generateElementByRect,\n} from '@aiscene/shared/extractor/dom-util';\nimport {\n cropByRect,\n paddingToMatchBlockByBase64,\n preProcessImageUrl,\n scaleImage,\n} from '@aiscene/shared/img';\nimport { getDebug } from '@aiscene/shared/logger';\nimport type { LocateResultElement } from '@aiscene/shared/types';\nimport { assert } from '@aiscene/shared/utils';\nimport type {\n ChatCompletionSystemMessageParam,\n ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { adaptBboxToRect, expandSearchArea, mergeRects } from '../common';\nimport { parseAutoGLMLocateResponse } from './auto-glm/parser';\nimport { getAutoGLMLocatePrompt } from './auto-glm/prompt';\nimport { isAutoGLM } from './auto-glm/util';\nimport {\n extractDataQueryPrompt,\n parseXMLExtractionResponse,\n systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n findElementPrompt,\n systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n sectionLocatorInstruction,\n systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n orderSensitiveJudgePrompt,\n systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport { appendAIMemoryToText } from './prompt/util';\nimport {\n AIResponseParseError,\n callAI,\n callAIWithObjectResponse,\n callAIWithStringResponse,\n} from './service-caller/index';\n\nexport type AIArgs = [\n ChatCompletionSystemMessageParam,\n ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport async function buildSearchAreaConfig(options: {\n context: UIContext;\n baseRect: Rect;\n modelFamily: IModelConfig['modelFamily'];\n}): Promise<{ rect: Rect; imageBase64: string; scale: number }> {\n const { context, baseRect, modelFamily } = options;\n const scaleRatio = 2;\n const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n const croppedResult = await cropByRect(\n context.screenshot.base64,\n sectionRect,\n modelFamily === 'qwen2.5-vl',\n );\n\n const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n sectionRect.width = scaledResult.width;\n sectionRect.height = scaledResult.height;\n return {\n rect: sectionRect,\n imageBase64: scaledResult.imageBase64,\n scale: scaleRatio,\n };\n}\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n if (typeof prompt === 'string') {\n return prompt;\n } else {\n return prompt.prompt;\n }\n};\n\nconst promptsToChatParam = async (\n multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n const msgs: ChatCompletionUserMessageParam[] = [];\n if (multimodalPrompt?.images?.length) {\n msgs.push({\n role: 'user',\n content: [\n {\n type: 'text',\n text: 'Next, I will provide all the reference images.',\n },\n ],\n });\n\n for (const item of multimodalPrompt.images) {\n const base64 = await preProcessImageUrl(\n item.url,\n !!multimodalPrompt.convertHttpImage2Base64,\n );\n\n msgs.push({\n role: 'user',\n content: [\n {\n type: 'text',\n text: `this is the reference image named '${item.name}':`,\n },\n ],\n });\n\n msgs.push({\n role: 'user',\n content: [\n {\n type: 'image_url',\n image_url: {\n url: base64,\n detail: 'high',\n },\n },\n ],\n });\n }\n }\n return msgs;\n};\n\nexport async function AiLocateElement(options: {\n context: UIContext;\n targetElementDescription: TUserPrompt;\n searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n modelConfig: IModelConfig;\n memories?: readonly string[];\n abortSignal?: AbortSignal;\n}): Promise<{\n parseResult: {\n elements: LocateResultElement[];\n errors?: string[];\n };\n rect?: Rect;\n rawResponse: string;\n usage?: AIUsageInfo;\n reasoning_content?: string;\n}> {\n const { context, targetElementDescription, modelConfig } = options;\n const { modelFamily } = modelConfig;\n const screenshotBase64 = context.screenshot.base64;\n\n assert(\n targetElementDescription,\n 'cannot find the target element description',\n );\n const targetElementDescriptionText = extraTextFromUserPrompt(\n targetElementDescription,\n );\n const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n const systemPrompt = isAutoGLM(modelFamily)\n ? getAutoGLMLocatePrompt(modelFamily)\n : systemPromptToLocateElement(modelFamily);\n\n let imagePayload = screenshotBase64;\n let imageWidth = context.shotSize.width;\n let imageHeight = context.shotSize.height;\n let originalImageWidth = imageWidth;\n let originalImageHeight = imageHeight;\n\n if (options.searchConfig) {\n assert(\n options.searchConfig.rect,\n 'searchArea is provided but its rect cannot be found. Failed to locate element',\n );\n assert(\n options.searchConfig.imageBase64,\n 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n );\n\n imagePayload = options.searchConfig.imageBase64;\n imageWidth = options.searchConfig.rect?.width;\n imageHeight = options.searchConfig.rect?.height;\n originalImageWidth = imageWidth;\n originalImageHeight = imageHeight;\n } else if (modelFamily === 'qwen2.5-vl') {\n const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n imageWidth = paddedResult.width;\n imageHeight = paddedResult.height;\n imagePayload = paddedResult.imageBase64;\n }\n\n const msgs: AIArgs = [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: [\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n {\n type: 'text',\n text: appendAIMemoryToText(\n isAutoGLM(modelFamily)\n ? `Tap: ${userInstructionPrompt}`\n : userInstructionPrompt,\n options.memories,\n ),\n },\n ],\n },\n ];\n\n if (typeof targetElementDescription !== 'string') {\n const addOns = await promptsToChatParam({\n images: targetElementDescription.images,\n convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n });\n msgs.push(...addOns);\n }\n\n if (isAutoGLM(modelFamily)) {\n const { content: rawResponseContent, usage } =\n await callAIWithStringResponse(msgs, modelConfig, {\n abortSignal: options.abortSignal,\n });\n\n debugInspect('auto-glm rawResponse:', rawResponseContent);\n\n const parsed = parseAutoGLMLocateResponse(rawResponseContent);\n\n debugInspect('auto-glm thinking:', parsed.think);\n debugInspect('auto-glm coordinates:', parsed.coordinates);\n\n let resRect: Rect \| undefined;\n let matchedElements: LocateResultElement[] = [];\n let errors: string[] = [];\n\n if (parsed.error \|\| !parsed.coordinates) {\n errors = [parsed.error \|\| 'Failed to parse auto-glm response'];\n debugInspect('auto-glm parse error:', errors[0]);\n } else {\n const { x, y } = parsed.coordinates;\n\n debugInspect('auto-glm coordinates [0-999]:', { x, y });\n\n // Convert auto-glm coordinates [0,999] to pixel bbox\n // Map from [0,999] to pixel coordinates\n const pixelX = Math.round((x * imageWidth) / 1000);\n const pixelY = Math.round((y * imageHeight) / 1000);\n\n debugInspect('auto-glm pixel coordinates:', { pixelX, pixelY });\n\n // Apply offset if searching in a cropped area\n let finalX = pixelX;\n let finalY = pixelY;\n if (options.searchConfig?.rect) {\n finalX += options.searchConfig.rect.left;\n finalY += options.searchConfig.rect.top;\n }\n\n const element: LocateResultElement = generateElementByPoint(\n [finalX, finalY],\n targetElementDescriptionText as string,\n );\n\n resRect = element.rect;\n debugInspect('auto-glm resRect:', resRect);\n\n if (element) {\n matchedElements = [element];\n }\n }\n\n return {\n rect: resRect,\n parseResult: {\n elements: matchedElements,\n errors,\n },\n rawResponse: rawResponseContent,\n usage,\n reasoning_content: parsed.think,\n };\n }\n\n let res: Awaited<\n ReturnType<\n typeof callAIWithObjectResponse<AIElementResponse \| [number, number]>\n >\n >;\n try {\n res = await callAIWithObjectResponse<AIElementResponse \| [number, number]>(\n msgs,\n modelConfig,\n {\n abortSignal: options.abortSignal,\n },\n );\n } catch (callError) {\n // Return error with usage and rawResponse if available\n const errorMessage =\n callError instanceof Error ? callError.message : String(callError);\n const rawResponse =\n callError instanceof AIResponseParseError\n ? callError.rawResponse\n : errorMessage;\n const usage =\n callError instanceof AIResponseParseError ? callError.usage : undefined;\n return {\n rect: undefined,\n parseResult: {\n elements: [],\n errors: [`AI call error: ${errorMessage}`],\n },\n rawResponse,\n usage,\n reasoning_content: undefined,\n };\n }\n\n const rawResponse = JSON.stringify(res.content);\n\n let resRect: Rect \| undefined;\n let matchedElements: LocateResultElement[] = [];\n let errors: string[] \| undefined =\n 'errors' in res.content ? res.content.errors : [];\n try {\n if (\n 'bbox' in res.content &&\n Array.isArray(res.content.bbox) &&\n res.content.bbox.length >= 1\n ) {\n resRect = adaptBboxToRect(\n res.content.bbox,\n imageWidth,\n imageHeight,\n options.searchConfig?.rect?.left,\n options.searchConfig?.rect?.top,\n originalImageWidth,\n originalImageHeight,\n modelFamily,\n options.searchConfig?.scale,\n );\n\n debugInspect('resRect', resRect);\n\n const element: LocateResultElement = generateElementByRect(\n resRect,\n targetElementDescriptionText as string,\n );\n errors = [];\n\n if (element) {\n matchedElements = [element];\n }\n }\n } catch (e) {\n const msg =\n e instanceof Error\n ? `Failed to parse bbox: ${e.message}`\n : 'unknown error in locate';\n if (!errors \|\| errors?.length === 0) {\n errors = [msg];\n } else {\n errors.push(`(${msg})`);\n }\n }\n\n return {\n rect: resRect,\n parseResult: {\n elements: matchedElements as LocateResultElement[],\n errors: errors as string[],\n },\n rawResponse,\n usage: res.usage,\n reasoning_content: res.reasoning_content,\n };\n}\n\nexport async function AiLocateSection(options: {\n context: UIContext;\n sectionDescription: TUserPrompt;\n modelConfig: IModelConfig;\n memories?: readonly string[];\n abortSignal?: AbortSignal;\n}): Promise<{\n rect?: Rect;\n imageBase64?: string;\n scale?: number;\n error?: string;\n rawResponse: string;\n usage?: AIUsageInfo;\n}> {\n const { context, sectionDescription, modelConfig } = options;\n const { modelFamily } = modelConfig;\n const screenshotBase64 = context.screenshot.base64;\n\n const systemPrompt = systemPromptToLocateSection(modelFamily);\n const sectionLocatorInstructionText = sectionLocatorInstruction(\n extraTextFromUserPrompt(sectionDescription),\n );\n const msgs: AIArgs = [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: [\n {\n type: 'image_url',\n image_url: {\n url: screenshotBase64,\n detail: 'high',\n },\n },\n {\n type: 'text',\n text: appendAIMemoryToText(\n sectionLocatorInstructionText,\n options.memories,\n ),\n },\n ],\n },\n ];\n\n if (typeof sectionDescription !== 'string') {\n const addOns = await promptsToChatParam({\n images: sectionDescription.images,\n convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n });\n msgs.push(...addOns);\n }\n\n let result: Awaited<\n ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n >;\n try {\n result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n msgs,\n modelConfig,\n {\n abortSignal: options.abortSignal,\n },\n );\n } catch (callError) {\n // Return error with usage and rawResponse if available\n const errorMessage =\n callError instanceof Error ? callError.message : String(callError);\n const rawResponse =\n callError instanceof AIResponseParseError\n ? callError.rawResponse\n : errorMessage;\n const usage =\n callError instanceof AIResponseParseError ? callError.usage : undefined;\n return {\n rect: undefined,\n imageBase64: undefined,\n error: `AI call error: ${errorMessage}`,\n rawResponse,\n usage,\n };\n }\n\n let searchAreaConfig:\n \| Awaited<ReturnType<typeof buildSearchAreaConfig>>\n \| undefined;\n const sectionBbox = result.content.bbox;\n if (sectionBbox) {\n const targetRect = adaptBboxToRect(\n sectionBbox,\n context.shotSize.width,\n context.shotSize.height,\n 0,\n 0,\n context.shotSize.width,\n context.shotSize.height,\n modelFamily,\n );\n debugSection('original targetRect %j', targetRect);\n\n const referenceBboxList = result.content.references_bbox \|\| [];\n debugSection('referenceBboxList %j', referenceBboxList);\n\n const referenceRects = referenceBboxList\n .filter((bbox) => Array.isArray(bbox))\n .map((bbox) => {\n return adaptBboxToRect(\n bbox,\n context.shotSize.width,\n context.shotSize.height,\n 0,\n 0,\n context.shotSize.width,\n context.shotSize.height,\n modelFamily,\n );\n });\n debugSection('referenceRects %j', referenceRects);\n\n // merge the sectionRect and referenceRects\n const mergedRect = mergeRects([targetRect, ...referenceRects]);\n debugSection('mergedRect %j', mergedRect);\n\n const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n const originalWidth = expandedRect.width;\n const originalHeight = expandedRect.height;\n debugSection('expanded sectionRect %j', expandedRect);\n\n searchAreaConfig = await buildSearchAreaConfig({\n context,\n baseRect: mergedRect,\n modelFamily,\n });\n\n debugSection(\n 'scaled sectionRect from %dx%d to %dx%d (scale=%d)',\n originalWidth,\n originalHeight,\n searchAreaConfig.rect.width,\n searchAreaConfig.rect.height,\n searchAreaConfig.scale,\n );\n }\n\n return {\n rect: searchAreaConfig?.rect,\n imageBase64: searchAreaConfig?.imageBase64,\n scale: searchAreaConfig?.scale,\n error: result.content.error,\n rawResponse: JSON.stringify(result.content),\n usage: result.usage,\n };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n dataQuery: string \| Record<string, string>;\n multimodalPrompt?: TMultimodalPrompt;\n context: UIContext;\n pageDescription?: string;\n extractOption?: ServiceExtractOption;\n modelConfig: IModelConfig;\n memories?: readonly string[];\n}) {\n const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n options;\n const systemPrompt = systemPromptToExtract();\n const screenshotBase64 = context.screenshot.base64;\n\n const extractDataPromptText = extractDataQueryPrompt(\n options.pageDescription \|\| '',\n dataQuery,\n );\n\n const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n if (extractOption?.screenshotIncluded !== false) {\n userContent.push({\n type: 'image_url',\n image_url: {\n url: screenshotBase64,\n detail: 'high',\n },\n });\n }\n\n userContent.push({\n type: 'text',\n text: appendAIMemoryToText(extractDataPromptText, options.memories),\n });\n\n const msgs: AIArgs = [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: userContent,\n },\n ];\n\n if (multimodalPrompt) {\n const addOns = await promptsToChatParam({\n images: multimodalPrompt.images,\n convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n });\n msgs.push(...addOns);\n }\n\n const {\n content: rawResponse,\n usage,\n reasoning_content,\n } = await callAI(msgs, modelConfig);\n\n // Parse XML response to JSON object\n let parseResult: AIDataExtractionResponse<T>;\n try {\n parseResult = parseXMLExtractionResponse<T>(rawResponse);\n } catch (parseError) {\n // Throw AIResponseParseError with usage and rawResponse preserved\n const errorMessage =\n parseError instanceof Error ? parseError.message : String(parseError);\n throw new AIResponseParseError(\n `XML parse error: ${errorMessage}`,\n rawResponse,\n usage,\n );\n }\n\n return {\n parseResult,\n rawResponse,\n usage,\n reasoning_content,\n };\n}\n\nexport async function AiJudgeOrderSensitive(\n description: string,\n callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n modelConfig: IModelConfig,\n): Promise<{\n isOrderSensitive: boolean;\n usage?: AIUsageInfo;\n}> {\n const systemPrompt = systemPromptToJudgeOrderSensitive();\n const userPrompt = orderSensitiveJudgePrompt(description);\n\n const msgs: AIArgs = [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: userPrompt,\n },\n ];\n\n debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n const result = await callAIFn(msgs, modelConfig);\n\n return {\n isOrderSensitive: result.content.isOrderSensitive ?? false,\n usage: result.usage,\n };\n}\n"],"names":["debugInspect","getDebug","debugSection","buildSearchAreaConfig","options","context","baseRect","modelFamily","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","targetElementDescription","modelConfig","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","isAutoGLM","getAutoGLMLocatePrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","paddedResult","paddingToMatchBlockByBase64","appendAIMemoryToText","addOns","rawResponseContent","usage","callAIWithStringResponse","parsed","parseAutoGLMLocateResponse","resRect","matchedElements","errors","x","y","pixelX","Math","pixelY","finalX","finalY","element","generateElementByPoint","res","callAIWithObjectResponse","callError","errorMessage","Error","String","rawResponse","AIResponseParseError","undefined","JSON","Array","adaptBboxToRect","generateElementByRect","e","msg","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","searchAreaConfig","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandedRect","originalWidth","originalHeight","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","reasoning_content","callAI","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","callAIFn","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;;AA8DA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAEvB,eAAeE,sBAAsBC,OAI3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGH;IAC3C,MAAMI,aAAa;IACnB,MAAMC,cAAcC,iBAAiBJ,UAAUD,QAAQ,QAAQ;IAE/D,MAAMM,gBAAgB,MAAMC,WAC1BP,QAAQ,UAAU,CAAC,MAAM,EACzBI,aACAF,AAAgB,iBAAhBA;IAGF,MAAMM,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjEC,YAAY,KAAK,GAAGI,aAAa,KAAK;IACtCJ,YAAY,MAAM,GAAGI,aAAa,MAAM;IACxC,OAAO;QACL,MAAMJ;QACN,aAAaI,aAAa,WAAW;QACrC,OAAOL;IACT;AACF;AAEA,MAAMO,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBnB,OAOrC;IAUC,MAAM,EAAEC,OAAO,EAAEmB,wBAAwB,EAAEC,WAAW,EAAE,GAAGrB;IAC3D,MAAM,EAAEG,WAAW,EAAE,GAAGkB;IACxB,MAAMC,mBAAmBrB,QAAQ,UAAU,CAAC,MAAM;IAElDsB,OACEH,0BACA;IAEF,MAAMI,+BAA+Bb,wBACnCS;IAEF,MAAMK,wBAAwBC,kBAAkBF;IAChD,MAAMG,eAAeC,UAAUzB,eAC3B0B,uBAAuB1B,eACvB2B,4BAA4B3B;IAEhC,IAAI4B,eAAeT;IACnB,IAAIU,aAAa/B,QAAQ,QAAQ,CAAC,KAAK;IACvC,IAAIgC,cAAchC,QAAQ,QAAQ,CAAC,MAAM;IACzC,IAAIiC,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIjC,QAAQ,YAAY,EAAE;QACxBuB,OACEvB,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFuB,OACEvB,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGF+B,eAAe/B,QAAQ,YAAY,CAAC,WAAW;QAC/CgC,aAAahC,QAAQ,YAAY,CAAC,IAAI,EAAE;QACxCiC,cAAcjC,QAAQ,YAAY,CAAC,IAAI,EAAE;QACzCkC,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAI9B,AAAgB,iBAAhBA,aAA8B;QACvC,MAAMiC,eAAe,MAAMC,4BAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAMrB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASY;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMO,qBACJV,UAAUzB,eACN,CAAC,KAAK,EAAEsB,uBAAuB,GAC/BA,uBACJzB,QAAQ,QAAQ;gBAEpB;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOoB,0BAAuC;QAChD,MAAMmB,SAAS,MAAM1B,mBAAmB;YACtC,QAAQO,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAL,KAAK,IAAI,IAAIwB;IACf;IAEA,IAAIX,UAAUzB,cAAc;QAC1B,MAAM,EAAE,SAASqC,kBAAkB,EAAEC,KAAK,EAAE,GAC1C,MAAMC,yBAAyB3B,MAAMM,aAAa;YAChD,aAAarB,QAAQ,WAAW;QAClC;QAEFJ,aAAa,yBAAyB4C;QAEtC,MAAMG,SAASC,2BAA2BJ;QAE1C5C,aAAa,sBAAsB+C,OAAO,KAAK;QAC/C/C,aAAa,yBAAyB+C,OAAO,WAAW;QAExD,IAAIE;QACJ,IAAIC,kBAAyC,EAAE;QAC/C,IAAIC,SAAmB,EAAE;QAEzB,IAAIJ,OAAO,KAAK,IAAI,CAACA,OAAO,WAAW,EAAE;YACvCI,SAAS;gBAACJ,OAAO,KAAK,IAAI;aAAoC;YAC9D/C,aAAa,yBAAyBmD,MAAM,CAAC,EAAE;QACjD,OAAO;YACL,MAAM,EAAEC,CAAC,EAAEC,CAAC,EAAE,GAAGN,OAAO,WAAW;YAEnC/C,aAAa,iCAAiC;gBAAEoD;gBAAGC;YAAE;YAIrD,MAAMC,SAASC,KAAK,KAAK,CAAEH,IAAIhB,aAAc;YAC7C,MAAMoB,SAASD,KAAK,KAAK,CAAEF,IAAIhB,cAAe;YAE9CrC,aAAa,+BAA+B;gBAAEsD;gBAAQE;YAAO;YAG7D,IAAIC,SAASH;YACb,IAAII,SAASF;YACb,IAAIpD,QAAQ,YAAY,EAAE,MAAM;gBAC9BqD,UAAUrD,QAAQ,YAAY,CAAC,IAAI,CAAC,IAAI;gBACxCsD,UAAUtD,QAAQ,YAAY,CAAC,IAAI,CAAC,GAAG;YACzC;YAEA,MAAMuD,UAA+BC,uBACnC;gBAACH;gBAAQC;aAAO,EAChB9B;YAGFqB,UAAUU,QAAQ,IAAI;YACtB3D,aAAa,qBAAqBiD;YAElC,IAAIU,SACFT,kBAAkB;gBAACS;aAAQ;QAE/B;QAEA,OAAO;YACL,MAAMV;YACN,aAAa;gBACX,UAAUC;gBACVC;YACF;YACA,aAAaP;YACbC;YACA,mBAAmBE,OAAO,KAAK;QACjC;IACF;IAEA,IAAIc;IAKJ,IAAI;QACFA,MAAM,MAAMC,yBACV3C,MACAM,aACA;YACE,aAAarB,QAAQ,WAAW;QAClC;IAEJ,EAAE,OAAO2D,WAAW;QAElB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMnB,QACJkB,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAGM;QAChE,OAAO;YACL,MAAMA;YACN,aAAa;gBACX,UAAU,EAAE;gBACZ,QAAQ;oBAAC,CAAC,eAAe,EAAEL,cAAc;iBAAC;YAC5C;YACAG;YACAtB;YACA,mBAAmBwB;QACrB;IACF;IAEA,MAAMF,cAAcG,KAAK,SAAS,CAACT,IAAI,OAAO;IAE9C,IAAIZ;IACJ,IAAIC,kBAAyC,EAAE;IAC/C,IAAIC,SACF,YAAYU,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBU,MAAM,OAAO,CAACV,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;YACAZ,UAAUuB,gBACRX,IAAI,OAAO,CAAC,IAAI,EAChBzB,YACAC,aACAjC,QAAQ,YAAY,EAAE,MAAM,MAC5BA,QAAQ,YAAY,EAAE,MAAM,KAC5BkC,oBACAC,qBACAhC,aACAH,QAAQ,YAAY,EAAE;YAGxBJ,aAAa,WAAWiD;YAExB,MAAMU,UAA+Bc,sBACnCxB,SACArB;YAEFuB,SAAS,EAAE;YAEX,IAAIQ,SACFT,kBAAkB;gBAACS;aAAQ;QAE/B;IACF,EAAE,OAAOe,GAAG;QACV,MAAMC,MACJD,aAAaT,QACT,CAAC,sBAAsB,EAAES,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACvB,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEwB,IAAI,CAAC,CAAC;aAFtBxB,SAAS;YAACwB;SAAI;IAIlB;IAEA,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAgB;QACA,OAAON,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAee,gBAAgBxE,OAMrC;IAQC,MAAM,EAAEC,OAAO,EAAEwE,kBAAkB,EAAEpD,WAAW,EAAE,GAAGrB;IACrD,MAAM,EAAEG,WAAW,EAAE,GAAGkB;IACxB,MAAMC,mBAAmBrB,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM0B,eAAe+C,4BAA4BvE;IACjD,MAAMwE,gCAAgCC,0BACpCjE,wBAAwB8D;IAE1B,MAAM1D,OAAe;QACnB;YAAE,MAAM;YAAU,SAASY;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMgB,qBACJqC,+BACA3E,QAAQ,QAAQ;gBAEpB;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOyE,oBAAiC;QAC1C,MAAMlC,SAAS,MAAM1B,mBAAmB;YACtC,QAAQ4D,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA1D,KAAK,IAAI,IAAIwB;IACf;IAEA,IAAIsC;IAGJ,IAAI;QACFA,SAAS,MAAMnB,yBACb3C,MACAM,aACA;YACE,aAAarB,QAAQ,WAAW;QAClC;IAEJ,EAAE,OAAO2D,WAAW;QAElB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMnB,QACJkB,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAGM;QAChE,OAAO;YACL,MAAMA;YACN,aAAaA;YACb,OAAO,CAAC,eAAe,EAAEL,cAAc;YACvCG;YACAtB;QACF;IACF;IAEA,IAAIqC;IAGJ,MAAMC,cAAcF,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIE,aAAa;QACf,MAAMC,aAAaZ,gBACjBW,aACA9E,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvB,GACA,GACAA,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvBE;QAEFL,aAAa,0BAA0BkF;QAEvC,MAAMC,oBAAoBJ,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9D/E,aAAa,wBAAwBmF;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAShB,MAAM,OAAO,CAACgB,OAC/B,GAAG,CAAC,CAACA,OACGf,gBACLe,MACAlF,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvB,GACA,GACAA,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvBE;QAGNL,aAAa,qBAAqBoF;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DpF,aAAa,iBAAiBsF;QAE9B,MAAME,eAAehF,iBAAiB8E,YAAYnF,QAAQ,QAAQ;QAClE,MAAMsF,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1CxF,aAAa,2BAA2BwF;QAExCR,mBAAmB,MAAM/E,sBAAsB;YAC7CE;YACA,UAAUmF;YACVjF;QACF;QAEAL,aACE,qDACAyF,eACAC,gBACAV,iBAAiB,IAAI,CAAC,KAAK,EAC3BA,iBAAiB,IAAI,CAAC,MAAM,EAC5BA,iBAAiB,KAAK;IAE1B;IAEA,OAAO;QACL,MAAMA,kBAAkB;QACxB,aAAaA,kBAAkB;QAC/B,OAAOA,kBAAkB;QACzB,OAAOD,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAaX,KAAK,SAAS,CAACW,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeY,qBAAwBzF,OAQ7C;IACC,MAAM,EAAE0F,SAAS,EAAEzF,OAAO,EAAE0F,aAAa,EAAE7E,gBAAgB,EAAEO,WAAW,EAAE,GACxErB;IACF,MAAM2B,eAAeiE;IACrB,MAAMtE,mBAAmBrB,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM4F,wBAAwBC,uBAC5B9F,QAAQ,eAAe,IAAI,IAC3B0F;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKzE;YACL,QAAQ;QACV;IACF;IAGFyE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMzD,qBAAqBuD,uBAAuB7F,QAAQ,QAAQ;IACpE;IAEA,MAAMe,OAAe;QACnB;YAAE,MAAM;YAAU,SAASY;QAAa;QACxC;YACE,MAAM;YACN,SAASoE;QACX;KACD;IAED,IAAIjF,kBAAkB;QACpB,MAAMyB,SAAS,MAAM1B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAIwB;IACf;IAEA,MAAM,EACJ,SAASwB,WAAW,EACpBtB,KAAK,EACLuD,iBAAiB,EAClB,GAAG,MAAMC,OAAOlF,MAAMM;IAGvB,IAAI6E;IACJ,IAAI;QACFA,cAAcC,2BAA8BpC;IAC9C,EAAE,OAAOqC,YAAY;QAEnB,MAAMxC,eACJwC,sBAAsBvC,QAAQuC,WAAW,OAAO,GAAGtC,OAAOsC;QAC5D,MAAM,IAAIpC,qBACR,CAAC,iBAAiB,EAAEJ,cAAc,EAClCG,aACAtB;IAEJ;IAEA,OAAO;QACLyD;QACAnC;QACAtB;QACAuD;IACF;AACF;AAEO,eAAeK,sBACpBC,WAAmB,EACnBC,QAAwE,EACxElF,WAAyB;IAKzB,MAAMM,eAAe6E;IACrB,MAAMC,aAAaC,0BAA0BJ;IAE7C,MAAMvF,OAAe;QACnB;YAAE,MAAM;YAAU,SAASY;QAAa;QACxC;YACE,MAAM;YACN,SAAS8E;QACX;KACD;IAED7G,aAAa,yCAAyC0G;IAEtD,MAAMzB,SAAS,MAAM0B,SAASxF,MAAMM;IAEpC,OAAO;QACL,kBAAkBwD,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/model-adapter/chat-completion.mjs ADDED Viewed

@@ -0,0 +1,69 @@
+const defaultImageDetail = (_input)=>void 0;
+const defaultChatCompletionParams = ({ midsceneDefaults, userConfig })=>({
+        config: {
+            temperature: userConfig.temperature ?? midsceneDefaults.temperature
+        }
+    });
+const midsceneChatCompletionDefaults = {
+    temperature: 0
+};
+function createDefaultExtractContentAndReasoning(reasoningContentKeys) {
+    return (message)=>{
+        if (!message) return {
+            content: '',
+            reasoning_content: ''
+        };
+        const messageRecord = message;
+        const rawReasoning = reasoningContentKeys.map((key)=>messageRecord[key]).find((value)=>'string' == typeof value);
+        const messageReasoning = 'string' == typeof rawReasoning ? rawReasoning : '';
+        return {
+            content: 'string' == typeof message.content ? message.content : '',
+            reasoning_content: messageReasoning
+        };
+    };
+}
+function resolveExtractContentAndReasoning(chatCompletion) {
+    const messageExtraction = chatCompletion?.messageExtraction;
+    if (messageExtraction?.kind === 'custom') return messageExtraction.extractContentAndReasoning;
+    return createDefaultExtractContentAndReasoning(messageExtraction?.reasoningContentKeys ?? [
+        'reasoning_content'
+    ]);
+}
+function resolveChatCompletion(chatCompletion) {
+    const buildChatCompletionParams = chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;
+    const resolveImageDetail = chatCompletion?.resolveImageDetail ?? defaultImageDetail;
+    const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];
+    const extractContentAndReasoning = resolveExtractContentAndReasoning(chatCompletion);
+    const useReasoningAsContentFallback = chatCompletion?.useReasoningAsContentFallback ?? false;
+    const replayRawAssistantMessage = chatCompletion?.replayRawAssistantMessage ?? false;
+    return {
+        unsupportedUserConfig,
+        buildChatCompletionParams: (input)=>{
+            const context = {
+                ...input,
+                userConfig: input.userConfig ?? {},
+                midsceneDefaults: midsceneChatCompletionDefaults
+            };
+            const params = buildChatCompletionParams(context);
+            const retryAttempt = input.semanticRetryAttempt ?? 0;
+            if (retryAttempt > 0 && input.userConfig?.temperature === void 0 && 0 === params.config.temperature) return {
+                config: {
+                    ...params.config,
+                    temperature: 0.2
+                }
+            };
+            return params;
+        },
+        resolveImageDetail: (input)=>resolveImageDetail({
+                ...input,
+                userConfig: input.userConfig ?? {},
+                midsceneDefaults: midsceneChatCompletionDefaults
+            }),
+        extractContentAndReasoning,
+        useReasoningAsContentFallback,
+        replayRawAssistantMessage
+    };
+}
+export { resolveChatCompletion };
+//# sourceMappingURL=chat-completion.mjs.map

package/dist/es/ai-model/model-adapter/chat-completion.mjs.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"ai-model/model-adapter/chat-completion.mjs","sources":["../../../../src/ai-model/model-adapter/chat-completion.ts"],"sourcesContent":["import type {\n ChatCompletionAdapter,\n ChatCompletionCallContext,\n ExtractContentAndReasoning,\n MidsceneChatCompletionDefaults,\n ModelAdapterDefinition,\n} from './types';\n\nconst defaultImageDetail = (_input: unknown) => undefined;\n\nconst defaultChatCompletionParams = ({\n midsceneDefaults,\n userConfig,\n}: ChatCompletionCallContext) => ({\n config: {\n temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n },\n});\n\nconst midsceneChatCompletionDefaults: MidsceneChatCompletionDefaults = {\n temperature: 0,\n};\n\nfunction createDefaultExtractContentAndReasoning(\n reasoningContentKeys: string[],\n): ExtractContentAndReasoning {\n return (message) => {\n if (!message) {\n return {\n content: '',\n reasoning_content: '',\n };\n }\n\n const messageRecord = message as Record<string, unknown>;\n const rawReasoning = reasoningContentKeys\n .map((key) => messageRecord[key])\n .find((value) => typeof value === 'string');\n const messageReasoning =\n typeof rawReasoning === 'string' ? rawReasoning : '';\n\n return {\n content: typeof message.content === 'string' ? message.content : '',\n reasoning_content: messageReasoning,\n };\n };\n}\n\nfunction resolveExtractContentAndReasoning(\n chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ExtractContentAndReasoning {\n const messageExtraction = chatCompletion?.messageExtraction;\n if (messageExtraction?.kind === 'custom') {\n return messageExtraction.extractContentAndReasoning;\n }\n\n return createDefaultExtractContentAndReasoning(\n messageExtraction?.reasoningContentKeys ?? ['reasoning_content'],\n );\n}\n\nexport function resolveChatCompletion(\n chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ChatCompletionAdapter {\n const buildChatCompletionParams =\n chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;\n const resolveImageDetail =\n chatCompletion?.resolveImageDetail ?? defaultImageDetail;\n const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];\n const extractContentAndReasoning =\n resolveExtractContentAndReasoning(chatCompletion);\n const useReasoningAsContentFallback =\n chatCompletion?.useReasoningAsContentFallback ?? false;\n const replayRawAssistantMessage =\n chatCompletion?.replayRawAssistantMessage ?? false;\n\n return {\n unsupportedUserConfig,\n buildChatCompletionParams: (input) => {\n const context = {\n ...input,\n userConfig: input.userConfig ?? {},\n midsceneDefaults: midsceneChatCompletionDefaults,\n };\n const params = buildChatCompletionParams(context);\n const retryAttempt = input.semanticRetryAttempt ?? 0;\n // Only perturb the default deterministic request after a semantic parse\n // failure: preserve an explicit user temperature, and avoid adding a\n // temperature to adapters whose normal parameters do not include one.\n if (\n retryAttempt > 0 &&\n input.userConfig?.temperature === undefined &&\n params.config.temperature === 0\n ) {\n return {\n config: {\n ...params.config,\n temperature: 0.2,\n },\n };\n }\n return params;\n },\n resolveImageDetail: (input) =>\n resolveImageDetail({\n ...input,\n userConfig: input.userConfig ?? {},\n midsceneDefaults: midsceneChatCompletionDefaults,\n }),\n extractContentAndReasoning,\n useReasoningAsContentFallback,\n replayRawAssistantMessage,\n };\n}\n"],"names":["defaultImageDetail","_input","undefined","defaultChatCompletionParams","midsceneDefaults","userConfig","midsceneChatCompletionDefaults","createDefaultExtractContentAndReasoning","reasoningContentKeys","message","messageRecord","rawReasoning","key","value","messageReasoning","resolveExtractContentAndReasoning","chatCompletion","messageExtraction","resolveChatCompletion","buildChatCompletionParams","resolveImageDetail","unsupportedUserConfig","extractContentAndReasoning","useReasoningAsContentFallback","replayRawAssistantMessage","input","context","params","retryAttempt"],"mappings":"AAQA,MAAMA,qBAAqB,CAACC,SAAoBC;AAEhD,MAAMC,8BAA8B,CAAC,EACnCC,gBAAgB,EAChBC,UAAU,EACgB,GAAM;QAChC,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEA,MAAME,iCAAiE;IACrE,aAAa;AACf;AAEA,SAASC,wCACPC,oBAA8B;IAE9B,OAAO,CAACC;QACN,IAAI,CAACA,SACH,OAAO;YACL,SAAS;YACT,mBAAmB;QACrB;QAGF,MAAMC,gBAAgBD;QACtB,MAAME,eAAeH,qBAClB,GAAG,CAAC,CAACI,MAAQF,aAAa,CAACE,IAAI,EAC/B,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA;QAC1B,MAAMC,mBACJ,AAAwB,YAAxB,OAAOH,eAA4BA,eAAe;QAEpD,OAAO;YACL,SAAS,AAA2B,YAA3B,OAAOF,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;YACjE,mBAAmBK;QACrB;IACF;AACF;AAEA,SAASC,kCACPC,cAAwD;IAExD,MAAMC,oBAAoBD,gBAAgB;IAC1C,IAAIC,mBAAmB,SAAS,UAC9B,OAAOA,kBAAkB,0BAA0B;IAGrD,OAAOV,wCACLU,mBAAmB,wBAAwB;QAAC;KAAoB;AAEpE;AAEO,SAASC,sBACdF,cAAwD;IAExD,MAAMG,4BACJH,gBAAgB,6BAA6Bb;IAC/C,MAAMiB,qBACJJ,gBAAgB,sBAAsBhB;IACxC,MAAMqB,wBAAwBL,gBAAgB,yBAAyB,EAAE;IACzE,MAAMM,6BACJP,kCAAkCC;IACpC,MAAMO,gCACJP,gBAAgB,iCAAiC;IACnD,MAAMQ,4BACJR,gBAAgB,6BAA6B;IAE/C,OAAO;QACLK;QACA,2BAA2B,CAACI;YAC1B,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBnB;YACpB;YACA,MAAMqB,SAASR,0BAA0BO;YACzC,MAAME,eAAeH,MAAM,oBAAoB,IAAI;YAInD,IACEG,eAAe,KACfH,MAAM,UAAU,EAAE,gBAAgBvB,UAClCyB,AAA8B,MAA9BA,OAAO,MAAM,CAAC,WAAW,EAEzB,OAAO;gBACL,QAAQ;oBACN,GAAGA,OAAO,MAAM;oBAChB,aAAa;gBACf;YACF;YAEF,OAAOA;QACT;QACA,oBAAoB,CAACF,QACnBL,mBAAmB;gBACjB,GAAGK,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBnB;YACpB;QACFgB;QACAC;QACAC;IACF;AACF"}

package/dist/es/ai-model/model-adapter/custom-planning-action.mjs ADDED Viewed

@@ -0,0 +1,6 @@
+function getTapLocatedPixelBbox(actions) {
+    for (const action of actions)if ('Tap' === action.type) return action.param.locate.locatedPixelBbox;
+}
+export { getTapLocatedPixelBbox };
+//# sourceMappingURL=custom-planning-action.mjs.map

package/dist/es/ai-model/model-adapter/custom-planning-action.mjs.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"ai-model/model-adapter/custom-planning-action.mjs","sources":["../../../../src/ai-model/model-adapter/custom-planning-action.ts"],"sourcesContent":["import type { LocateResultPoint, PixelBbox, PlanningAction } from '@/types';\n\n// AutoGLM and UI-TARS both produce point coordinates before normalization.\ntype PreNormalizedPlanningLocateParam = {\n prompt: string;\n point: LocateResultPoint;\n};\n\nexport type LocateActionParam = {\n locate: PreNormalizedPlanningLocateParam;\n};\n\nexport type LocatePlanningAction<TType extends string> =\n PlanningAction<LocateActionParam> & {\n type: TType;\n };\n\nexport type ScrollPlanningAction = PlanningAction<\n LocateActionParam & {\n distance: number;\n direction: 'up' | 'down' | 'left' | 'right';\n }\n> & {\n type: 'Scroll';\n};\n\nexport type DragAndDropPlanningAction = PlanningAction<{\n from: PreNormalizedPlanningLocateParam;\n to: PreNormalizedPlanningLocateParam;\n}> & {\n type: 'DragAndDrop';\n};\n\ntype TapPlanningActionWithLocatedPixelBbox = PlanningAction<{\n locate: {\n locatedPixelBbox: PixelBbox;\n };\n}>;\n\nexport function getTapLocatedPixelBbox(\n actions: PlanningAction[],\n): PixelBbox | undefined {\n for (const action of actions) {\n if (action.type !== 'Tap') {\n continue;\n }\n\n return (action as TapPlanningActionWithLocatedPixelBbox).param.locate\n .locatedPixelBbox;\n }\n\n return undefined;\n}\n"],"names":["getTapLocatedPixelBbox","actions","action"],"mappings":"AAuCO,SAASA,uBACdC,OAAyB;IAEzB,KAAK,MAAMC,UAAUD,QACnB,IAAIC,AAAgB,UAAhBA,OAAO,IAAI,EAIf,OAAQA,OAAiD,KAAK,CAAC,MAAM,CAClE,gBAAgB;AAIvB"}

package/dist/es/ai-model/model-adapter/custom-planning-types.mjs ADDED Viewed

File without changes

package/dist/es/ai-model/model-adapter/default-insight-protocol.mjs ADDED Viewed

@@ -0,0 +1,28 @@
+import { extractXMLTag } from "../shared/xml.mjs";
+const dataOutputTagName = 'data-json';
+const createDefaultInsightProtocol = ({ jsonParser })=>({
+        dataOutput: {
+            tagNames: [
+                dataOutputTagName
+            ],
+            rules: '',
+            placeholder: `<${dataOutputTagName}>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</${dataOutputTagName}>`,
+            buildExample: (serializedData)=>`<${dataOutputTagName}>
+${serializedData}
+</${dataOutputTagName}>`,
+            parse: (content)=>{
+                const dataJsonContent = extractXMLTag(content, dataOutputTagName);
+                if (!dataJsonContent) throw new Error('Missing required field: data-json');
+                try {
+                    return jsonParser(dataJsonContent, {
+                        source: 'generic-object'
+                    });
+                } catch (error) {
+                    throw new Error(`Failed to parse data-json: ${error}`);
+                }
+            }
+        }
+    });
+export { createDefaultInsightProtocol };
+//# sourceMappingURL=default-insight-protocol.mjs.map

package/dist/es/ai-model/model-adapter/default-insight-protocol.mjs.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"ai-model/model-adapter/default-insight-protocol.mjs","sources":["../../../../src/ai-model/model-adapter/default-insight-protocol.ts"],"sourcesContent":["import { extractXMLTag } from '../shared/xml';\nimport type { InsightProtocolFactory } from './insight-protocol';\n\nconst dataOutputTagName = 'data-json';\n\nexport const createDefaultInsightProtocol: InsightProtocolFactory = ({\n jsonParser,\n}) => ({\n dataOutput: {\n tagNames: [dataOutputTagName],\n rules: '',\n placeholder: `<${dataOutputTagName}>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</${dataOutputTagName}>`,\n buildExample: (serializedData) => `<${dataOutputTagName}>\n${serializedData}\n</${dataOutputTagName}>`,\n parse: <T>(content: string): T => {\n const dataJsonContent = extractXMLTag(content, dataOutputTagName);\n if (!dataJsonContent) {\n throw new Error('Missing required field: data-json');\n }\n\n try {\n return jsonParser(dataJsonContent, {\n source: 'generic-object',\n }) as T;\n } catch (error) {\n throw new Error(`Failed to parse data-json: ${error}`);\n }\n },\n },\n});\n"],"names":["dataOutputTagName","createDefaultInsightProtocol","jsonParser","serializedData","content","dataJsonContent","extractXMLTag","Error","error"],"mappings":";AAGA,MAAMA,oBAAoB;AAEnB,MAAMC,+BAAuD,CAAC,EACnEC,UAAU,EACX,GAAM;QACL,YAAY;YACV,UAAU;gBAACF;aAAkB;YAC7B,OAAO;YACP,aAAa,CAAC,CAAC,EAAEA,kBAAkB,uLAAuL,EAAEA,kBAAkB,CAAC,CAAC;YAChP,cAAc,CAACG,iBAAmB,CAAC,CAAC,EAAEH,kBAAkB;AAC5D,EAAEG,eAAe;EACf,EAAEH,kBAAkB,CAAC,CAAC;YACpB,OAAO,CAAII;gBACT,MAAMC,kBAAkBC,cAAcF,SAASJ;gBAC/C,IAAI,CAACK,iBACH,MAAM,IAAIE,MAAM;gBAGlB,IAAI;oBACF,OAAOL,WAAWG,iBAAiB;wBACjC,QAAQ;oBACV;gBACF,EAAE,OAAOG,OAAO;oBACd,MAAM,IAAID,MAAM,CAAC,2BAA2B,EAAEC,OAAO;gBACvD;YACF;QACF;IACF"}

package/dist/es/ai-model/model-adapter/default-locate-protocol.mjs ADDED Viewed

@@ -0,0 +1,145 @@
+import { getPreferredLanguage } from "@aiscene/shared/env";
+import { assertJsonObject } from "../shared/json.mjs";
+import { formatLocateExampleValue } from "../shared/model-locate-result/index.mjs";
+const markdownCodeFence = '```';
+const defaultLocateSystemPromptIntroduction = `## Role:
+You are an AI assistant that helps identify UI elements.
+## Objective:
+- Identify elements in screenshots that match the user's description.
+- Provide the coordinates of the element that matches the user's description.`;
+const defaultSearchAreaSystemPromptIntroduction = `## Role:
+You are an AI assistant that helps identify UI elements.
+## Objective:
+- Find a section containing the target element
+- If the description mentions reference elements, also locate sections containing those references`;
+const buildResponseInstructions = (promptSpec)=>{
+    const preferredLanguage = getPreferredLanguage();
+    const resultKey = promptSpec.resultKey;
+    const exampleValueText = formatLocateExampleValue(promptSpec.exampleValues[0]);
+    const resultFieldDescription = `the ${promptSpec.resultNoun} of the element that matches the user's description`;
+    return `## Output Format:
+${markdownCodeFence}json
+{
+  "${resultKey}": ${promptSpec.resultValueSchema},  // ${promptSpec.resultValueDescription}
+  "error": string // optional
+}
+${markdownCodeFence}
+Fields:
+* \`${resultKey}\` is ${resultFieldDescription}
+* \`error\` is an optional error message (if any)
+For example, when an element is found:
+${markdownCodeFence}json
+{
+  "${resultKey}": ${exampleValueText}
+}
+${markdownCodeFence}
+When no element is found:
+${markdownCodeFence}json
+{
+  "${resultKey}": [],
+  "error": "I can see ..., but {some element} is not found. Use ${preferredLanguage}."
+}
+${markdownCodeFence}
+`;
+};
+const buildUserPrompt = (targetElementDescription)=>`Find: ${targetElementDescription}`;
+const buildSearchAreaResponseInstructions = (promptSpec)=>{
+    const preferredLanguage = getPreferredLanguage();
+    const resultKey = promptSpec.resultKey;
+    const exampleValueText = formatLocateExampleValue(promptSpec.exampleValues[0]);
+    const resultJsonProperty = `"${resultKey}": ${promptSpec.resultValueSchema},  // ${promptSpec.resultValueDescription}`;
+    const resultFieldDescription = `${promptSpec.resultNoun} of the section containing the target element`;
+    const referenceFieldDescription = `Optional array of ${promptSpec.resultNounPlural} of reference elements`;
+    return `## Output Format:
+${markdownCodeFence}json
+{
+  ${resultJsonProperty}
+  "references_${resultKey}"?: [
+    ${promptSpec.resultValueSchema},
+    ...
+  ],
+  "error"?: string
+}
+${markdownCodeFence}
+Fields:
+* \`${resultKey}\` - ${resultFieldDescription}
+* \`references_${resultKey}\` - ${referenceFieldDescription}
+* \`error\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.
+Example:
+If the description is "delete button on the second row with title 'Peter'", return:
+${markdownCodeFence}json
+{
+  "${resultKey}": ${exampleValueText},
+  "references_${resultKey}": [${exampleValueText}]
+}
+${markdownCodeFence}
+`;
+};
+const buildSearchAreaUserPrompt = (sectionDescription)=>`Find section containing: ${sectionDescription}`;
+const createParseRawResponse = (jsonParser, source, options)=>(content, promptSpec)=>{
+        const parsedResponse = jsonParser(content, {
+            source
+        });
+        assertJsonObject(parsedResponse);
+        const record = parsedResponse;
+        const target = void 0 !== record[promptSpec.resultKey] ? record[promptSpec.resultKey] : options.acceptBbox2dAlias && 'bbox' === promptSpec.resultKey ? record.bbox_2d : void 0;
+        const error = 'string' == typeof record.error ? record.error : void 0;
+        if (void 0 === target) throw new Error(`Missing required coordinate field "${promptSpec.resultKey}". Expected "${promptSpec.resultKey}": ${promptSpec.resultValueSchema}; use an empty array when no element is found.${error ? ` Model error: ${error}` : ''}`);
+        if (Array.isArray(target) && 0 === target.length) return {
+            kind: 'not-found',
+            ...error ? {
+                error
+            } : {}
+        };
+        if (!options.includeReferences) return {
+            kind: 'located',
+            target,
+            ...error ? {
+                error
+            } : {}
+        };
+        const rawReferences = record[`references_${promptSpec.resultKey}`];
+        const references = null == rawReferences ? void 0 : Array.isArray(rawReferences) ? rawReferences : [
+            rawReferences
+        ];
+        return {
+            kind: 'located',
+            target,
+            ...references?.length ? {
+                references
+            } : {},
+            ...error ? {
+                error
+            } : {}
+        };
+    };
+const createDefaultElementProtocol = ({ jsonParser }, options = {})=>({
+        systemPromptIntroduction: defaultLocateSystemPromptIntroduction,
+        buildResponseInstructions,
+        buildUserPrompt,
+        expectedJsonObjectResponse: true,
+        parseRawResponse: createParseRawResponse(jsonParser, 'locate', {
+            includeReferences: false,
+            acceptBbox2dAlias: options.acceptBbox2dAlias
+        })
+    });
+const createDefaultSearchAreaProtocol = ({ jsonParser }, options = {})=>({
+        systemPromptIntroduction: defaultSearchAreaSystemPromptIntroduction,
+        buildResponseInstructions: buildSearchAreaResponseInstructions,
+        buildUserPrompt: buildSearchAreaUserPrompt,
+        expectedJsonObjectResponse: true,
+        parseRawResponse: createParseRawResponse(jsonParser, 'section-locator', {
+            includeReferences: true,
+            acceptBbox2dAlias: options.acceptBbox2dAlias
+        })
+    });
+export { createDefaultElementProtocol, createDefaultSearchAreaProtocol };
+//# sourceMappingURL=default-locate-protocol.mjs.map

package/dist/es/ai-model/model-adapter/default-locate-protocol.mjs.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"ai-model/model-adapter/default-locate-protocol.mjs","sources":["../../../../src/ai-model/model-adapter/default-locate-protocol.ts"],"sourcesContent":["import { getPreferredLanguage } from '@aiscene/shared/env';\nimport {\n type JsonParser,\n type JsonParserSource,\n assertJsonObject,\n} from '../shared/json';\nimport {\n type LocateResultPromptSpec,\n formatLocateExampleValue,\n} from '../shared/model-locate-result';\nimport type {\n ParsedLocateResponse,\n StandardLocateProtocol,\n StandardLocateProtocolContext,\n} from './locate-protocol';\n\nconst markdownCodeFence = '```';\n\nconst defaultLocateSystemPromptIntroduction = `## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Identify elements in screenshots that match the user's description.\n- Provide the coordinates of the element that matches the user's description.`;\n\nconst defaultSearchAreaSystemPromptIntroduction = `## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Find a section containing the target element\n- If the description mentions reference elements, also locate sections containing those references`;\n\nconst buildResponseInstructions = (promptSpec: LocateResultPromptSpec) => {\n const preferredLanguage = getPreferredLanguage();\n const resultKey = promptSpec.resultKey;\n const exampleValueText = formatLocateExampleValue(\n promptSpec.exampleValues[0],\n );\n const resultFieldDescription = `the ${promptSpec.resultNoun} of the element that matches the user's description`;\n return `## Output Format:\n${markdownCodeFence}json\n{\n \"${resultKey}\": ${promptSpec.resultValueSchema}, // ${promptSpec.resultValueDescription}\n \"error\": string // optional\n}\n${markdownCodeFence}\n\nFields:\n* \\`${resultKey}\\` is ${resultFieldDescription}\n* \\`error\\` is an optional error message (if any)\n\nFor example, when an element is found:\n${markdownCodeFence}json\n{\n \"${resultKey}\": ${exampleValueText}\n}\n${markdownCodeFence}\n\nWhen no element is found:\n${markdownCodeFence}json\n{\n \"${resultKey}\": [],\n \"error\": \"I can see ..., but {some element} is not found. Use ${preferredLanguage}.\"\n}\n${markdownCodeFence}\n`;\n};\n\nconst buildUserPrompt = (targetElementDescription: string) =>\n `Find: ${targetElementDescription}`;\n\nconst buildSearchAreaResponseInstructions = (\n promptSpec: LocateResultPromptSpec,\n) => {\n const preferredLanguage = getPreferredLanguage();\n const resultKey = promptSpec.resultKey;\n const exampleValueText = formatLocateExampleValue(\n promptSpec.exampleValues[0],\n );\n const resultJsonProperty = `\"${resultKey}\": ${promptSpec.resultValueSchema}, // ${promptSpec.resultValueDescription}`;\n const resultFieldDescription = `${promptSpec.resultNoun} of the section containing the target element`;\n const referenceFieldDescription = `Optional array of ${promptSpec.resultNounPlural} of reference elements`;\n return `## Output Format:\n${markdownCodeFence}json\n{\n ${resultJsonProperty}\n \"references_${resultKey}\"?: [\n ${promptSpec.resultValueSchema},\n ...\n ],\n \"error\"?: string\n}\n${markdownCodeFence}\n\nFields:\n* \\`${resultKey}\\` - ${resultFieldDescription}\n* \\`references_${resultKey}\\` - ${referenceFieldDescription}\n* \\`error\\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.\n\nExample:\nIf the description is \"delete button on the second row with title 'Peter'\", return:\n${markdownCodeFence}json\n{\n \"${resultKey}\": ${exampleValueText},\n \"references_${resultKey}\": [${exampleValueText}]\n}\n${markdownCodeFence}\n`;\n};\n\nconst buildSearchAreaUserPrompt = (sectionDescription: string) =>\n `Find section containing: ${sectionDescription}`;\n\ntype ParseRawResponseOptions = {\n includeReferences: boolean;\n acceptBbox2dAlias?: boolean;\n};\n\nconst createParseRawResponse =\n (\n jsonParser: JsonParser,\n source: JsonParserSource,\n options: ParseRawResponseOptions,\n ) =>\n (\n content: string,\n promptSpec: LocateResultPromptSpec,\n ): ParsedLocateResponse => {\n const parsedResponse = jsonParser(content, {\n source,\n });\n assertJsonObject(parsedResponse);\n const record = parsedResponse;\n const target =\n record[promptSpec.resultKey] !== undefined\n ? record[promptSpec.resultKey]\n : options.acceptBbox2dAlias && promptSpec.resultKey === 'bbox'\n ? record.bbox_2d\n : undefined;\n const error = typeof record.error === 'string' ? record.error : undefined;\n if (target === undefined) {\n throw new Error(\n `Missing required coordinate field \"${promptSpec.resultKey}\". Expected \"${promptSpec.resultKey}\": ${promptSpec.resultValueSchema}; use an empty array when no element is found.${error ? ` Model error: ${error}` : ''}`,\n );\n }\n if (Array.isArray(target) && target.length === 0) {\n return { kind: 'not-found', ...(error ? { error } : {}) };\n }\n\n if (!options.includeReferences) {\n return { kind: 'located', target, ...(error ? { error } : {}) };\n }\n\n const rawReferences = record[`references_${promptSpec.resultKey}`];\n const references =\n rawReferences === undefined || rawReferences === null\n ? undefined\n : Array.isArray(rawReferences)\n ? rawReferences\n : [rawReferences];\n return {\n kind: 'located',\n target,\n ...(references?.length ? { references } : {}),\n ...(error ? { error } : {}),\n };\n };\n\ntype DefaultLocateProtocolOptions = {\n acceptBbox2dAlias?: boolean;\n};\n\nexport const createDefaultElementProtocol = (\n { jsonParser }: StandardLocateProtocolContext,\n options: DefaultLocateProtocolOptions = {},\n): StandardLocateProtocol => ({\n systemPromptIntroduction: defaultLocateSystemPromptIntroduction,\n buildResponseInstructions,\n buildUserPrompt,\n expectedJsonObjectResponse: true,\n parseRawResponse: createParseRawResponse(jsonParser, 'locate', {\n includeReferences: false,\n acceptBbox2dAlias: options.acceptBbox2dAlias,\n }),\n});\n\nexport const createDefaultSearchAreaProtocol = (\n { jsonParser }: StandardLocateProtocolContext,\n options: DefaultLocateProtocolOptions = {},\n): StandardLocateProtocol => ({\n systemPromptIntroduction: defaultSearchAreaSystemPromptIntroduction,\n buildResponseInstructions: buildSearchAreaResponseInstructions,\n buildUserPrompt: buildSearchAreaUserPrompt,\n expectedJsonObjectResponse: true,\n parseRawResponse: createParseRawResponse(jsonParser, 'section-locator', {\n includeReferences: true,\n acceptBbox2dAlias: options.acceptBbox2dAlias,\n }),\n});\n"],"names":["markdownCodeFence","defaultLocateSystemPromptIntroduction","defaultSearchAreaSystemPromptIntroduction","buildResponseInstructions","promptSpec","preferredLanguage","getPreferredLanguage","resultKey","exampleValueText","formatLocateExampleValue","resultFieldDescription","buildUserPrompt","targetElementDescription","buildSearchAreaResponseInstructions","resultJsonProperty","referenceFieldDescription","buildSearchAreaUserPrompt","sectionDescription","createParseRawResponse","jsonParser","source","options","content","parsedResponse","assertJsonObject","record","target","undefined","error","Error","Array","rawReferences","references","createDefaultElementProtocol","createDefaultSearchAreaProtocol"],"mappings":";;;AAgBA,MAAMA,oBAAoB;AAE1B,MAAMC,wCAAwC,CAAC;;;;;6EAK8B,CAAC;AAE9E,MAAMC,4CAA4C,CAAC;;;;;kGAK+C,CAAC;AAEnG,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,oBAAoBC;IAC1B,MAAMC,YAAYH,WAAW,SAAS;IACtC,MAAMI,mBAAmBC,yBACvBL,WAAW,aAAa,CAAC,EAAE;IAE7B,MAAMM,yBAAyB,CAAC,IAAI,EAAEN,WAAW,UAAU,CAAC,mDAAmD,CAAC;IAChH,OAAO,CAAC;AACV,EAAEJ,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEH,WAAW,iBAAiB,CAAC,MAAM,EAAEA,WAAW,sBAAsB,CAAC;;;AAG3F,EAAEJ,kBAAkB;;;IAGhB,EAAEO,UAAU,MAAM,EAAEG,uBAAuB;;;;AAI/C,EAAEV,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEC,iBAAiB;;AAErC,EAAER,kBAAkB;;;AAGpB,EAAEA,kBAAkB;;GAEjB,EAAEO,UAAU;gEACiD,EAAEF,kBAAkB;;AAEpF,EAAEL,kBAAkB;AACpB,CAAC;AACD;AAEA,MAAMW,kBAAkB,CAACC,2BACvB,CAAC,MAAM,EAAEA,0BAA0B;AAErC,MAAMC,sCAAsC,CAC1CT;IAEA,MAAMC,oBAAoBC;IAC1B,MAAMC,YAAYH,WAAW,SAAS;IACtC,MAAMI,mBAAmBC,yBACvBL,WAAW,aAAa,CAAC,EAAE;IAE7B,MAAMU,qBAAqB,CAAC,CAAC,EAAEP,UAAU,GAAG,EAAEH,WAAW,iBAAiB,CAAC,MAAM,EAAEA,WAAW,sBAAsB,EAAE;IACtH,MAAMM,yBAAyB,GAAGN,WAAW,UAAU,CAAC,6CAA6C,CAAC;IACtG,MAAMW,4BAA4B,CAAC,kBAAkB,EAAEX,WAAW,gBAAgB,CAAC,sBAAsB,CAAC;IAC1G,OAAO,CAAC;AACV,EAAEJ,kBAAkB;;EAElB,EAAEc,mBAAmB;cACT,EAAEP,UAAU;IACtB,EAAEH,WAAW,iBAAiB,CAAC;;;;;AAKnC,EAAEJ,kBAAkB;;;IAGhB,EAAEO,UAAU,KAAK,EAAEG,uBAAuB;eAC/B,EAAEH,UAAU,KAAK,EAAEQ,0BAA0B;yEACa,EAAEV,kBAAkB;;;;AAI7F,EAAEL,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEC,iBAAiB;cACvB,EAAED,UAAU,IAAI,EAAEC,iBAAiB;;AAEjD,EAAER,kBAAkB;AACpB,CAAC;AACD;AAEA,MAAMgB,4BAA4B,CAACC,qBACjC,CAAC,yBAAyB,EAAEA,oBAAoB;AAOlD,MAAMC,yBACJ,CACEC,YACAC,QACAC,UAEF,CACEC,SACAlB;QAEA,MAAMmB,iBAAiBJ,WAAWG,SAAS;YACzCF;QACF;QACAI,iBAAiBD;QACjB,MAAME,SAASF;QACf,MAAMG,SACJD,AAAiCE,WAAjCF,MAAM,CAACrB,WAAW,SAAS,CAAC,GACxBqB,MAAM,CAACrB,WAAW,SAAS,CAAC,GAC5BiB,QAAQ,iBAAiB,IAAIjB,AAAyB,WAAzBA,WAAW,SAAS,GAC/CqB,OAAO,OAAO,GACdE;QACR,MAAMC,QAAQ,AAAwB,YAAxB,OAAOH,OAAO,KAAK,GAAgBA,OAAO,KAAK,GAAGE;QAChE,IAAID,AAAWC,WAAXD,QACF,MAAM,IAAIG,MACR,CAAC,mCAAmC,EAAEzB,WAAW,SAAS,CAAC,aAAa,EAAEA,WAAW,SAAS,CAAC,GAAG,EAAEA,WAAW,iBAAiB,CAAC,8CAA8C,EAAEwB,QAAQ,CAAC,cAAc,EAAEA,OAAO,GAAG,IAAI;QAG5N,IAAIE,MAAM,OAAO,CAACJ,WAAWA,AAAkB,MAAlBA,OAAO,MAAM,EACxC,OAAO;YAAE,MAAM;YAAa,GAAIE,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAAE;QAG1D,IAAI,CAACP,QAAQ,iBAAiB,EAC5B,OAAO;YAAE,MAAM;YAAWK;YAAQ,GAAIE,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAAE;QAGhE,MAAMG,gBAAgBN,MAAM,CAAC,CAAC,WAAW,EAAErB,WAAW,SAAS,EAAE,CAAC;QAClE,MAAM4B,aACJD,QAAAA,gBACIJ,SACAG,MAAM,OAAO,CAACC,iBACZA,gBACA;YAACA;SAAc;QACvB,OAAO;YACL,MAAM;YACNL;YACA,GAAIM,YAAY,SAAS;gBAAEA;YAAW,IAAI,CAAC,CAAC;YAC5C,GAAIJ,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC5B;IACF;AAMK,MAAMK,+BAA+B,CAC1C,EAAEd,UAAU,EAAiC,EAC7CE,UAAwC,CAAC,CAAC,GACd;QAC5B,0BAA0BpB;QAC1BE;QACAQ;QACA,4BAA4B;QAC5B,kBAAkBO,uBAAuBC,YAAY,UAAU;YAC7D,mBAAmB;YACnB,mBAAmBE,QAAQ,iBAAiB;QAC9C;IACF;AAEO,MAAMa,kCAAkC,CAC7C,EAAEf,UAAU,EAAiC,EAC7CE,UAAwC,CAAC,CAAC,GACd;QAC5B,0BAA0BnB;QAC1B,2BAA2BW;QAC3B,iBAAiBG;QACjB,4BAA4B;QAC5B,kBAAkBE,uBAAuBC,YAAY,mBAAmB;YACtE,mBAAmB;YACnB,mBAAmBE,QAAQ,iBAAiB;QAC9C;IACF"}