@midscene/core 0.8.16 → 0.8.17-beta-20250103010342.0

package/dist/lib/ai-model.js

@@ -1033,10 +1033,10 @@ var require_dist = __commonJS({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/async_caller.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/async_caller.js
 var import_p_retry, import_p_queue, STATUS_NO_RETRY, STATUS_IGNORE, AsyncCaller;
 var init_async_caller = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/async_caller.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/async_caller.js"() {
     "use strict";
     import_p_retry = __toESM(require_p_retry());
     import_p_queue = __toESM(require_dist());
@@ -1158,7 +1158,7 @@ var init_async_caller = __esm({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/messages.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/messages.js
 function isLangChainMessage(message) {
   return typeof (message == null ? void 0 : message._getType) === "function";
 }
@@ -1173,25 +1173,25 @@ function convertLangChainMessageToExample(message) {
   return converted;
 }
 var init_messages = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/messages.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/messages.js"() {
     "use strict";
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/_uuid.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/_uuid.js
 function assertUuid(str) {
   if (!validate_default(str)) {
     throw new Error(`Invalid UUID: ${str}`);
   }
 }
 var init_uuid = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/_uuid.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/_uuid.js"() {
     "use strict";
     init_esm_node();
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/warn.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/warn.js
 function warnOnce(message) {
   if (!warnedMessages[message]) {
     console.warn(message);
@@ -1200,13 +1200,13 @@ function warnOnce(message) {
 }
 var warnedMessages;
 var init_warn = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/warn.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/warn.js"() {
     "use strict";
     warnedMessages = {};
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/client.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/client.js
 async function mergeRuntimeEnvIntoRunCreates(runs) {
   const runtimeEnv = await getRuntimeEnvironment();
   const envVars = getLangChainEnvVarsMetadata();
@@ -1244,7 +1244,7 @@ function trimQuotes(str) {
 }
 var getTracingSamplingRate, isLocalhost, raiseForStatus, handle429, Queue, DEFAULT_BATCH_SIZE_LIMIT_BYTES, Client;
 var init_client = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/client.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/client.js"() {
     "use strict";
     init_esm_node();
     init_async_caller();
@@ -3005,10 +3005,10 @@ var init_client = __esm({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/index.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/index.js
 var __version__;
 var init_dist = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/index.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/index.js"() {
     "use strict";
     init_client();
     init_run_trees();
@@ -3016,7 +3016,7 @@ var init_dist = __esm({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/env.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/env.js
 async function getRuntimeEnvironment() {
   if (runtimeEnvironment === void 0) {
     const env = getEnv();
@@ -3113,7 +3113,7 @@ function getShas() {
 }
 var globalEnv, isBrowser, isWebWorker, isJsDom, isDeno, isNode, getEnv, runtimeEnvironment, cachedCommitSHAs;
 var init_env = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/env.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/env.js"() {
     "use strict";
     init_dist();
     isBrowser = () => typeof window !== "undefined" && typeof window.document !== "undefined";
@@ -3143,10 +3143,10 @@ var init_env = __esm({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/env.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/env.js
 var isTracingEnabled;
 var init_env2 = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/env.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/env.js"() {
     "use strict";
     init_env();
     isTracingEnabled = (tracingEnabled) => {
@@ -3164,7 +3164,7 @@ var init_env2 = __esm({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/run_trees.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/run_trees.js
 function stripNonAlphanumeric(input) {
   return input.replace(/[-:.]/g, "");
 }
@@ -3188,7 +3188,7 @@ function isRunnableConfigLike(x) {
 }
 var Baggage, RunTree;
 var init_run_trees = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/run_trees.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/run_trees.js"() {
     "use strict";
     init_esm_node();
     init_env();
@@ -3612,13 +3612,13 @@ var init_run_trees = __esm({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/singletons/traceable.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/singletons/traceable.js
 function isTraceableFunction(x) {
   return typeof x === "function" && "langsmith:traceable" in x;
 }
 var MockAsyncLocalStorage, AsyncLocalStorageProvider, AsyncLocalStorageProviderSingleton, ROOT;
 var init_traceable = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/singletons/traceable.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/singletons/traceable.js"() {
     "use strict";
     MockAsyncLocalStorage = class {
       getStore() {
@@ -3658,7 +3658,7 @@ var init_traceable = __esm({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/asserts.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/asserts.js
 function isPromiseMethod(x) {
   if (x === "then" || x === "catch" || x === "finally") {
     return true;
@@ -3674,7 +3674,7 @@ function isKVMap(x) {
 }
 var isAsyncIterable, isIteratorLike, GeneratorFunction, isGenerator, isThenable, isReadableStream;
 var init_asserts = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/utils/asserts.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/utils/asserts.js"() {
     "use strict";
     isAsyncIterable = (x) => x != null && typeof x === "object" && // eslint-disable-next-line @typescript-eslint/no-explicit-any
     typeof x[Symbol.asyncIterator] === "function";
@@ -3690,7 +3690,7 @@ var init_asserts = __esm({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/traceable.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/traceable.js
 function traceable(wrappedFunc, config) {
   const { aggregator, argsConfigPath, ...runTreeConfig } = config != null ? config : {};
   const traceableFunc = (...args) => {
@@ -3888,7 +3888,7 @@ function traceable(wrappedFunc, config) {
 }
 var import_node_async_hooks, handleRunInputs, handleRunOutputs, getTracingRunTree, getSerializablePromise, convertSerializableArg;
 var init_traceable2 = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/traceable.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/traceable.js"() {
     "use strict";
     import_node_async_hooks = require("async_hooks");
     init_run_trees();
@@ -4068,7 +4068,7 @@ var init_traceable2 = __esm({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/wrappers/openai.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/wrappers/openai.js
 function _combineChatCompletionChoices(choices) {
   var _a, _b;
   const reversedChoices = choices.slice().reverse();
@@ -4142,7 +4142,7 @@ function _combineChatCompletionChoices(choices) {
 }
 var chatAggregator, textAggregator, wrapOpenAI, _wrapClient, wrapSDK;
 var init_openai = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/wrappers/openai.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/wrappers/openai.js"() {
     "use strict";
     init_traceable2();
     chatAggregator = (chunks) => {
@@ -4254,22 +4254,22 @@ var init_openai = __esm({
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/wrappers/index.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/wrappers/index.js
 var init_wrappers = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/dist/wrappers/index.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/dist/wrappers/index.js"() {
     "use strict";
     init_openai();
   }
 });
 
-// ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/wrappers.js
+// ../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/wrappers.js
 var wrappers_exports = {};
 __export(wrappers_exports, {
   wrapOpenAI: () => wrapOpenAI,
   wrapSDK: () => wrapSDK
 });
 var init_wrappers2 = __esm({
-  "../../node_modules/.pnpm/langsmith@0.1.36_openai@4.57.1_zod@3.23.8_/node_modules/langsmith/wrappers.js"() {
+  "../../node_modules/.pnpm/langsmith@0.1.36_@langchain+core@0.3.26_openai@4.57.1_zod@3.23.8___langchain@0.3.8_@langchain_pnf6d76o2t3a6oyjkhgz2xqm4a/node_modules/langsmith/wrappers.js"() {
     "use strict";
     init_wrappers();
   }
@@ -4313,6 +4313,7 @@ var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
 var MIDSCENE_MODEL_TEXT_ONLY = "MIDSCENE_MODEL_TEXT_ONLY";
 var MIDSCENE_CACHE = "MIDSCENE_CACHE";
 var MATCH_BY_POSITION = "MATCH_BY_POSITION";
+var MIDSCENE_API_TYPE = "MIDSCENE_API_TYPE";
 var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
 var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
 var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
@@ -4375,8 +4376,7 @@ var getAIConfigInJson = (configKey) => {
 
 // src/ai-model/common.ts
 var import_node_assert = __toESM(require("assert"));
-async function callAiFn(options) {
-  const { msgs, AIActionType: AIActionTypeValue } = options;
+async function callAiFn(msgs, AIActionTypeValue) {
   (0, import_node_assert.default)(
     checkAIConfig(),
     "Cannot find config for AI model service. You should set it before using. https://midscenejs.com/model-provider.html"
@@ -4387,19 +4387,9 @@ async function callAiFn(options) {
   );
   return { content, usage };
 }
-function transformUserMessages(msgs) {
-  const textOnly = Boolean(getAIConfig(MIDSCENE_MODEL_TEXT_ONLY));
-  if (!textOnly)
-    return msgs;
-  return msgs.reduce((res, msg) => {
-    if (msg.type === "text") {
-      res += msg.text;
-    }
-    return res;
-  }, "");
-}
 
 // src/ai-model/prompt/element_inspector.ts
+var import_prompts = require("@langchain/core/prompts");
 function systemPromptToFindElement() {
   if (getAIConfig(MATCH_BY_POSITION)) {
     return systemPromptToFindElementPosition();
@@ -4542,36 +4532,31 @@ Output Example:
 }
 function systemPromptToFindElementPosition() {
   return `
-    ## Role:
-    You are an expert in software page image (2D) and page element text analysis.
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
 
-    ## Objective:
-    Based on screenshots and descriptions, find specific coordinates
+## Output Format
+\`\`\`
+Action_Summary: ...
+Action: ...
+\`\`\`
 
-    ## Output Format:
+## Action Space
+click(start_box='[x1, y1, x2, y2]')
+long_press(start_box='[x1, y1, x2, y2]', time='')
+type(content='')
+scroll(direction='down or up or right or left')
+open_app(app_name='')
+navigate_back()
+navigate_home()
+WAIT()
+finished() # Submit the task regardless of whether it succeeds or fails.
 
-    Please return the result in JSON format as follows:
+## Note
+- Use Chinese in \`Action_Summary\` part.
 
-    \`\`\`json
-    {
-      "elements": [
-        {
-          // Describe the reason for finding this element, replace with actual value in practice
-          "reason": "Reason for finding element 4: It is located in the upper right corner, is an image type, and according to the screenshot, it is a shopping cart icon button",
-          // If the target element includes text information, extract the text information; if it does not, do not extract it
-          "text": "",
-          // position of this element
-          "position": { x: number, y: number }
-        }
-    ],
-    "errors": []// Return an error if there is no target element on the picture
-    }
-    \`\`\`
+## User Instruction
   `;
 }
-function multiDescription(multi) {
-  return multi ? "multiple elements matching the description (two or more)" : "The element closest to the description (only one)";
-}
 var findElementSchema = {
   type: "json_schema",
   json_schema: {
@@ -4616,6 +4601,23 @@ var findElementSchema = {
     }
   }
 };
+var findElementPrompt = new import_prompts.PromptTemplate({
+  template: `
+    Here is the item user want to find. Just go ahead:
+    =====================================
+    {{
+      "description": "{targetElementDescription}",
+      "multi": {multi}
+    }}
+    =====================================
+
+    pageDescription: {pageDescription}
+  `,
+  inputVariables: ["pageDescription", "targetElementDescription", "multi"]
+});
+
+// src/ai-model/prompt/planning.ts
+var import_prompts3 = require("@langchain/core/prompts");
 
 // src/ai-model/prompt/util.ts
 var import_node_assert2 = __toESM(require("assert"));
@@ -4624,6 +4626,7 @@ var import_node_assert2 = __toESM(require("assert"));
 var import_img = require("@midscene/shared/img");
 
 // src/ai-model/prompt/util.ts
+var import_prompts2 = require("@langchain/core/prompts");
 var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
 var contextFormatIntro = `
 The user will give you a screenshot and some of the texts on it. There may be some none-English characters (like Chinese) on it, indicating it's an non-English app. If some text is shown on screenshot but not introduced by the JSON description, use the information you see on screenshot.`;
@@ -4652,6 +4655,22 @@ Return in the following JSON format:
 }
 `;
 }
+var extractDataPrompt = new import_prompts2.PromptTemplate({
+  template: `
+pageDescription: {pageDescription}
+
+Use your extract_data_from_UI skill to find the following data, placing it in the \`data\` field
+DATA_DEMAND start:
+=====================================
+{dataKeys}
+
+{dataQuery}
+
+=====================================
+DATA_DEMAND ends.
+  `,
+  inputVariables: ["pageDescription", "dataKeys", "dataQuery"]
+});
 function systemPromptToAssert() {
   return `
 ${characteristic}
@@ -4700,7 +4719,7 @@ function truncateText(text, maxLength = 100) {
   }
   return "";
 }
-function elementByPosition(elementsInfo, position) {
+function elementByPositionWithElementInfo(elementsInfo, position) {
   (0, import_node_assert2.default)(typeof position !== "undefined", "position is required for query");
   const item = elementsInfo.find((item2) => {
     return item2.rect.left <= position.x && position.x <= item2.rect.left + item2.rect.width && item2.rect.top <= position.y && position.y <= item2.rect.top + item2.rect.height;
@@ -4754,23 +4773,26 @@ async function describeUserPage(context, opt) {
     const { id, ...rest } = item;
     return `id=${id}: ${JSON.stringify(rest)}`;
   }).join("\n\n");
+  const pageJSONDescription = getAIConfig(MATCH_BY_POSITION) ? "" : `Some of the elements are marked with a rectangle in the screenshot, some are not. 
+ Json description of all the page elements:
+${contentList}`;
+  const sizeDescription = describeSize({ width, height });
   return {
     description: `
-The size of the page: ${describeSize({ width, height })}
-Some of the elements are marked with a rectangle in the screenshot, some are not.
+  The size of the page: ${sizeDescription} 
 
-${// if match by id, use the description of the element
-    getAIConfig(MATCH_BY_POSITION) ? "" : `Json description of all the page elements:
-${contentList}`}
+  ${pageJSONDescription}
 `,
     elementById(id) {
       (0, import_node_assert2.default)(typeof id !== "undefined", "id is required for query");
       const item = idElementMap[`${id}`];
       return item;
     },
-    elementByPosition(position) {
-      return elementByPosition(elementsInfo, position);
-    }
+    elementByPosition(position, size) {
+      console.log("elementByPosition", { position, size });
+      return elementByPositionWithElementInfo(elementsInfo, position);
+    },
+    size: { width, height }
   };
 }
 function cropFieldInformation(elementsInfo, truncateTextLength, filterNonTextContent = false) {
@@ -4824,19 +4846,35 @@ function cropFieldInformation(elementsInfo, truncateTextLength, filterNonTextCon
 // src/ai-model/prompt/planning.ts
 var quickAnswerFormat = () => {
   const matchByPosition = getAIConfig(MATCH_BY_POSITION);
-  const description = `
-  ${matchByPosition ? `"position": { x: number; y: number } // Represents the position of the element; replace with actual values in practice (ensure it reflects the element's position)` : '"id": string // Represents the ID of the element; replace with actual values in practice'}
-  `;
-  const format = matchByPosition ? '"position": { x: number; y: number }' : '"id": string';
-  const sample = matchByPosition ? '{"position": { x: 100, y: 200 }}' : '{"id": "c81c4e9a33"}';
+  const locationFormat = {
+    position: {
+      description: `"position": { x: number; y: number } // Represents the position of the element; replace with actual values in practice (ensure it reflects the element's position)`,
+      format: '"position": { x: number; y: number }',
+      sample: '{"prompt": "the search bar"}',
+      locateParam: `{
+        "prompt"?: string // the description of the element to find. It can only be omitted when locate is null.
+      } | null // If it's not on the page, the LocateParam should be null`
+    },
+    id: {
+      description: '"id": string // Represents the ID of the element; replace with actual values in practice',
+      format: '"id": string',
+      sample: `{"id": "c81c4e9a33", "prompt": "the search bar"}`,
+      locateParam: `{
+        "id": string, // the id of the element found. It should either be the id marked with a rectangle in the screenshot or the id described in the description.
+        "prompt"?: string // the description of the element to find. It can only be omitted when locate is null.
+      } | null // If it's not on the page, the LocateParam should be null`
+    }
+  };
+  const type = matchByPosition ? "position" : "id";
+  const format = locationFormat[type];
   return {
-    description,
-    format,
-    sample
+    description: format.description,
+    format: format.format,
+    sample: format.sample,
+    locateParam: format.locateParam
   };
 };
-function systemPromptToTaskPlanning() {
-  return `
+var systemTemplate = `
 ## Role
 
 You are a versatile professional in software UI automation. Your outstanding contributions will impact the user experience of billions of users.
@@ -4870,76 +4908,70 @@ You are a versatile professional in software UI automation. Your outstanding con
 
 The \`locate\` param is commonly used in the \`param\` field of the action, means to locate the target element to perform the action, it follows the following scheme:
 
-type LocateParam = {
-  "id": string, // the id of the element found. It should either be the id marked with a rectangle in the screenshot or the id described in the description.
-  prompt?: string // the description of the element to find. It can only be omitted when locate is null.
-} | null // If it's not on the page, the LocateParam should be null
+type LocateParam = {locateParam}
 
 ### Supported actions
 
 Each action has a \`type\` and corresponding \`param\`. To be detailed:
 - type: 'Tap', tap the located element
-  * { locate: LocateParam, param: null }
+  * {{ locate: {sample}, param: null }}
 - type: 'Hover', move mouse over to the located element
-  * { locate: LocateParam, param: null }
+  * {{ locate: LocateParam, param: null }}
 - type: 'Input', replace the value in the input field
-  * { locate: LocateParam, param: { value: string } }
+  * {{ locate: LocateParam, param: {{ value: string }} }}
   * \`value\` is the final required input value based on the existing input. No matter what modifications are required, just provide the final value to replace the existing input value. 
 - type: 'KeyboardPress', press a key
-  * { param: { value: string } }
+  * {{ param: {{ value: string }} }}
 - type: 'Scroll', scroll up or down.
-  * { 
+  * {{ 
       locate: LocateParam | null, 
-      param: { 
+      param: {{ 
         direction: 'down'(default) | 'up' | 'right' | 'left', 
         scrollType: 'once' (default) | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft', 
         distance: null | number 
-      } 
-    }
+      }} 
+    }}
     * To scroll some specific element, put the element at the center of the region in the \`locate\` field. If it's a page scroll, put \`null\` in the \`locate\` field. 
     * \`param\` is required in this action. If some fields are not specified, use direction \`down\`, \`once\` scroll type, and \`null\` distance.
 - type: 'FalsyConditionStatement'
-  * { param: null }
+  * {{ param: null }}
   * use this action when the instruction is an "if" statement and the condition is falsy.
 - type: 'Sleep'
-  * { param: { timeMs: number } }
+  * {{ param: {{ timeMs: number }} }}
 
 ## How to compose the \`taskWillBeAccomplished\` and \`furtherPlan\` fields ?
 
 \`taskWillBeAccomplished\` is a boolean field, means whether the task will be accomplished after all the actions.
 
-\`furtherPlan\` is used when the task cannot be accomplished. It follows the scheme { whatHaveDone: string, whatToDoNext: string }:
+\`furtherPlan\` is used when the task cannot be accomplished. It follows the scheme {{ whatHaveDone: string, whatToDoNext: string }}:
 - \`whatHaveDone\`: a string, describe what have been done after the previous actions.
 - \`whatToDoNext\`: a string, describe what should be done next after the previous actions has finished. It should be a concise and clear description of the actions to be performed. Make sure you don't lose any necessary steps user asked.
-
+`;
+var outputTemplate = `
 ## Output JSON Format:
 
 The JSON format is as follows:
 
-{
+{{
   "actions": [
-    {
+    {{
       "thought": "Reasons for generating this task, and why this task is feasible on this page",
       "type": "Tap",
       "param": null,
-      "locate": {
-        {"id": "c81c4e9a33"},
-        prompt: "the search bar"
-      } | null,
-    },
+      "locate": {sample} | null,
+    }},
     // ... more actions
   ],
   "taskWillBeAccomplished": boolean,
-  "furtherPlan": { "whatHaveDone": string, "whatToDoNext": string } | null,
+  "furtherPlan": {{ "whatHaveDone": string, "whatToDoNext": string }} | null,
   "error"?: string
-}
-
-## Example #1 : How to decompose a task
+}}
+Here is an example of how to decompose a task:
 
 When a user says 'Click the language switch button, wait 1s, click "English"', the user will give you the description like this:
 
 ====================
-${samplePageDescription}
+{pageDescription}
 ====================
 
 By viewing the page screenshot and description, you should consider this and output the JSON:
@@ -4949,110 +4981,119 @@ By viewing the page screenshot and description, you should consider this and out
 * The "English" option button is not shown in the screenshot now, it means it may only show after the previous actions are finished. So the last action will have a \`null\` value in the \`locate\` field. 
 * The task cannot be accomplished (because we cannot see the "English" option now), so a \`furtherPlan\` field is needed.
 
-{
+{{
   "actions":[
-    {
+    {{
+      "type": "Tap", 
       "thought": "Click the language switch button to open the language options.",
-      "type": "Tap",
       "param": null,
-      "locate": {
-        ${quickAnswerFormat().sample},
-        "prompt": "the language switch button with the text '中文'"
-      }
-    },
-    {
-      "thought": "Wait for 1 second to ensure the language options are displayed.",
+      "locate": {sample},
+    }},
+    {{
       "type": "Sleep",
-      "param": { "timeMs": 1000 },
-    },
-    {
-      "thought": "Locate the 'English' option in the language menu.", 
+      "thought": "Wait for 1 second to ensure the language options are displayed.",
+      "param": {{ "timeMs": 1000 }},
+    }},
+    {{
       "type": "Tap",
-      "param": null,
+      "thought": "Locate the 'English' option in the language menu.",
+      "param": null, 
       "locate": null
-    },
+    }},
   ],
   "error": null,
   "taskWillBeAccomplished": false,
-  "furtherPlan": { 
-    "whatToDoNext": "find the 'English' option and click on it", 
-    "whatHaveDone": "Click the language switch button and wait 1s" 
-  }
-}
+  "furtherPlan": {{
+    "whatToDoNext": "find the 'English' option and click on it",
+    "whatHaveDone": "Click the language switch button and wait 1s"
+  }}
+}}
 
-## Example #2 : Tolerate the error situation only when the instruction is an "if" statement
+Here is another example of how to tolerate error situations only when the instruction is an "if" statement:
 
 If the user says "If there is a popup, close it", you should consider this and output the JSON:
 
 * By viewing the page screenshot and description, you cannot find the popup, so the condition is falsy.
 * The instruction itself is an "if" statement, it means the user can tolerate this situation, so you should leave a \`FalsyConditionStatement\` action.
 
-{
-  "actions": [{
-      "thought": "There is no popup on the page",
+{{
+  "actions": [{{
       "type": "FalsyConditionStatement",
+      "thought": "There is no popup on the page",
       "param": null
-    }
+    }}
   ],
   "taskWillBeAccomplished": true,
   "furtherPlan": null
-}
+}}
 
 For contrast, if the user says "Close the popup" in this situation, you should consider this and output the JSON:
 
-{
+{{
   "actions": [],
   "error": "The instruction and page context are irrelevant, there is no popup on the page",
   "taskWillBeAccomplished": true,
   "furtherPlan": null
-}
+}}
 
-## Example #3 : When task is accomplished, don't plan more actions
+Here is an example of when task is accomplished, don't plan more actions:
 
 When the user ask to "Wait 4s", you should consider this:
 
-{
+{{
   "actions": [
-    {
-      "thought": "Wait for 4 seconds",
+    {{
       "type": "Sleep",
-      "param": { "timeMs": 4000 },
-    },
+      "thought": "Wait for 4 seconds",
+      "param": {{ "timeMs": 4000 }},
+    }},
   ],
   "taskWillBeAccomplished": true,
   "furtherPlan": null // All steps have been included in the actions, so no further plan is needed
-}
+}}
 
-## Bad case #1 : Missing \`prompt\` in the 'Locate' field; Missing \`furtherPlan\` field when the task won't be accomplished
+Here is an example of what NOT to do:
 
 Wrong output:
 
-{
+{{
   "actions":[
-    {
-      "thought": "Click the language switch button to open the language options.",
+    {{
       "type": "Tap",
+      "thought": "Click the language switch button to open the language options.",
       "param": null,
-      "locate": {
-        ${quickAnswerFormat().sample}, // WRONG:prompt is missing
-      }
-    },
-    {
+      "locate": {{
+        {sample}, // WRONG:prompt is missing
+      }}
+    }},
+    {{
+      "type": "Tap", 
       "thought": "Click the English option",
-      "type": "Tap",
       "param": null,
       "locate": null, // This means the 'English' option is not shown in the screenshot, the task cannot be accomplished
-    }
+    }}
   ],
   "taskWillBeAccomplished": false,
   // WRONG: should not be null
   "furtherPlan": null,
-}
+}}
 
-Reason: 
+Reason:
 * The \`prompt\` is missing in the first 'Locate' action
 * Since the option button is not shown in the screenshot, the task cannot be accomplished, so a \`furtherPlan\` field is needed.
 `;
+async function systemPromptToTaskPlanning() {
+  const promptTemplate = new import_prompts3.PromptTemplate({
+    template: `${systemTemplate}
+
+${outputTemplate}`,
+    inputVariables: ["pageDescription", "sample", "locateParam"]
+  });
+  return await promptTemplate.format({
+    pageDescription: samplePageDescription,
+    sample: quickAnswerFormat().sample,
+    locateParam: quickAnswerFormat().locateParam
+  });
 }
 var planSchema = {
   type: "json_schema",
@@ -5136,6 +5177,34 @@ var planSchema = {
     }
   }
 };
+var taskBackgroundContext = async (originalPrompt, whatHaveDone) => {
+  if (!originalPrompt || !whatHaveDone) {
+    return "";
+  }
+  return `
+    For your information, this is a task that some important person handed to you. Here is the original task description and what have been done after the previous actions:
+    =====================================
+    Original task description:
+    ${originalPrompt}
+    =====================================
+    What have been done:
+    ${whatHaveDone}
+    =====================================
+`;
+};
+var automationUserPrompt = new import_prompts3.PromptTemplate({
+  template: `
+    Here is the instruction:
+    =====================================
+    {userPrompt}
+    =====================================
+
+    {taskBackgroundContext}
+
+    pageDescription: {pageDescription}
+  `,
+  inputVariables: ["pageDescription", "userPrompt", "taskBackgroundContext"]
+});
 
 // src/ai-model/openai/index.ts
 function checkAIConfig(preferVendor) {
@@ -5158,7 +5227,9 @@ function getModelName() {
   }
   return modelName;
 }
-async function createChatClient() {
+async function createChatClient({
+  AIActionTypeValue
+}) {
   let openai;
   const extraConfig = getAIConfigInJson(MIDSCENE_OPENAI_INIT_CONFIG_JSON);
   const socksProxy = getAIConfig(MIDSCENE_OPENAI_SOCKS_PROXY);
@@ -5209,6 +5280,10 @@ async function createChatClient() {
       apiKey: getAIConfig(OPENAI_API_KEY),
       httpAgent: socksAgent,
       ...extraConfig,
+      defaultHeaders: {
+        ...(extraConfig == null ? void 0 : extraConfig.defaultHeaders) || {},
+        [MIDSCENE_API_TYPE]: AIActionTypeValue.toString()
+      },
       dangerouslyAllowBrowser: true
     });
   }
@@ -5241,8 +5316,10 @@ async function createChatClient() {
   }
   throw new Error("Openai SDK or Anthropic SDK is not initialized");
 }
-async function call(messages, responseFormat) {
-  const { completion, style } = await createChatClient();
+async function call(messages, AIActionTypeValue, responseFormat) {
+  const { completion, style } = await createChatClient({
+    AIActionTypeValue
+  });
   const shouldPrintTiming = typeof getAIConfig(MIDSCENE_DEBUG_AI_PROFILE) === "string";
   const maxTokens = getAIConfig(OPENAI_MAX_TOKENS);
   const startTime = Date.now();
@@ -5325,47 +5402,47 @@ async function callToGetJSONObject(messages, AIActionTypeValue) {
       responseFormat = { type: "json_object" /* JSON */ };
     }
   }
-  const safeJsonParse = (input) => {
-    try {
-      return JSON.parse(input);
-    } catch (e) {
-      return null;
-    }
-  };
-  const response = await call(messages, responseFormat);
+  const response = await call(messages, AIActionTypeValue, responseFormat);
   (0, import_node_assert3.default)(response, "empty response");
-  let jsonContent = safeJsonParse(response.content);
-  if (jsonContent)
-    return { content: jsonContent, usage: response.usage };
-  const cleanJsonString = extractJSONFromCodeBlock(response.content);
-  try {
-    jsonContent = JSON.parse(cleanJsonString);
-  } catch (e) {
-  }
-  if (jsonContent)
-    return { content: jsonContent, usage: response.usage };
+  const jsonContent = safeParseJson(response.content);
+  return { content: jsonContent, usage: response.usage };
+}
+function extractJSONFromCodeBlock(response) {
   try {
-    jsonContent = import_dirty_json.default.parse(cleanJsonString);
+    const jsonMatch = response.match(/^\s*(\{[\s\S]*\})\s*$/);
+    if (jsonMatch) {
+      return jsonMatch[1];
+    }
+    const codeBlockMatch = response.match(
+      /```(?:json)?\s*(\{[\s\S]*?\})\s*```/
+    );
+    if (codeBlockMatch) {
+      return codeBlockMatch[1];
+    }
+    const jsonLikeMatch = response.match(/\{[\s\S]*\}/);
+    if (jsonLikeMatch) {
+      return jsonLikeMatch[0];
+    }
   } catch (e) {
   }
-  if (jsonContent)
-    return { content: jsonContent, usage: response.usage };
-  throw Error(`failed to parse json response: ${response.content}`);
+  return response;
 }
-function extractJSONFromCodeBlock(response) {
-  const jsonMatch = response.match(/^\s*(\{[\s\S]*\})\s*$/);
-  if (jsonMatch) {
-    return jsonMatch[1];
+function safeParseJson(input) {
+  var _a;
+  const cleanJsonString = extractJSONFromCodeBlock(input);
+  if (cleanJsonString.match(/\((\d+),(\d+)\)/)) {
+    return (_a = cleanJsonString.match(/\((\d+),(\d+)\)/)) == null ? void 0 : _a.slice(1).map(Number);
   }
-  const codeBlockMatch = response.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/);
-  if (codeBlockMatch) {
-    return codeBlockMatch[1];
+  try {
+    return JSON.parse(cleanJsonString);
+  } catch (e) {
   }
-  const jsonLikeMatch = response.match(/\{[\s\S]*\}/);
-  if (jsonLikeMatch) {
-    return jsonLikeMatch[0];
+  try {
+    return import_dirty_json.default.parse(cleanJsonString);
+  } catch (e) {
+    console.log("e:", e);
   }
-  return response;
+  throw Error(`failed to parse json response: ${input}`);
 }
 
 // src/ai-model/inspect.ts
@@ -5374,76 +5451,82 @@ var liteContextConfig = {
   filterNonTextContent: true,
   truncateTextLength: 200
 };
-function transformElementPositionToId(aiResult, elementsInfo) {
+function transformToAbsoluteCoords(relativePosition, size) {
+  return {
+    x: Number((relativePosition.x / 1e3 * size.width).toFixed(3)),
+    y: Number((relativePosition.y / 1e3 * size.height).toFixed(3))
+  };
+}
+async function transformElementPositionToId(aiResult, elementsInfo, size, screenshotBase64) {
+  if (Array.isArray(aiResult)) {
+    const relativePosition = aiResult;
+    const absolutePosition = transformToAbsoluteCoords(
+      {
+        x: relativePosition[0],
+        y: relativePosition[1]
+      },
+      size
+    );
+    const element = elementByPositionWithElementInfo(
+      elementsInfo,
+      absolutePosition
+    );
+    (0, import_node_assert4.default)(
+      element,
+      `inspect: no id found with position: ${JSON.stringify({ absolutePosition })}`
+    );
+    return {
+      errors: [],
+      elements: [
+        {
+          id: element.id
+        }
+      ]
+    };
+  }
   return {
     errors: aiResult.errors,
-    elements: aiResult.elements.map((item) => {
-      var _a;
-      if ("id" in item) {
-        return item;
-      }
-      const { position } = item;
-      const id = (_a = elementByPosition(elementsInfo, position)) == null ? void 0 : _a.id;
-      (0, import_node_assert4.default)(
-        id,
-        `inspect: no id found with position: ${JSON.stringify({ position })}`
-      );
-      return {
-        ...item,
-        id
-      };
-    })
+    elements: aiResult.elements
   };
 }
+function getQuickAnswer(quickAnswer, elementById) {
+  if (!quickAnswer) {
+    return void 0;
+  }
+  if ("id" in quickAnswer && quickAnswer.id && elementById(quickAnswer.id)) {
+    return {
+      parseResult: {
+        elements: [quickAnswer],
+        errors: []
+      },
+      rawResponse: quickAnswer,
+      elementById
+    };
+  }
+}
 async function AiInspectElement(options) {
   const { context, multi, targetElementDescription, callAI } = options;
   const { screenshotBase64, screenshotBase64WithElementMarker } = context;
-  const { description, elementById, elementByPosition: elementByPosition2 } = await describeUserPage(context);
-  if (options.quickAnswer) {
-    if ("id" in options.quickAnswer) {
-      if (elementById(options.quickAnswer.id)) {
-        return {
-          parseResult: {
-            elements: [options.quickAnswer],
-            errors: []
-          },
-          elementById
-        };
-      }
-      if (!targetElementDescription) {
-        return {
-          parseResult: {
-            elements: [],
-            errors: [
-              `inspect: cannot find the target by id: ${options.quickAnswer.id}, and no target element description is provided`
-            ]
-          },
-          elementById
-        };
-      }
-    }
-    if ("position" in options.quickAnswer && elementByPosition2(options.quickAnswer.position)) {
-      return {
-        parseResult: transformElementPositionToId(
-          {
-            elements: [options.quickAnswer]
-          },
-          context.content
-        ),
-        elementById
-      };
-    }
+  const { description, elementById, elementByPosition, size } = await describeUserPage(context);
+  const quickAnswer = getQuickAnswer(options.quickAnswer, elementById);
+  if (quickAnswer) {
+    return quickAnswer;
   }
   (0, import_node_assert4.default)(
     targetElementDescription,
     "cannot find the target element description"
   );
+  const userInstructionPrompt = await findElementPrompt.format({
+    pageDescription: description,
+    targetElementDescription,
+    multi
+  });
   const systemPrompt = systemPromptToFindElement();
   const msgs = [
     { role: "system", content: systemPrompt },
     {
       role: "user",
-      content: transformUserMessages([
+      content: [
         {
           type: "image_url",
           image_url: {
@@ -5452,46 +5535,23 @@ async function AiInspectElement(options) {
         },
         {
           type: "text",
-          text: `
-pageDescription: 
-
-${description}
-
-Here is the item user want to find. Just go ahead:
-=====================================
-${JSON.stringify({
-            description: targetElementDescription,
-            multi: multiDescription(multi)
-          })}
-=====================================`
+          text: userInstructionPrompt
         }
-      ])
+      ]
     }
   ];
-  if (callAI) {
-    const res = await callAI({
-      msgs,
-      AIActionType: 1 /* INSPECT_ELEMENT */
-    });
-    return {
-      parseResult: transformElementPositionToId(res.content, context.content),
-      rawResponse: res,
-      elementById,
-      usage: res.usage
-    };
-  }
-  const inspectElement = await callAiFn({
-    msgs,
-    AIActionType: 1 /* INSPECT_ELEMENT */
-  });
+  const callAIFn = callAI || callToGetJSONObject;
+  const res = await callAIFn(msgs, 1 /* INSPECT_ELEMENT */);
   return {
-    parseResult: transformElementPositionToId(
-      inspectElement.content,
-      context.content
+    parseResult: await transformElementPositionToId(
+      res.content,
+      context.content,
+      size,
+      screenshotBase64
     ),
-    rawResponse: inspectElement,
+    rawResponse: res.content,
     elementById,
-    usage: inspectElement.usage
+    usage: res.usage
   };
 }
 async function AiExtractElementInfo(options) {
@@ -5502,11 +5562,25 @@ async function AiExtractElementInfo(options) {
     context,
     liteContextConfig
   );
+  let dataKeys = "";
+  let dataQueryText = "";
+  if (typeof dataQuery === "string") {
+    dataKeys = "";
+    dataQueryText = dataQuery;
+  } else {
+    dataKeys = `return in key-value style object, keys are ${Object.keys(dataQuery).join(",")}`;
+    dataQueryText = JSON.stringify(dataQuery, null, 2);
+  }
+  const extractDataPromptText = await extractDataPrompt.format({
+    pageDescription: description,
+    dataKeys,
+    dataQuery: dataQueryText
+  });
   const msgs = [
     { role: "system", content: systemPrompt },
     {
       role: "user",
-      content: transformUserMessages([
+      content: [
         {
           type: "image_url",
           image_url: {
@@ -5515,25 +5589,15 @@ async function AiExtractElementInfo(options) {
         },
         {
           type: "text",
-          text: `
-pageDescription: ${description}
-
-Use your extract_data_from_UI skill to find the following data, placing it in the \`data\` field
-DATA_DEMAND start:
-=====================================
-${typeof dataQuery === "object" ? `return in key-value style object, keys are ${Object.keys(dataQuery).join(",")}` : ""}
-${typeof dataQuery === "string" ? dataQuery : JSON.stringify(dataQuery, null, 2)}
-=====================================
-DATA_DEMAND ends.
-          `
+          text: extractDataPromptText
         }
-      ])
+      ]
     }
   ];
-  const result = await callAiFn({
+  const result = await callAiFn(
     msgs,
-    AIActionType: 2 /* EXTRACT_DATA */
-  });
+    2 /* EXTRACT_DATA */
+  );
   return {
     parseResult: result.content,
     elementById,
@@ -5550,7 +5614,7 @@ async function AiAssert(options) {
     { role: "system", content: systemPrompt },
     {
       role: "user",
-      content: transformUserMessages([
+      content: [
         {
           type: "image_url",
           image_url: {
@@ -5569,43 +5633,39 @@ async function AiAssert(options) {
     =====================================
   `
         }
-      ])
+      ]
     }
   ];
-  const { content: assertResult, usage } = await callAiFn({
+  const { content: assertResult, usage } = await callAiFn(
     msgs,
-    AIActionType: 0 /* ASSERT */
-  });
+    0 /* ASSERT */
+  );
   return {
     content: assertResult,
     usage
   };
 }
 
-// src/ai-model/automation/index.ts
+// src/ai-model/automation.ts
 var import_node_assert5 = __toESM(require("assert"));
 async function plan(userPrompt, opts) {
   const { callAI, context } = opts || {};
   const { screenshotBase64, screenshotBase64WithElementMarker } = context;
-  const { description: pageDescription, elementByPosition: elementByPosition2 } = await describeUserPage(context);
-  const systemPrompt = systemPromptToTaskPlanning();
-  let taskBackgroundContext = "";
-  if (opts.originalPrompt && opts.whatHaveDone) {
-    taskBackgroundContext = `For your information, this is a task that some important person handed to you. Here is the original task description and what have been done after the previous actions:
-=====================================
-Original task description:
-${opts.originalPrompt}
-=====================================
-What have been done:
-${opts.whatHaveDone}
-=====================================
-`;
-  }
+  const { description: pageDescription, elementByPosition } = await describeUserPage(context);
+  const systemPrompt = await systemPromptToTaskPlanning();
+  const userInstructionPrompt = await automationUserPrompt.format({
+    pageDescription,
+    userPrompt,
+    taskBackgroundContext: taskBackgroundContext(
+      opts.originalPrompt,
+      opts.whatHaveDone
+    )
+  });
   const msgs = [
     { role: "system", content: systemPrompt },
     {
       role: "user",
-      content: transformUserMessages([
+      content: [
         {
           type: "image_url",
           image_url: {
@@ -5615,28 +5675,13 @@ ${opts.whatHaveDone}
         },
         {
           type: "text",
-          text: `
-pageDescription:
- 
-${pageDescription}
-
-
-Here is the instruction:
-=====================================
-${userPrompt}
-=====================================
-
-${taskBackgroundContext}
-`.trim()
+          text: userInstructionPrompt
         }
-      ])
+      ]
     }
   ];
-  const call2 = callAI || callAiFn;
-  const { content, usage } = await call2({
-    msgs,
-    AIActionType: 3 /* PLAN */
-  });
+  const call3 = callAI || callAiFn;
+  const { content, usage } = await call3(msgs, 3 /* PLAN */);
   const planFromAI = content;
   const actions = (planFromAI == null ? void 0 : planFromAI.actions) || [];
   (0, import_node_assert5.default)(planFromAI, "can't get plans from AI");