misoai-web 1.0.3 → 1.0.5

package/dist/es/agent.js

@@ -0,0 +1,2676 @@
+// src/common/agent.ts
+import { Insight } from "misoai-core";
+import yaml4 from "js-yaml";
+
+// src/yaml/player.ts
+import { existsSync, mkdirSync, writeFileSync } from "fs";
+import { dirname, join, resolve } from "path";
+import { assert, ifInBrowser } from "misoai-shared/utils";
+import { getMidsceneRunSubDir } from "misoai-shared/common";
+var ScriptPlayer = class {
+  constructor(script, setupAgent, onTaskStatusChange) {
+    this.script = script;
+    this.setupAgent = setupAgent;
+    this.onTaskStatusChange = onTaskStatusChange;
+    this.taskStatusList = [];
+    this.status = "init";
+    this.unnamedResultIndex = 0;
+    this.pageAgent = null;
+    this.result = {};
+    if (ifInBrowser) {
+      this.output = void 0;
+    } else if (script.target?.output) {
+      this.output = resolve(process.cwd(), script.target.output);
+    } else {
+      this.output = join(getMidsceneRunSubDir("output"), `${process.pid}.json`);
+    }
+    this.taskStatusList = (script.tasks || []).map((task, taskIndex) => ({
+      ...task,
+      index: taskIndex,
+      status: "init",
+      totalSteps: task.flow?.length || 0
+    }));
+  }
+  setResult(key, value) {
+    const keyToUse = key || this.unnamedResultIndex++;
+    if (this.result[keyToUse]) {
+      console.warn(`result key ${keyToUse} already exists, will overwrite`);
+    }
+    this.result[keyToUse] = value;
+    this.flushResult();
+  }
+  setPlayerStatus(status, error) {
+    this.status = status;
+    this.errorInSetup = error;
+  }
+  notifyCurrentTaskStatusChange(taskIndex) {
+    const taskIndexToNotify = typeof taskIndex === "number" ? taskIndex : this.currentTaskIndex;
+    if (typeof taskIndexToNotify !== "number") {
+      return;
+    }
+    const taskStatus = this.taskStatusList[taskIndexToNotify];
+    if (this.onTaskStatusChange) {
+      this.onTaskStatusChange(taskStatus);
+    }
+  }
+  async setTaskStatus(index, statusValue, error) {
+    this.taskStatusList[index].status = statusValue;
+    if (error) {
+      this.taskStatusList[index].error = error;
+    }
+    this.notifyCurrentTaskStatusChange(index);
+  }
+  setTaskIndex(taskIndex) {
+    this.currentTaskIndex = taskIndex;
+  }
+  flushResult() {
+    if (Object.keys(this.result).length && this.output) {
+      const output = resolve(process.cwd(), this.output);
+      const outputDir = dirname(output);
+      if (!existsSync(outputDir)) {
+        mkdirSync(outputDir, { recursive: true });
+      }
+      writeFileSync(output, JSON.stringify(this.result, void 0, 2));
+    }
+  }
+  async playTask(taskStatus, agent) {
+    const { flow } = taskStatus;
+    assert(flow, "missing flow in task");
+    for (const flowItemIndex in flow) {
+      const currentStep = Number.parseInt(flowItemIndex, 10);
+      taskStatus.currentStep = currentStep;
+      const flowItem = flow[flowItemIndex];
+      if ("aiAction" in flowItem || "ai" in flowItem) {
+        const actionTask = flowItem;
+        const prompt = actionTask.aiAction || actionTask.ai;
+        assert(prompt, "missing prompt for ai (aiAction)");
+        assert(
+          typeof prompt === "string",
+          "prompt for aiAction must be a string"
+        );
+        await agent.aiAction(prompt, {
+          cacheable: actionTask.cacheable
+        });
+      } else if ("aiAssert" in flowItem) {
+        const assertTask = flowItem;
+        const prompt = assertTask.aiAssert;
+        assert(prompt, "missing prompt for aiAssert");
+        assert(
+          typeof prompt === "string",
+          "prompt for aiAssert must be a string"
+        );
+        await agent.aiAssert(prompt);
+      } else if ("aiQuery" in flowItem) {
+        const queryTask = flowItem;
+        const prompt = queryTask.aiQuery;
+        assert(prompt, "missing prompt for aiQuery");
+        assert(
+          typeof prompt === "string",
+          "prompt for aiQuery must be a string"
+        );
+        const queryResult = await agent.aiQuery(prompt);
+        this.setResult(queryTask.name, queryResult);
+      } else if ("aiNumber" in flowItem) {
+        const numberTask = flowItem;
+        const prompt = numberTask.aiNumber;
+        assert(prompt, "missing prompt for number");
+        assert(
+          typeof prompt === "string",
+          "prompt for number must be a string"
+        );
+        const numberResult = await agent.aiNumber(prompt);
+        this.setResult(numberTask.name, numberResult);
+      } else if ("aiString" in flowItem) {
+        const stringTask = flowItem;
+        const prompt = stringTask.aiString;
+        assert(prompt, "missing prompt for string");
+        assert(
+          typeof prompt === "string",
+          "prompt for string must be a string"
+        );
+        const stringResult = await agent.aiString(prompt);
+        this.setResult(stringTask.name, stringResult);
+      } else if ("aiBoolean" in flowItem) {
+        const booleanTask = flowItem;
+        const prompt = booleanTask.aiBoolean;
+        assert(prompt, "missing prompt for boolean");
+        assert(
+          typeof prompt === "string",
+          "prompt for boolean must be a string"
+        );
+        const booleanResult = await agent.aiBoolean(prompt);
+        this.setResult(booleanTask.name, booleanResult);
+      } else if ("aiLocate" in flowItem) {
+        const locateTask = flowItem;
+        const prompt = locateTask.aiLocate;
+        assert(prompt, "missing prompt for aiLocate");
+        assert(
+          typeof prompt === "string",
+          "prompt for aiLocate must be a string"
+        );
+        const locateResult = await agent.aiLocate(prompt);
+        this.setResult(locateTask.name, locateResult);
+      } else if ("aiWaitFor" in flowItem) {
+        const waitForTask = flowItem;
+        const prompt = waitForTask.aiWaitFor;
+        assert(prompt, "missing prompt for aiWaitFor");
+        assert(
+          typeof prompt === "string",
+          "prompt for aiWaitFor must be a string"
+        );
+        const timeout = waitForTask.timeout;
+        await agent.aiWaitFor(prompt, { timeoutMs: timeout });
+      } else if ("sleep" in flowItem) {
+        const sleepTask = flowItem;
+        const ms = sleepTask.sleep;
+        let msNumber = ms;
+        if (typeof ms === "string") {
+          msNumber = Number.parseInt(ms, 10);
+        }
+        assert(
+          msNumber && msNumber > 0,
+          `ms for sleep must be greater than 0, but got ${ms}`
+        );
+        await new Promise((resolve2) => setTimeout(resolve2, msNumber));
+      } else if ("aiTap" in flowItem) {
+        const tapTask = flowItem;
+        await agent.aiTap(tapTask.aiTap, tapTask);
+      } else if ("aiHover" in flowItem) {
+        const hoverTask = flowItem;
+        await agent.aiHover(hoverTask.aiHover, hoverTask);
+      } else if ("aiInput" in flowItem) {
+        const inputTask = flowItem;
+        await agent.aiInput(inputTask.aiInput, inputTask.locate, inputTask);
+      } else if ("aiKeyboardPress" in flowItem) {
+        const keyboardPressTask = flowItem;
+        await agent.aiKeyboardPress(
+          keyboardPressTask.aiKeyboardPress,
+          keyboardPressTask.locate,
+          keyboardPressTask
+        );
+      } else if ("aiScroll" in flowItem) {
+        const scrollTask = flowItem;
+        await agent.aiScroll(scrollTask, scrollTask.locate, scrollTask);
+      } else if ("javascript" in flowItem) {
+        const evaluateJavaScriptTask = flowItem;
+        const result = await agent.evaluateJavaScript(
+          evaluateJavaScriptTask.javascript
+        );
+        this.setResult(evaluateJavaScriptTask.name, result);
+      } else {
+        throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
+      }
+    }
+    this.reportFile = agent.reportFile;
+  }
+  async run() {
+    const { target, web, android, tasks } = this.script;
+    const webEnv = web || target;
+    const androidEnv = android;
+    const platform = webEnv || androidEnv;
+    this.setPlayerStatus("running");
+    let agent = null;
+    let freeFn = [];
+    try {
+      const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(
+        platform
+      );
+      agent = newAgent;
+      const originalOnTaskStartTip = agent.onTaskStartTip;
+      agent.onTaskStartTip = (tip) => {
+        if (this.status === "running") {
+          this.agentStatusTip = tip;
+        }
+        originalOnTaskStartTip?.(tip);
+      };
+      freeFn = [
+        ...newFreeFn || [],
+        {
+          name: "restore-agent-onTaskStartTip",
+          fn: () => {
+            if (agent) {
+              agent.onTaskStartTip = originalOnTaskStartTip;
+            }
+          }
+        }
+      ];
+    } catch (e) {
+      this.setPlayerStatus("error", e);
+      return;
+    }
+    this.pageAgent = agent;
+    let taskIndex = 0;
+    this.setPlayerStatus("running");
+    let errorFlag = false;
+    while (taskIndex < tasks.length) {
+      const taskStatus = this.taskStatusList[taskIndex];
+      this.setTaskStatus(taskIndex, "running");
+      this.setTaskIndex(taskIndex);
+      try {
+        await this.playTask(taskStatus, this.pageAgent);
+        this.setTaskStatus(taskIndex, "done");
+      } catch (e) {
+        this.setTaskStatus(taskIndex, "error", e);
+        if (taskStatus.continueOnError) {
+        } else {
+          this.reportFile = agent.reportFile;
+          errorFlag = true;
+          break;
+        }
+      }
+      this.reportFile = agent.reportFile;
+      taskIndex++;
+    }
+    if (errorFlag) {
+      this.setPlayerStatus("error");
+    } else {
+      this.setPlayerStatus("done");
+    }
+    this.agentStatusTip = "";
+    for (const fn of freeFn) {
+      try {
+        await fn.fn();
+      } catch (e) {
+      }
+    }
+  }
+};
+
+// src/yaml/builder.ts
+import yaml from "js-yaml";
+
+// src/yaml/utils.ts
+import { assert as assert2 } from "misoai-shared/utils";
+import yaml2 from "js-yaml";
+function interpolateEnvVars(content) {
+  return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
+    const value = process.env[envVar.trim()];
+    if (value === void 0) {
+      throw new Error(`Environment variable "${envVar.trim()}" is not defined`);
+    }
+    return value;
+  });
+}
+function parseYamlScript(content, filePath, ignoreCheckingTarget) {
+  let processedContent = content;
+  if (content.indexOf("android") !== -1 && content.match(/deviceId:\s*(\d+)/)) {
+    let matchedDeviceId;
+    processedContent = content.replace(
+      /deviceId:\s*(\d+)/g,
+      (match, deviceId) => {
+        matchedDeviceId = deviceId;
+        return `deviceId: '${deviceId}'`;
+      }
+    );
+    console.warn(
+      `please use string-style deviceId in yaml script, for example: deviceId: "${matchedDeviceId}"`
+    );
+  }
+  const interpolatedContent = interpolateEnvVars(processedContent);
+  const obj = yaml2.load(interpolatedContent, {
+    schema: yaml2.JSON_SCHEMA
+  });
+  const pathTip = filePath ? `, failed to load ${filePath}` : "";
+  const android = typeof obj.android !== "undefined" ? Object.assign({}, obj.android || {}) : void 0;
+  const webConfig = obj.web || obj.target;
+  const web = typeof webConfig !== "undefined" ? Object.assign({}, webConfig || {}) : void 0;
+  if (!ignoreCheckingTarget) {
+    assert2(
+      web || android,
+      `at least one of "target", "web", or "android" properties is required in yaml script${pathTip}`
+    );
+    assert2(
+      web && !android || !web && android,
+      `only one of "target", "web", or "android" properties is allowed in yaml script${pathTip}`
+    );
+    if (web || android) {
+      assert2(
+        typeof web === "object" || typeof android === "object",
+        `property "target/web/android" must be an object${pathTip}`
+      );
+    }
+  }
+  assert2(obj.tasks, `property "tasks" is required in yaml script ${pathTip}`);
+  assert2(
+    Array.isArray(obj.tasks),
+    `property "tasks" must be an array in yaml script, but got ${obj.tasks}`
+  );
+  return obj;
+}
+
+// src/common/agent.ts
+import {
+  groupedActionDumpFileExt,
+  reportHTMLContent,
+  stringifyDumpData,
+  writeLogFile
+} from "misoai-core/utils";
+import {
+  DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,
+  DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT
+} from "misoai-shared/constants";
+import { getAIConfigInBoolean, vlLocateMode } from "misoai-shared/env";
+import { getDebug as getDebug4 } from "misoai-shared/logger";
+import { assert as assert7 } from "misoai-shared/utils";
+
+// src/common/tasks.ts
+import {
+  Executor,
+  plan
+} from "misoai-core";
+import {
+  elementByPositionWithElementInfo as elementByPositionWithElementInfo2,
+  resizeImageForUiTars,
+  vlmPlanning
+} from "misoai-core/ai-model";
+import { sleep } from "misoai-core/utils";
+import { NodeType } from "misoai-shared/constants";
+import { getDebug } from "misoai-shared/logger";
+import { assert as assert4 } from "misoai-shared/utils";
+
+// src/common/ui-utils.ts
+function typeStr(task) {
+  return task.subType && task.subType !== "Plan" ? `${task.type} / ${task.subType || ""}` : task.type;
+}
+function getKeyCommands(value) {
+  const keys = Array.isArray(value) ? value : [value];
+  return keys.reduce((acc, k) => {
+    const includeMeta = keys.includes("Meta") || keys.includes("Control");
+    if (includeMeta && (k === "a" || k === "A")) {
+      return acc.concat([{ key: k, command: "SelectAll" }]);
+    }
+    if (includeMeta && (k === "c" || k === "C")) {
+      return acc.concat([{ key: k, command: "Copy" }]);
+    }
+    if (includeMeta && (k === "v" || k === "V")) {
+      return acc.concat([{ key: k, command: "Paste" }]);
+    }
+    return acc.concat([{ key: k }]);
+  }, []);
+}
+function locateParamStr(locate) {
+  if (!locate) {
+    return "";
+  }
+  if (typeof locate === "string") {
+    return locate;
+  }
+  return locate.prompt;
+}
+function scrollParamStr(scrollParam) {
+  if (!scrollParam) {
+    return "";
+  }
+  return `${scrollParam.direction || "down"}, ${scrollParam.scrollType || "once"}, ${scrollParam.distance || "distance-not-set"}`;
+}
+function taskTitleStr(type, prompt) {
+  if (prompt) {
+    return `${type} - ${prompt}`;
+  }
+  return type;
+}
+function paramStr(task) {
+  let value;
+  if (task.type === "Planning") {
+    value = task?.param?.userInstruction;
+  }
+  if (task.type === "Insight") {
+    value = task?.param?.prompt || task?.param?.id || task?.param?.dataDemand || task?.param?.assertion;
+  }
+  if (task.type === "Action") {
+    const locate = task?.locate;
+    const locateStr = locate ? locateParamStr(locate) : "";
+    value = task.thought || "";
+    if (typeof task?.param?.timeMs === "number") {
+      value = `${task?.param?.timeMs}ms`;
+    } else if (typeof task?.param?.scrollType === "string") {
+      value = scrollParamStr(task?.param);
+    } else if (typeof task?.param?.value !== "undefined") {
+      value = task?.param?.value;
+    }
+    if (locateStr) {
+      if (value) {
+        value = `${locateStr} - ${value}`;
+      } else {
+        value = locateStr;
+      }
+    }
+  }
+  if (typeof value === "undefined")
+    return "";
+  return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
+}
+
+// src/common/utils.ts
+import { elementByPositionWithElementInfo } from "misoai-core/ai-model";
+import { uploadTestInfoToServer } from "misoai-core/utils";
+import { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from "misoai-shared/env";
+import {
+  generateElementByPosition,
+  getNodeFromCacheList,
+  traverseTree,
+  treeToList
+} from "misoai-shared/extractor";
+import { resizeImgBase64 } from "misoai-shared/img";
+import { assert as assert3, logMsg, uuid } from "misoai-shared/utils";
+import dayjs from "dayjs";
+
+// src/web-element.ts
+var WebElementInfo = class {
+  constructor({
+    content,
+    rect,
+    // page,
+    locator,
+    id,
+    attributes,
+    indexId,
+    xpaths
+  }) {
+    this.content = content;
+    this.rect = rect;
+    this.center = [
+      Math.floor(rect.left + rect.width / 2),
+      Math.floor(rect.top + rect.height / 2)
+    ];
+    this.locator = locator;
+    this.id = id;
+    this.attributes = attributes;
+    this.indexId = indexId;
+    this.xpaths = xpaths;
+  }
+};
+
+// src/common/utils.ts
+async function parseContextFromWebPage(page, _opt) {
+  assert3(page, "page is required");
+  if (page._forceUsePageContext) {
+    return await page._forceUsePageContext();
+  }
+  const url = await page.url();
+  uploadTestInfoToServer({ testUrl: url });
+  let screenshotBase64;
+  let tree;
+  await Promise.all([
+    page.screenshotBase64().then((base64) => {
+      screenshotBase64 = base64;
+    }),
+    page.getElementsNodeTree().then(async (treeRoot) => {
+      tree = treeRoot;
+    })
+  ]);
+  const webTree = traverseTree(tree, (elementInfo) => {
+    const { rect, id, content, attributes, locator, indexId } = elementInfo;
+    return new WebElementInfo({
+      rect,
+      locator,
+      id,
+      content,
+      attributes,
+      indexId
+    });
+  });
+  assert3(screenshotBase64, "screenshotBase64 is required");
+  const elementsInfo = treeToList(webTree);
+  const size = await page.size();
+  if (size.dpr && size.dpr > 1) {
+    screenshotBase64 = await resizeImgBase64(screenshotBase64, {
+      width: size.width,
+      height: size.height
+    });
+  }
+  return {
+    content: elementsInfo,
+    tree: webTree,
+    size,
+    screenshotBase64,
+    url
+  };
+}
+function reportFileName(tag = "web") {
+  const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);
+  const dateTimeInFileName = dayjs().format("YYYY-MM-DD_HH-mm-ss");
+  const uniqueId = uuid().substring(0, 8);
+  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;
+}
+function printReportMsg(filepath) {
+  logMsg(`Midscene - report file updated: ${filepath}`);
+}
+function replaceIllegalPathCharsAndSpace(str) {
+  return str.replace(/[/\\:*?"<>| ]/g, "-");
+}
+function matchElementFromPlan(planLocateParam, tree) {
+  if (!planLocateParam) {
+    return void 0;
+  }
+  if (planLocateParam.id) {
+    return getNodeFromCacheList(planLocateParam.id);
+  }
+  if (planLocateParam.bbox) {
+    const centerPosition = {
+      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),
+      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2)
+    };
+    let element = elementByPositionWithElementInfo(tree, centerPosition);
+    if (!element) {
+      element = generateElementByPosition(centerPosition);
+    }
+    return element;
+  }
+  return void 0;
+}
+
+// src/common/tasks.ts
+var debug = getDebug("page-task-executor");
+var replanningCountLimit = 10;
+var isAndroidPage = (page) => {
+  return page.pageType === "android";
+};
+var PageTaskExecutor = class {
+  constructor(page, insight, opts) {
+    this.conversationHistory = [];
+    this.page = page;
+    this.insight = insight;
+    this.taskCache = opts.taskCache;
+    this.onTaskStartCallback = opts?.onTaskStart;
+  }
+  async recordScreenshot(timing) {
+    const base64 = await this.page.screenshotBase64();
+    const item = {
+      type: "screenshot",
+      ts: Date.now(),
+      screenshot: base64,
+      timing
+    };
+    return item;
+  }
+  async getElementXpath(pageContext, element) {
+    let elementId = element?.id;
+    if (element?.attributes?.nodeType === NodeType.POSITION) {
+      await this.insight.contextRetrieverFn("locate");
+      const info = elementByPositionWithElementInfo2(
+        pageContext.tree,
+        {
+          x: element.center[0],
+          y: element.center[1]
+        },
+        {
+          requireStrictDistance: false,
+          filterPositionElements: true
+        }
+      );
+      if (info?.id) {
+        elementId = info.id;
+      } else {
+        debug(
+          "no element id found for position node, will not update cache",
+          element
+        );
+      }
+    }
+    if (!elementId) {
+      return void 0;
+    }
+    try {
+      const result = await this.page.getXpathsById(elementId);
+      return result;
+    } catch (error) {
+      debug("getXpathsById error: ", error);
+    }
+  }
+  prependExecutorWithScreenshot(taskApply, appendAfterExecution = false) {
+    const taskWithScreenshot = {
+      ...taskApply,
+      executor: async (param, context, ...args) => {
+        const recorder = [];
+        const { task } = context;
+        task.recorder = recorder;
+        const shot = await this.recordScreenshot(`before ${task.type}`);
+        recorder.push(shot);
+        const result = await taskApply.executor(param, context, ...args);
+        if (taskApply.type === "Action") {
+          await Promise.all([
+            (async () => {
+              await sleep(100);
+              if (this.page.waitUntilNetworkIdle) {
+                try {
+                  await this.page.waitUntilNetworkIdle();
+                } catch (error) {
+                }
+              }
+            })(),
+            sleep(200)
+          ]);
+        }
+        if (appendAfterExecution) {
+          const shot2 = await this.recordScreenshot("after Action");
+          recorder.push(shot2);
+        }
+        return result;
+      }
+    };
+    return taskWithScreenshot;
+  }
+  async convertPlanToExecutable(plans, opts) {
+    const tasks = [];
+    plans.forEach((plan2) => {
+      if (plan2.type === "Locate") {
+        if (plan2.locate === null || plan2.locate?.id === null || plan2.locate?.id === "null") {
+          return;
+        }
+        const taskFind = {
+          type: "Insight",
+          subType: "Locate",
+          param: plan2.locate ? {
+            ...plan2.locate,
+            cacheable: opts?.cacheable
+          } : void 0,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param, taskContext) => {
+            const { task } = taskContext;
+            assert4(
+              param?.prompt || param?.id || param?.bbox,
+              "No prompt or id or position or bbox to locate"
+            );
+            let insightDump;
+            let usage;
+            const dumpCollector = (dump) => {
+              insightDump = dump;
+              usage = dump?.taskInfo?.usage;
+              task.log = {
+                dump: insightDump
+              };
+              task.usage = usage;
+            };
+            this.insight.onceDumpUpdatedFn = dumpCollector;
+            const shotTime = Date.now();
+            const pageContext = await this.insight.contextRetrieverFn("locate");
+            task.pageContext = pageContext;
+            const recordItem = {
+              type: "screenshot",
+              ts: shotTime,
+              screenshot: pageContext.screenshotBase64,
+              timing: "before locate"
+            };
+            task.recorder = [recordItem];
+            let cacheHitFlag = false;
+            const cachePrompt = param.prompt;
+            const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);
+            const xpaths = locateCacheRecord?.cacheContent?.xpaths;
+            let elementFromCache = null;
+            try {
+              if (xpaths?.length && this.taskCache?.isCacheResultUsed && param?.cacheable !== false) {
+                for (let i = 0; i < xpaths.length; i++) {
+                  const element2 = await this.page.getElementInfoByXpath(
+                    xpaths[i]
+                  );
+                  if (element2?.id) {
+                    elementFromCache = element2;
+                    debug("cache hit, prompt: %s", cachePrompt);
+                    cacheHitFlag = true;
+                    debug(
+                      "found a new new element with same xpath, xpath: %s, id: %s",
+                      xpaths[i],
+                      element2?.id
+                    );
+                    break;
+                  }
+                }
+              }
+            } catch (error) {
+              debug("get element info by xpath error: ", error);
+            }
+            const startTime = Date.now();
+            const element = elementFromCache || // try to match element from cache
+            matchElementFromPlan(param, pageContext.tree) || // try to match element from plan
+            (await this.insight.locate(param, {
+              context: pageContext
+            })).element;
+            const aiCost = Date.now() - startTime;
+            let currentXpaths;
+            if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) {
+              const elementXpaths = await this.getElementXpath(
+                pageContext,
+                element
+              );
+              if (elementXpaths?.length) {
+                currentXpaths = elementXpaths;
+                this.taskCache.updateOrAppendCacheRecord(
+                  {
+                    type: "locate",
+                    prompt: cachePrompt,
+                    xpaths: elementXpaths
+                  },
+                  locateCacheRecord
+                );
+              } else {
+                debug(
+                  "no xpaths found, will not update cache",
+                  cachePrompt,
+                  elementXpaths
+                );
+              }
+            }
+            if (!element) {
+              throw new Error(`Element not found: ${param.prompt}`);
+            }
+            return {
+              output: {
+                element
+              },
+              pageContext,
+              cache: {
+                hit: cacheHitFlag,
+                originalXpaths: xpaths,
+                currentXpaths
+              },
+              aiCost
+            };
+          }
+        };
+        tasks.push(taskFind);
+      } else if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
+        const assertPlan = plan2;
+        const taskAssert = {
+          type: "Insight",
+          subType: "Assert",
+          param: assertPlan.param,
+          thought: assertPlan.thought,
+          locate: assertPlan.locate,
+          executor: async (param, taskContext) => {
+            const { task } = taskContext;
+            let insightDump;
+            const dumpCollector = (dump) => {
+              insightDump = dump;
+            };
+            this.insight.onceDumpUpdatedFn = dumpCollector;
+            const assertion = await this.insight.assert(
+              assertPlan.param.assertion
+            );
+            if (!assertion.pass) {
+              if (plan2.type === "Assert") {
+                task.output = assertion;
+                task.log = {
+                  dump: insightDump
+                };
+                throw new Error(
+                  assertion.thought || "Assertion failed without reason"
+                );
+              }
+              task.error = assertion.thought;
+            }
+            return {
+              output: assertion,
+              log: {
+                dump: insightDump
+              },
+              usage: assertion.usage
+            };
+          }
+        };
+        tasks.push(taskAssert);
+      } else if (plan2.type === "Input") {
+        const taskActionInput = {
+          type: "Action",
+          subType: "Input",
+          param: plan2.param,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (taskParam, { element }) => {
+            if (element) {
+              await this.page.clearInput(element);
+              if (!taskParam || !taskParam.value) {
+                return;
+              }
+              await this.page.keyboard.type(taskParam.value);
+            } else {
+              await this.page.keyboard.type(taskParam.value);
+            }
+          }
+        };
+        tasks.push(taskActionInput);
+      } else if (plan2.type === "KeyboardPress") {
+        const taskActionKeyboardPress = {
+          type: "Action",
+          subType: "KeyboardPress",
+          param: plan2.param,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (taskParam) => {
+            const keys = getKeyCommands(taskParam.value);
+            await this.page.keyboard.press(keys);
+          }
+        };
+        tasks.push(taskActionKeyboardPress);
+      } else if (plan2.type === "Tap") {
+        const taskActionTap = {
+          type: "Action",
+          subType: "Tap",
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param, { element }) => {
+            assert4(element, "Element not found, cannot tap");
+            await this.page.mouse.click(element.center[0], element.center[1]);
+          }
+        };
+        tasks.push(taskActionTap);
+      } else if (plan2.type === "Drag") {
+        const taskActionDrag = {
+          type: "Action",
+          subType: "Drag",
+          param: plan2.param,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (taskParam) => {
+            assert4(
+              taskParam?.start_box && taskParam?.end_box,
+              "No start_box or end_box to drag"
+            );
+            await this.page.mouse.drag(taskParam.start_box, taskParam.end_box);
+          }
+        };
+        tasks.push(taskActionDrag);
+      } else if (plan2.type === "Hover") {
+        const taskActionHover = {
+          type: "Action",
+          subType: "Hover",
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param, { element }) => {
+            assert4(element, "Element not found, cannot hover");
+            await this.page.mouse.move(element.center[0], element.center[1]);
+          }
+        };
+        tasks.push(taskActionHover);
+      } else if (plan2.type === "Scroll") {
+        const taskActionScroll = {
+          type: "Action",
+          subType: "Scroll",
+          param: plan2.param,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (taskParam, { element }) => {
+            const startingPoint = element ? {
+              left: element.center[0],
+              top: element.center[1]
+            } : void 0;
+            const scrollToEventName = taskParam?.scrollType;
+            if (scrollToEventName === "untilTop") {
+              await this.page.scrollUntilTop(startingPoint);
+            } else if (scrollToEventName === "untilBottom") {
+              await this.page.scrollUntilBottom(startingPoint);
+            } else if (scrollToEventName === "untilRight") {
+              await this.page.scrollUntilRight(startingPoint);
+            } else if (scrollToEventName === "untilLeft") {
+              await this.page.scrollUntilLeft(startingPoint);
+            } else if (scrollToEventName === "once" || !scrollToEventName) {
+              if (taskParam?.direction === "down" || !taskParam || !taskParam.direction) {
+                await this.page.scrollDown(
+                  taskParam?.distance || void 0,
+                  startingPoint
+                );
+              } else if (taskParam.direction === "up") {
+                await this.page.scrollUp(
+                  taskParam.distance || void 0,
+                  startingPoint
+                );
+              } else if (taskParam.direction === "left") {
+                await this.page.scrollLeft(
+                  taskParam.distance || void 0,
+                  startingPoint
+                );
+              } else if (taskParam.direction === "right") {
+                await this.page.scrollRight(
+                  taskParam.distance || void 0,
+                  startingPoint
+                );
+              } else {
+                throw new Error(
+                  `Unknown scroll direction: ${taskParam.direction}`
+                );
+              }
+              await sleep(500);
+            } else {
+              throw new Error(
+                `Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
+                  taskParam
+                )}`
+              );
+            }
+          }
+        };
+        tasks.push(taskActionScroll);
+      } else if (plan2.type === "Sleep") {
+        const taskActionSleep = {
+          type: "Action",
+          subType: "Sleep",
+          param: plan2.param,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (taskParam) => {
+            await sleep(taskParam?.timeMs || 3e3);
+          }
+        };
+        tasks.push(taskActionSleep);
+      } else if (plan2.type === "Error") {
+        const taskActionError = {
+          type: "Action",
+          subType: "Error",
+          param: plan2.param,
+          thought: plan2.thought || plan2.param?.thought,
+          locate: plan2.locate,
+          executor: async () => {
+            throw new Error(
+              plan2?.thought || plan2.param?.thought || "error without thought"
+            );
+          }
+        };
+        tasks.push(taskActionError);
+      } else if (plan2.type === "ExpectedFalsyCondition") {
+        const taskActionFalsyConditionStatement = {
+          type: "Action",
+          subType: "ExpectedFalsyCondition",
+          param: null,
+          thought: plan2.param?.reason,
+          locate: plan2.locate,
+          executor: async () => {
+          }
+        };
+        tasks.push(taskActionFalsyConditionStatement);
+      } else if (plan2.type === "Finished") {
+        const taskActionFinished = {
+          type: "Action",
+          subType: "Finished",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param) => {
+          }
+        };
+        tasks.push(taskActionFinished);
+      } else if (plan2.type === "AndroidHomeButton") {
+        const taskActionAndroidHomeButton = {
+          type: "Action",
+          subType: "AndroidHomeButton",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param) => {
+            assert4(
+              isAndroidPage(this.page),
+              "Cannot use home button on non-Android devices"
+            );
+            await this.page.home();
+          }
+        };
+        tasks.push(taskActionAndroidHomeButton);
+      } else if (plan2.type === "AndroidBackButton") {
+        const taskActionAndroidBackButton = {
+          type: "Action",
+          subType: "AndroidBackButton",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param) => {
+            assert4(
+              isAndroidPage(this.page),
+              "Cannot use back button on non-Android devices"
+            );
+            await this.page.back();
+          }
+        };
+        tasks.push(taskActionAndroidBackButton);
+      } else if (plan2.type === "AndroidRecentAppsButton") {
+        const taskActionAndroidRecentAppsButton = {
+          type: "Action",
+          subType: "AndroidRecentAppsButton",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param) => {
+            assert4(
+              isAndroidPage(this.page),
+              "Cannot use recent apps button on non-Android devices"
+            );
+            await this.page.recentApps();
+          }
+        };
+        tasks.push(taskActionAndroidRecentAppsButton);
+      } else {
+        throw new Error(`Unknown or unsupported task type: ${plan2.type}`);
+      }
+    });
+    const wrappedTasks = tasks.map(
+      (task, index) => {
+        if (task.type === "Action") {
+          return this.prependExecutorWithScreenshot(
+            task,
+            index === tasks.length - 1
+          );
+        }
+        return task;
+      }
+    );
+    return {
+      tasks: wrappedTasks
+    };
+  }
+  async setupPlanningContext(executorContext) {
+    const shotTime = Date.now();
+    const pageContext = await this.insight.contextRetrieverFn("locate");
+    const recordItem = {
+      type: "screenshot",
+      ts: shotTime,
+      screenshot: pageContext.screenshotBase64,
+      timing: "before planning"
+    };
+    executorContext.task.recorder = [recordItem];
+    executorContext.task.pageContext = pageContext;
+    return {
+      pageContext
+    };
+  }
+  async loadYamlFlowAsPlanning(userInstruction, yamlString) {
+    const taskExecutor = new Executor(taskTitleStr("Action", userInstruction), {
+      onTaskStart: this.onTaskStartCallback
+    });
+    const task = {
+      type: "Planning",
+      subType: "LoadYaml",
+      locate: null,
+      param: {
+        userInstruction
+      },
+      executor: async (param, executorContext) => {
+        await this.setupPlanningContext(executorContext);
+        return {
+          output: {
+            actions: [],
+            more_actions_needed_by_instruction: false,
+            log: "",
+            yamlString
+          },
+          cache: {
+            hit: true
+          }
+        };
+      }
+    };
+    await taskExecutor.append(task);
+    await taskExecutor.flush();
+    return {
+      executor: taskExecutor
+    };
+  }
+  planningTaskFromPrompt(userInstruction, log, actionContext) {
+    const task = {
+      type: "Planning",
+      subType: "Plan",
+      locate: null,
+      param: {
+        userInstruction,
+        log
+      },
+      executor: async (param, executorContext) => {
+        const startTime = Date.now();
+        const { pageContext } = await this.setupPlanningContext(executorContext);
+        const planResult = await plan(param.userInstruction, {
+          context: pageContext,
+          log: param.log,
+          actionContext,
+          pageType: this.page.pageType
+        });
+        const {
+          actions,
+          log: log2,
+          more_actions_needed_by_instruction,
+          error,
+          usage,
+          rawResponse,
+          sleep: sleep2
+        } = planResult;
+        executorContext.task.log = {
+          ...executorContext.task.log || {},
+          rawResponse
+        };
+        executorContext.task.usage = usage;
+        let stopCollecting = false;
+        let bboxCollected = false;
+        let planParsingError = "";
+        const finalActions = (actions || []).reduce(
+          (acc, planningAction) => {
+            if (stopCollecting) {
+              return acc;
+            }
+            if (planningAction.locate) {
+              if (bboxCollected && planningAction.locate.bbox) {
+                delete planningAction.locate.bbox;
+              }
+              if (planningAction.locate.bbox) {
+                bboxCollected = true;
+              }
+              acc.push({
+                type: "Locate",
+                locate: planningAction.locate,
+                param: null,
+                thought: planningAction.locate.prompt
+              });
+            } else if (["Tap", "Hover", "Input"].includes(planningAction.type)) {
+              planParsingError = `invalid planning response: ${JSON.stringify(planningAction)}`;
+              stopCollecting = true;
+              return acc;
+            }
+            acc.push(planningAction);
+            return acc;
+          },
+          []
+        );
+        if (sleep2) {
+          const timeNow = Date.now();
+          const timeRemaining = sleep2 - (timeNow - startTime);
+          if (timeRemaining > 0) {
+            finalActions.push({
+              type: "Sleep",
+              param: {
+                timeMs: timeRemaining
+              },
+              locate: null
+            });
+          }
+        }
+        if (finalActions.length === 0) {
+          assert4(
+            !more_actions_needed_by_instruction || sleep2,
+            error ? `Failed to plan: ${error}` : planParsingError || "No plan found"
+          );
+        }
+        return {
+          output: {
+            actions: finalActions,
+            more_actions_needed_by_instruction,
+            log: log2,
+            yamlFlow: planResult.yamlFlow
+          },
+          cache: {
+            hit: false
+          },
+          pageContext
+        };
+      }
+    };
+    return task;
+  }
+  planningTaskToGoal(userInstruction) {
+    const task = {
+      type: "Planning",
+      subType: "Plan",
+      locate: null,
+      param: {
+        userInstruction
+      },
+      executor: async (param, executorContext) => {
+        const { pageContext } = await this.setupPlanningContext(executorContext);
+        const imagePayload = await resizeImageForUiTars(
+          pageContext.screenshotBase64,
+          pageContext.size
+        );
+        this.appendConversationHistory({
+          role: "user",
+          content: [
+            {
+              type: "image_url",
+              image_url: {
+                url: imagePayload
+              }
+            }
+          ]
+        });
+        const startTime = Date.now();
+        const planResult = await vlmPlanning({
+          userInstruction: param.userInstruction,
+          conversationHistory: this.conversationHistory,
+          size: pageContext.size
+        });
+        const aiCost = Date.now() - startTime;
+        const { actions, action_summary } = planResult;
+        this.appendConversationHistory({
+          role: "assistant",
+          content: action_summary
+        });
+        return {
+          output: {
+            actions,
+            thought: actions[0]?.thought,
+            actionType: actions[0].type,
+            more_actions_needed_by_instruction: true,
+            log: "",
+            yamlFlow: planResult.yamlFlow
+          },
+          cache: {
+            hit: false
+          },
+          aiCost
+        };
+      }
+    };
+    return task;
+  }
+  async runPlans(title, plans, opts) {
+    const taskExecutor = new Executor(title, {
+      onTaskStart: this.onTaskStartCallback
+    });
+    const { tasks } = await this.convertPlanToExecutable(plans, opts);
+    await taskExecutor.append(tasks);
+    const result = await taskExecutor.flush();
+    return {
+      output: result,
+      executor: taskExecutor
+    };
+  }
+  async action(userPrompt, actionContext, opts) {
+    const taskExecutor = new Executor(taskTitleStr("Action", userPrompt), {
+      onTaskStart: this.onTaskStartCallback
+    });
+    let planningTask = this.planningTaskFromPrompt(userPrompt, void 0, actionContext);
+    let replanCount = 0;
+    const logList = [];
+    const yamlFlow = [];
+    while (planningTask) {
+      if (replanCount > replanningCountLimit) {
+        const errorMsg = "Replanning too many times, please split the task into multiple steps";
+        return this.appendErrorPlan(taskExecutor, errorMsg);
+      }
+      await taskExecutor.append(planningTask);
+      const planResult = await taskExecutor.flush();
+      if (taskExecutor.isInErrorState()) {
+        return {
+          output: planResult,
+          executor: taskExecutor
+        };
+      }
+      const plans = planResult.actions || [];
+      yamlFlow.push(...planResult.yamlFlow || []);
+      let executables;
+      try {
+        executables = await this.convertPlanToExecutable(plans, opts);
+        taskExecutor.append(executables.tasks);
+      } catch (error) {
+        return this.appendErrorPlan(
+          taskExecutor,
+          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(
+            plans
+          )}`
+        );
+      }
+      await taskExecutor.flush();
+      if (taskExecutor.isInErrorState()) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      if (planResult?.log) {
+        logList.push(planResult.log);
+      }
+      if (!planResult.more_actions_needed_by_instruction) {
+        planningTask = null;
+        break;
+      }
+      planningTask = this.planningTaskFromPrompt(
+        userPrompt,
+        logList.length > 0 ? `- ${logList.join("\n- ")}` : void 0,
+        actionContext
+      );
+      replanCount++;
+    }
+    return {
+      output: {
+        yamlFlow
+      },
+      executor: taskExecutor
+    };
+  }
+  async actionToGoal(userPrompt, opts) {
+    const taskExecutor = new Executor(taskTitleStr("Action", userPrompt), {
+      onTaskStart: this.onTaskStartCallback
+    });
+    this.conversationHistory = [];
+    const isCompleted = false;
+    let currentActionNumber = 0;
+    const maxActionNumber = 40;
+    const yamlFlow = [];
+    while (!isCompleted && currentActionNumber < maxActionNumber) {
+      currentActionNumber++;
+      const planningTask = this.planningTaskToGoal(userPrompt);
+      await taskExecutor.append(planningTask);
+      const output = await taskExecutor.flush();
+      if (taskExecutor.isInErrorState()) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      const plans = output.actions;
+      yamlFlow.push(...output.yamlFlow || []);
+      let executables;
+      try {
+        executables = await this.convertPlanToExecutable(plans, opts);
+        taskExecutor.append(executables.tasks);
+      } catch (error) {
+        return this.appendErrorPlan(
+          taskExecutor,
+          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(
+            plans
+          )}`
+        );
+      }
+      await taskExecutor.flush();
+      if (taskExecutor.isInErrorState()) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      if (plans[0].type === "Finished") {
+        break;
+      }
+    }
+    return {
+      output: {
+        yamlFlow
+      },
+      executor: taskExecutor
+    };
+  }
+  async createTypeQueryTask(type, demand) {
+    const taskExecutor = new Executor(
+      taskTitleStr(
+        type,
+        typeof demand === "string" ? demand : JSON.stringify(demand)
+      ),
+      {
+        onTaskStart: this.onTaskStartCallback
+      }
+    );
+    const queryTask = {
+      type: "Insight",
+      subType: type,
+      locate: null,
+      param: {
+        dataDemand: demand
+        // for user param presentation in report right sidebar
+      },
+      executor: async (param) => {
+        let insightDump;
+        const dumpCollector = (dump) => {
+          insightDump = dump;
+        };
+        this.insight.onceDumpUpdatedFn = dumpCollector;
+        const ifTypeRestricted = type !== "Query";
+        let demandInput = demand;
+        if (ifTypeRestricted) {
+          demandInput = {
+            result: `${type}, ${demand}`
+          };
+        }
+        const { data, usage } = await this.insight.extract(demandInput);
+        let outputResult = data;
+        if (ifTypeRestricted) {
+          assert4(data?.result !== void 0, "No result in query data");
+          outputResult = data.result;
+        }
+        return {
+          output: outputResult,
+          log: { dump: insightDump },
+          usage
+        };
+      }
+    };
+    await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));
+    const output = await taskExecutor.flush();
+    return {
+      output,
+      executor: taskExecutor
+    };
+  }
+  async query(demand) {
+    return this.createTypeQueryTask("Query", demand);
+  }
+  async boolean(prompt) {
+    return this.createTypeQueryTask("Boolean", prompt);
+  }
+  async number(prompt) {
+    return this.createTypeQueryTask("Number", prompt);
+  }
+  async string(prompt) {
+    return this.createTypeQueryTask("String", prompt);
+  }
+  async assert(assertion) {
+    const description = `assert: ${assertion}`;
+    const taskExecutor = new Executor(taskTitleStr("Assert", description), {
+      onTaskStart: this.onTaskStartCallback
+    });
+    const assertionPlan = {
+      type: "Assert",
+      param: {
+        assertion
+      },
+      locate: null
+    };
+    const { tasks } = await this.convertPlanToExecutable([assertionPlan]);
+    await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));
+    const output = await taskExecutor.flush();
+    return {
+      output,
+      executor: taskExecutor
+    };
+  }
+  /**
+   * Append a message to the conversation history
+   * For user messages with images:
+   * - Keep max 4 user image messages in history
+   * - Remove oldest user image message when limit reached
+   * For assistant messages:
+   * - Simply append to history
+   * @param conversationHistory Message to append
+   */
+  appendConversationHistory(conversationHistory) {
+    if (conversationHistory.role === "user") {
+      const userImgItems = this.conversationHistory.filter(
+        (item) => item.role === "user"
+      );
+      if (userImgItems.length >= 4 && conversationHistory.role === "user") {
+        const firstUserImgIndex = this.conversationHistory.findIndex(
+          (item) => item.role === "user"
+        );
+        if (firstUserImgIndex >= 0) {
+          this.conversationHistory.splice(firstUserImgIndex, 1);
+        }
+      }
+    }
+    this.conversationHistory.push(conversationHistory);
+  }
+  async appendErrorPlan(taskExecutor, errorMsg) {
+    const errorPlan = {
+      type: "Error",
+      param: {
+        thought: errorMsg
+      },
+      locate: null
+    };
+    const { tasks } = await this.convertPlanToExecutable([errorPlan]);
+    await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));
+    await taskExecutor.flush();
+    return {
+      output: void 0,
+      executor: taskExecutor
+    };
+  }
+  async waitFor(assertion, opt) {
+    const description = `waitFor: ${assertion}`;
+    const taskExecutor = new Executor(taskTitleStr("WaitFor", description), {
+      onTaskStart: this.onTaskStartCallback
+    });
+    const { timeoutMs, checkIntervalMs } = opt;
+    assert4(assertion, "No assertion for waitFor");
+    assert4(timeoutMs, "No timeoutMs for waitFor");
+    assert4(checkIntervalMs, "No checkIntervalMs for waitFor");
+    const overallStartTime = Date.now();
+    let startTime = Date.now();
+    let errorThought = "";
+    while (Date.now() - overallStartTime < timeoutMs) {
+      startTime = Date.now();
+      const assertPlan = {
+        type: "AssertWithoutThrow",
+        param: {
+          assertion
+        },
+        locate: null
+      };
+      const { tasks: assertTasks } = await this.convertPlanToExecutable([
+        assertPlan
+      ]);
+      await taskExecutor.append(
+        this.prependExecutorWithScreenshot(assertTasks[0])
+      );
+      const output = await taskExecutor.flush();
+      if (output?.pass) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      errorThought = output?.thought || `unknown error when waiting for assertion: ${assertion}`;
+      const now = Date.now();
+      if (now - startTime < checkIntervalMs) {
+        const timeRemaining = checkIntervalMs - (now - startTime);
+        const sleepPlan = {
+          type: "Sleep",
+          param: {
+            timeMs: timeRemaining
+          },
+          locate: null
+        };
+        const { tasks: sleepTasks } = await this.convertPlanToExecutable([
+          sleepPlan
+        ]);
+        await taskExecutor.append(
+          this.prependExecutorWithScreenshot(sleepTasks[0])
+        );
+        await taskExecutor.flush();
+      }
+    }
+    return this.appendErrorPlan(
+      taskExecutor,
+      `waitFor timeout: ${errorThought}`
+    );
+  }
+};
+
+// src/common/plan-builder.ts
+import { getDebug as getDebug2 } from "misoai-shared/logger";
+import { assert as assert5 } from "misoai-shared/utils";
+var debug2 = getDebug2("plan-builder");
+function buildPlans(type, locateParam, param) {
+  let returnPlans = [];
+  const locatePlan = locateParam ? {
+    type: "Locate",
+    locate: locateParam,
+    param: locateParam,
+    thought: ""
+  } : null;
+  if (type === "Tap" || type === "Hover") {
+    assert5(locateParam, `missing locate info for action "${type}"`);
+    assert5(locatePlan, `missing locate info for action "${type}"`);
+    const tapPlan = {
+      type,
+      param: null,
+      thought: "",
+      locate: locateParam
+    };
+    returnPlans = [locatePlan, tapPlan];
+  }
+  if (type === "Input" || type === "KeyboardPress") {
+    if (type === "Input") {
+      assert5(locateParam, `missing locate info for action "${type}"`);
+    }
+    assert5(param, `missing param for action "${type}"`);
+    const inputPlan = {
+      type,
+      param,
+      thought: "",
+      locate: locateParam
+    };
+    if (locatePlan) {
+      returnPlans = [locatePlan, inputPlan];
+    } else {
+      returnPlans = [inputPlan];
+    }
+  }
+  if (type === "Scroll") {
+    assert5(param, `missing param for action "${type}"`);
+    const scrollPlan = {
+      type,
+      param,
+      thought: "",
+      locate: locateParam
+    };
+    if (locatePlan) {
+      returnPlans = [locatePlan, scrollPlan];
+    } else {
+      returnPlans = [scrollPlan];
+    }
+  }
+  if (type === "Sleep") {
+    assert5(param, `missing param for action "${type}"`);
+    const sleepPlan = {
+      type,
+      param,
+      thought: "",
+      locate: null
+    };
+    returnPlans = [sleepPlan];
+  }
+  if (type === "Locate") {
+    assert5(locateParam, `missing locate info for action "${type}"`);
+    const locatePlan2 = {
+      type,
+      param: locateParam,
+      locate: locateParam,
+      thought: ""
+    };
+    returnPlans = [locatePlan2];
+  }
+  if (returnPlans) {
+    debug2("buildPlans", returnPlans);
+    return returnPlans;
+  }
+  throw new Error(`Not supported type: ${type}`);
+}
+
+// src/common/task-cache.ts
+import assert6 from "assert";
+import { existsSync as existsSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
+import { join as join2 } from "path";
+import { getMidsceneRunSubDir as getMidsceneRunSubDir2 } from "misoai-shared/common";
+import { getDebug as getDebug3 } from "misoai-shared/logger";
+import { ifInBrowser as ifInBrowser2 } from "misoai-shared/utils";
+import yaml3 from "js-yaml";
+import semver from "semver";
+
+// package.json
+var version = "1.0.5";
+
+// src/common/task-cache.ts
+var debug3 = getDebug3("cache");
+var lowestSupportedMidsceneVersion = "0.16.10";
+var cacheFileExt = ".cache.yaml";
+var TaskCache = class {
+  // Track matched records
+  constructor(cacheId, isCacheResultUsed, cacheFilePath) {
+    this.matchedCacheIndices = /* @__PURE__ */ new Set();
+    assert6(cacheId, "cacheId is required");
+    this.cacheId = replaceIllegalPathCharsAndSpace(cacheId);
+    this.cacheFilePath = ifInBrowser2 ? void 0 : cacheFilePath || join2(getMidsceneRunSubDir2("cache"), `${this.cacheId}${cacheFileExt}`);
+    this.isCacheResultUsed = isCacheResultUsed;
+    let cacheContent;
+    if (this.cacheFilePath) {
+      cacheContent = this.loadCacheFromFile();
+    }
+    if (!cacheContent) {
+      cacheContent = {
+        midsceneVersion: version,
+        cacheId: this.cacheId,
+        caches: []
+      };
+    }
+    this.cache = cacheContent;
+    this.cacheOriginalLength = this.cache.caches.length;
+  }
+  matchCache(prompt, type) {
+    for (let i = 0; i < this.cacheOriginalLength; i++) {
+      const item = this.cache.caches[i];
+      const key = `${type}:${prompt}:${i}`;
+      if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
+        this.matchedCacheIndices.add(key);
+        debug3(
+          "cache found and marked as used, type: %s, prompt: %s, index: %d",
+          type,
+          prompt,
+          i
+        );
+        return {
+          cacheContent: item,
+          updateFn: (cb) => {
+            debug3(
+              "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
+              type,
+              prompt,
+              i
+            );
+            cb(item);
+            debug3(
+              "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
+              type,
+              prompt,
+              i
+            );
+            this.flushCacheToFile();
+          }
+        };
+      }
+    }
+    debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
+    return void 0;
+  }
+  matchPlanCache(prompt) {
+    return this.matchCache(prompt, "plan");
+  }
+  matchLocateCache(prompt) {
+    return this.matchCache(prompt, "locate");
+  }
+  appendCache(cache) {
+    debug3("will append cache", cache);
+    this.cache.caches.push(cache);
+    this.flushCacheToFile();
+  }
+  loadCacheFromFile() {
+    const cacheFile = this.cacheFilePath;
+    assert6(cacheFile, "cache file path is required");
+    if (!existsSync2(cacheFile)) {
+      debug3("no cache file found, path: %s", cacheFile);
+      return void 0;
+    }
+    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, ".json");
+    if (existsSync2(jsonTypeCacheFile) && this.isCacheResultUsed) {
+      console.warn(
+        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`
+      );
+      return void 0;
+    }
+    try {
+      const data = readFileSync(cacheFile, "utf8");
+      const jsonData = yaml3.load(data);
+      if (!version) {
+        debug3("no midscene version info, will not read cache from file");
+        return void 0;
+      }
+      if (semver.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) && !jsonData.midsceneVersion.includes("beta")) {
+        console.warn(
+          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.
+Please delete the existing cache and rebuild it. Sorry for the inconvenience.
+cache file: ${cacheFile}`
+        );
+        return void 0;
+      }
+      debug3(
+        "cache loaded from file, path: %s, cache version: %s, record length: %s",
+        cacheFile,
+        jsonData.midsceneVersion,
+        jsonData.caches.length
+      );
+      jsonData.midsceneVersion = version;
+      return jsonData;
+    } catch (err) {
+      debug3(
+        "cache file exists but load failed, path: %s, error: %s",
+        cacheFile,
+        err
+      );
+      return void 0;
+    }
+  }
+  flushCacheToFile() {
+    if (!version) {
+      debug3("no midscene version info, will not write cache to file");
+      return;
+    }
+    if (!this.cacheFilePath) {
+      debug3("no cache file path, will not write cache to file");
+      return;
+    }
+    try {
+      const yamlData = yaml3.dump(this.cache);
+      writeFileSync2(this.cacheFilePath, yamlData);
+    } catch (err) {
+      debug3(
+        "write cache to file failed, path: %s, error: %s",
+        this.cacheFilePath,
+        err
+      );
+    }
+  }
+  updateOrAppendCacheRecord(newRecord, cachedRecord) {
+    if (cachedRecord) {
+      if (newRecord.type === "plan") {
+        cachedRecord.updateFn((cache) => {
+          cache.yamlWorkflow = newRecord.yamlWorkflow;
+        });
+      } else {
+        cachedRecord.updateFn((cache) => {
+          cache.xpaths = newRecord.xpaths;
+        });
+      }
+    } else {
+      this.appendCache(newRecord);
+    }
+  }
+};
+
+// src/common/agent.ts
+var debug4 = getDebug4("web-integration");
+var distanceOfTwoPoints = (p1, p2) => {
+  const [x1, y1] = p1;
+  const [x2, y2] = p2;
+  return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));
+};
+var includedInRect = (point, rect) => {
+  const [x, y] = point;
+  const { left, top, width, height } = rect;
+  return x >= left && x <= left + width && y >= top && y <= top + height;
+};
+var PageAgent = class {
+  constructor(page, opts) {
+    /**
+     * If true, the agent will not perform any actions
+     */
+    this.dryMode = false;
+    this.page = page;
+    this.opts = Object.assign(
+      {
+        generateReport: true,
+        autoPrintReportMsg: true,
+        groupName: "Midscene Report",
+        groupDescription: "",
+        enableCumulativeContext: true,
+        autoClearContext: false
+      },
+      opts || {}
+    );
+    this.initializeContextStore();
+    if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
+      this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
+      this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
+    }
+    this.onTaskStartTip = this.opts.onTaskStartTip;
+    this.insight = new Insight(
+      async (action) => {
+        return this.getUIContext(action);
+      }
+    );
+    if (opts?.cacheId && this.page.pageType !== "android") {
+      this.taskCache = new TaskCache(
+        opts.cacheId,
+        getAIConfigInBoolean("MIDSCENE_CACHE")
+        // if we should use cache to match the element
+      );
+    }
+    this.taskExecutor = new PageTaskExecutor(this.page, this.insight, {
+      taskCache: this.taskCache,
+      onTaskStart: this.callbackOnTaskStartTip.bind(this)
+    });
+    this.dump = this.resetDump();
+    this.reportFileName = reportFileName(
+      opts?.testId || this.page.pageType || "web"
+    );
+  }
+  /**
+   * Initialize context store for cumulative context functionality
+   */
+  async initializeContextStore() {
+    if (!this.opts.enableCumulativeContext) {
+      debug4("Cumulative context disabled via options");
+      return;
+    }
+    try {
+      const aiModel = await import("misoai-core/ai-model");
+      this.contextStore = aiModel.getContextStore();
+      debug4("Context store initialized successfully", {
+        autoClearContext: this.opts.autoClearContext,
+        testId: this.opts.testId
+      });
+      if (this.opts.autoClearContext) {
+        this.contextStore.clear();
+        debug4("Context store cleared due to autoClearContext option");
+      } else {
+        const existingData = this.contextStore.getAllData();
+        const existingSteps = this.contextStore.getRecentSteps(100).length;
+        debug4("Context store preserving existing data", {
+          existingDataKeys: Object.keys(existingData),
+          existingStepsCount: existingSteps
+        });
+      }
+    } catch (error) {
+      debug4("Failed to initialize context store:", error);
+      console.warn("⚠️ Could not initialize context store:", error);
+    }
+  }
+  /**
+   * Get the context store instance
+   */
+  getContextStore() {
+    return this.contextStore;
+  }
+  /**
+   * Clear the context store
+   */
+  clearContext() {
+    if (this.contextStore) {
+      this.contextStore.clear();
+    }
+  }
+  /**
+   * Get all stored data from context store
+   */
+  getStoredData() {
+    if (this.contextStore) {
+      return this.contextStore.getAllData();
+    }
+    return {};
+  }
+  /**
+   * Get step summary from context store
+   */
+  getStepSummary() {
+    if (this.contextStore) {
+      return this.contextStore.getStepSummary();
+    }
+    return "";
+  }
+  async getUIContext(action) {
+    if (action && (action === "extract" || action === "assert" || action === "captcha")) {
+      return await parseContextFromWebPage(this.page, {
+        ignoreMarker: true
+      });
+    }
+    return await parseContextFromWebPage(this.page, {
+      ignoreMarker: !!vlLocateMode()
+    });
+  }
+  // Helper method to call the insight.captcha method
+  async _callInsightCaptcha(options) {
+    const context = await this.getUIContext();
+    if (this.page.url) {
+      const url = await this.page.url();
+      context.url = url;
+    }
+    return this.insight.captcha(context, options);
+  }
+  async setAIActionContext(prompt) {
+    this.opts.aiActionContext = prompt;
+  }
+  resetDump() {
+    this.dump = {
+      groupName: this.opts.groupName,
+      groupDescription: this.opts.groupDescription,
+      executions: []
+    };
+    return this.dump;
+  }
+  appendExecutionDump(execution) {
+    const currentDump = this.dump;
+    currentDump.executions.push(execution);
+  }
+  dumpDataString() {
+    this.dump.groupName = this.opts.groupName;
+    this.dump.groupDescription = this.opts.groupDescription;
+    return stringifyDumpData(this.dump);
+  }
+  reportHTMLString() {
+    return reportHTMLContent(this.dumpDataString());
+  }
+  writeOutActionDumps() {
+    const { generateReport, autoPrintReportMsg } = this.opts;
+    this.reportFile = writeLogFile({
+      fileName: this.reportFileName,
+      fileExt: groupedActionDumpFileExt,
+      fileContent: this.dumpDataString(),
+      type: "dump",
+      generateReport
+    });
+    debug4("writeOutActionDumps", this.reportFile);
+    if (generateReport && autoPrintReportMsg && this.reportFile) {
+      printReportMsg(this.reportFile);
+    }
+  }
+  async callbackOnTaskStartTip(task) {
+    const param = paramStr(task);
+    const tip = param ? `${typeStr(task)} - ${param}` : typeStr(task);
+    if (this.onTaskStartTip) {
+      await this.onTaskStartTip(tip);
+    }
+  }
+  afterTaskRunning(executor, doNotThrowError = false) {
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState() && !doNotThrowError) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask?.error}`);
+    }
+    const lastTask = executor.tasks[executor.tasks.length - 1];
+    const allThoughts = executor.tasks.filter((task) => task.thought).map((task) => task.thought);
+    const allLocates = executor.tasks.filter((task) => task.locate).map((task) => task.locate);
+    const allPlans = executor.tasks.filter((task) => task.param?.plans).map((task) => task.param?.plans);
+    const planningTasks = executor.tasks.filter((task) => task.type === "Planning");
+    const insightTasks = executor.tasks.filter((task) => task.type === "Insight");
+    const actionTasks = executor.tasks.filter((task) => task.type === "Action");
+    const planning = planningTasks.length > 0 ? {
+      type: "Planning",
+      description: `Planning for task execution`,
+      steps: planningTasks.map((task) => task.thought || "Planning step")
+    } : void 0;
+    const insight = insightTasks.length > 0 ? {
+      type: "Insight",
+      description: `Insight for task execution`,
+      elements: insightTasks.map((task) => task.thought || "Insight element")
+    } : void 0;
+    const action = actionTasks.length > 0 ? {
+      type: "Action",
+      description: `Action for task execution`,
+      result: lastTask?.output
+    } : void 0;
+    const actionDetails = executor.tasks.map((task) => ({
+      type: task.type,
+      subType: task.subType,
+      status: task.status,
+      thought: task.thought
+    }));
+    const metadata = {
+      status: lastTask?.status,
+      start: lastTask?.timing?.start,
+      end: lastTask?.timing?.end,
+      totalTime: lastTask?.timing?.cost,
+      cache: lastTask?.cache,
+      usage: lastTask?.usage,
+      thought: allThoughts.length > 0 ? allThoughts.join("\n") : lastTask?.thought,
+      locate: allLocates.length > 0 ? allLocates : lastTask?.locate,
+      plan: allPlans.length > 0 ? allPlans : lastTask?.param?.plans,
+      // Add planning, insight, and action information
+      planning,
+      insight,
+      action,
+      actionDetails,
+      // Include raw tasks for debugging
+      tasks: executor.tasks.map((task) => ({
+        type: task.type,
+        subType: task.subType,
+        status: task.status,
+        thought: task.thought,
+        locate: task.locate,
+        timing: task.timing,
+        usage: task.usage,
+        cache: task.cache,
+        error: task.error
+      }))
+    };
+    return metadata;
+  }
+  buildDetailedLocateParam(locatePrompt, opt) {
+    assert7(locatePrompt, "missing locate prompt");
+    if (typeof opt === "object") {
+      const prompt = opt.prompt ?? locatePrompt;
+      const deepThink = opt.deepThink ?? false;
+      const cacheable = opt.cacheable ?? true;
+      return {
+        prompt,
+        deepThink,
+        cacheable
+      };
+    }
+    return {
+      prompt: locatePrompt
+    };
+  }
+  async aiTap(locatePrompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(
+      locatePrompt,
+      opt
+    );
+    const plans = buildPlans("Tap", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Tap", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiHover(locatePrompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(
+      locatePrompt,
+      opt
+    );
+    const plans = buildPlans("Hover", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Hover", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiInput(value, locatePrompt, opt) {
+    assert7(
+      typeof value === "string",
+      "input value must be a string, use empty string if you want to clear the input"
+    );
+    assert7(locatePrompt, "missing locate prompt for input");
+    const detailedLocateParam = this.buildDetailedLocateParam(
+      locatePrompt,
+      opt
+    );
+    const plans = buildPlans("Input", detailedLocateParam, {
+      value
+    });
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Input", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiKeyboardPress(keyName, locatePrompt, opt) {
+    assert7(keyName, "missing keyName for keyboard press");
+    const detailedLocateParam = locatePrompt ? this.buildDetailedLocateParam(locatePrompt, opt) : void 0;
+    const plans = buildPlans("KeyboardPress", detailedLocateParam, {
+      value: keyName
+    });
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("KeyboardPress", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiScroll(scrollParam, locatePrompt, opt) {
+    const detailedLocateParam = locatePrompt ? this.buildDetailedLocateParam(locatePrompt, opt) : void 0;
+    const plans = buildPlans("Scroll", detailedLocateParam, scrollParam);
+    const paramInTitle = locatePrompt ? `${locateParamStr(detailedLocateParam)} - ${scrollParamStr(scrollParam)}` : scrollParamStr(scrollParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Scroll", paramInTitle),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiAction(taskPrompt, opt) {
+    if (this.opts.enableCumulativeContext && this.contextStore) {
+      try {
+        const originalPrompt = taskPrompt;
+        const processedPrompt = this.contextStore.replaceAllReferences(taskPrompt, "action");
+        if (originalPrompt !== processedPrompt) {
+          debug4("Context replacement in aiAction:", {
+            original: originalPrompt,
+            processed: processedPrompt,
+            storedData: this.contextStore.getAllData()
+          });
+        }
+        this.contextStore.addStep({
+          type: "action",
+          summary: `Action: ${processedPrompt}`,
+          prompt: processedPrompt
+        });
+        debug4("Added action step to context store:", {
+          stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
+          totalSteps: this.contextStore.getRecentSteps(100).length
+        });
+        taskPrompt = processedPrompt;
+      } catch (error) {
+        debug4("Context store operation failed:", error);
+      }
+    }
+    const cacheable = opt?.cacheable;
+    const isVlmUiTars = vlLocateMode() === "vlm-ui-tars";
+    const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
+    if (matchedCache && this.taskCache?.isCacheResultUsed) {
+      const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
+        taskPrompt,
+        matchedCache.cacheContent?.yamlWorkflow
+      );
+      const metadata2 = this.afterTaskRunning(executor2);
+      debug4("matched cache, will call .runYaml to run the action");
+      const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
+      const result = await this.runYaml(yaml5);
+      return {
+        result: result.result,
+        metadata: metadata2
+      };
+    }
+    const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {
+      cacheable
+    }));
+    if (this.taskCache && output?.yamlFlow && cacheable !== false) {
+      const yamlContent = {
+        tasks: [
+          {
+            name: taskPrompt,
+            flow: output.yamlFlow
+          }
+        ]
+      };
+      const yamlFlowStr = yaml4.dump(yamlContent);
+      this.taskCache.updateOrAppendCacheRecord(
+        {
+          type: "plan",
+          prompt: taskPrompt,
+          yamlWorkflow: yamlFlowStr
+        },
+        matchedCache
+      );
+    }
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiQuery(demand) {
+    let processedDemand = demand;
+    let storageKey;
+    try {
+      const aiModel = await import("misoai-core/ai-model");
+      const contextStore = aiModel.getContextStore();
+      if (typeof demand === "string") {
+        const storageInstruction = contextStore.parseStorageInstruction(demand);
+        if (storageInstruction) {
+          storageKey = storageInstruction.key;
+          processedDemand = storageInstruction.cleanText;
+          contextStore._pendingAliases = storageInstruction.aliases;
+        } else {
+          const storageMatch = demand.match(/store\s+(?:as\s+)?(\w+)/i);
+          if (storageMatch) {
+            storageKey = storageMatch[1];
+            processedDemand = demand.replace(/,?\s*store\s+(?:as\s+)?\w+/i, "").trim();
+          }
+        }
+      }
+    } catch (error) {
+      debug4("Context store not available:", error);
+    }
+    const { output, executor } = await this.taskExecutor.query(processedDemand);
+    if (this.opts.enableCumulativeContext && this.contextStore) {
+      if (storageKey && output) {
+        try {
+          const pendingAliases = this.contextStore._pendingAliases;
+          if (pendingAliases) {
+            this.contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
+            delete this.contextStore._pendingAliases;
+            debug4("Stored query result with aliases:", {
+              key: storageKey,
+              value: output,
+              aliases: pendingAliases
+            });
+          } else {
+            this.contextStore.storeData(storageKey, output);
+            debug4("Stored query result:", {
+              key: storageKey,
+              value: output
+            });
+          }
+          this.contextStore.addStep({
+            type: "query",
+            summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
+            data: output,
+            prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
+          });
+          debug4("Added query step to context store:", {
+            storageKey,
+            totalStoredItems: Object.keys(this.contextStore.getAllData()).length,
+            totalSteps: this.contextStore.getRecentSteps(100).length
+          });
+        } catch (error) {
+          debug4("Failed to store query result:", error);
+        }
+      } else {
+        try {
+          this.contextStore.addStep({
+            type: "query",
+            summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
+            data: output,
+            prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
+          });
+        } catch (error) {
+          debug4("Failed to add query step:", error);
+        }
+      }
+    }
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiBoolean(prompt) {
+    const { output, executor } = await this.taskExecutor.boolean(prompt);
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiNumber(prompt) {
+    const { output, executor } = await this.taskExecutor.number(prompt);
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiString(prompt) {
+    const { output, executor } = await this.taskExecutor.string(prompt);
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async describeElementAtPoint(center, opt) {
+    const { verifyPrompt = true, retryLimit = 3 } = opt || {};
+    let success = false;
+    let retryCount = 0;
+    let resultPrompt = "";
+    let deepThink = opt?.deepThink || false;
+    let verifyResult;
+    while (!success && retryCount < retryLimit) {
+      if (retryCount >= 2) {
+        deepThink = true;
+      }
+      debug4(
+        "aiDescribe",
+        center,
+        "verifyPrompt",
+        verifyPrompt,
+        "retryCount",
+        retryCount,
+        "deepThink",
+        deepThink
+      );
+      const text = await this.insight.describe(center, { deepThink });
+      debug4("aiDescribe text", text);
+      assert7(text.description, `failed to describe element at [${center}]`);
+      resultPrompt = text.description;
+      verifyResult = await this.verifyLocator(
+        resultPrompt,
+        deepThink ? { deepThink: true } : void 0,
+        center,
+        opt
+      );
+      if (verifyResult.pass) {
+        success = true;
+      } else {
+        retryCount++;
+      }
+    }
+    return {
+      prompt: resultPrompt,
+      deepThink,
+      verifyResult
+    };
+  }
+  async verifyLocator(prompt, locateOpt, expectCenter, verifyLocateOption) {
+    debug4("verifyLocator", prompt, locateOpt, expectCenter, verifyLocateOption);
+    const locateResult = await this.aiLocate(prompt, locateOpt);
+    const { center: verifyCenter, rect: verifyRect } = locateResult.result;
+    const distance = distanceOfTwoPoints(expectCenter, verifyCenter);
+    const included = includedInRect(expectCenter, verifyRect);
+    const pass = distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || included;
+    const verifyResult = {
+      pass,
+      rect: verifyRect,
+      center: verifyCenter,
+      centerDistance: distance
+    };
+    debug4("aiDescribe verifyResult", verifyResult);
+    return verifyResult;
+  }
+  async aiLocate(prompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);
+    const plans = buildPlans("Locate", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    const { element } = output;
+    const result = {
+      rect: element?.rect,
+      center: element?.center
+    };
+    return {
+      result,
+      metadata
+    };
+  }
+  async aiAssert(assertion, msg, opt) {
+    let processedAssertion = assertion;
+    if (this.opts.enableCumulativeContext && this.contextStore) {
+      try {
+        const originalAssertion = assertion;
+        processedAssertion = this.contextStore.replaceAllReferences(assertion, "assertion");
+        if (originalAssertion !== processedAssertion) {
+          debug4("Context replacement in aiAssert:", {
+            original: originalAssertion,
+            processed: processedAssertion,
+            context: "assertion",
+            storedData: this.contextStore.getAllData()
+          });
+        }
+        this.contextStore.addStep({
+          type: "assertion",
+          summary: `Assertion: ${processedAssertion}`,
+          prompt: processedAssertion
+        });
+        debug4("Added assertion step to context store:", {
+          totalSteps: this.contextStore.getRecentSteps(100).length
+        });
+      } catch (error) {
+        debug4("Context store operation failed:", error);
+      }
+    }
+    let currentUrl = "";
+    if (this.page.url) {
+      try {
+        currentUrl = await this.page.url();
+      } catch (e) {
+      }
+    }
+    const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${processedAssertion}` : processedAssertion;
+    const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
+    const metadata = this.afterTaskRunning(executor, true);
+    if (output && opt?.keepRawResponse) {
+      return {
+        result: output,
+        metadata
+      };
+    }
+    if (!output?.pass) {
+      const errMsg = msg || `Assertion failed: ${assertion}`;
+      const reasonMsg = `Reason: ${output?.thought || executor.latestErrorTask()?.error || "(no_reason)"}`;
+      throw new Error(`${errMsg}
+${reasonMsg}`);
+    }
+    return {
+      result: true,
+      metadata
+    };
+  }
+  async aiCaptcha(options) {
+    const { deepThink = false, autoDetectComplexity = true } = options || {};
+    let shouldUseDeepThink = deepThink;
+    if (autoDetectComplexity && !deepThink) {
+      const context = await this.getUIContext();
+      const { screenshotBase64 } = context;
+      try {
+        const complexityAnalysisPrompt = `
+Analyze this screenshot and determine if it contains a complex CAPTCHA that would benefit from deep thinking.
+A complex CAPTCHA typically has one or more of these characteristics:
+- Distorted or overlapping text that is hard to read
+- Multiple images that need to be selected based on a specific criteria
+- Puzzles that require spatial reasoning
+- Multiple steps or verification methods
+- Small or hard-to-distinguish elements
+
+Return only "complex" or "simple" based on your analysis.
+`;
+        const complexityMsgs = [
+          { role: "system", content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity." },
+          {
+            role: "user",
+            content: [
+              {
+                type: "image_url",
+                image_url: {
+                  url: screenshotBase64,
+                  detail: "high"
+                }
+              },
+              {
+                type: "text",
+                text: complexityAnalysisPrompt
+              }
+            ]
+          }
+        ];
+        const complexityResult = await this.insight.aiVendorFn(
+          complexityMsgs,
+          { type: "extract_data" }
+        );
+        const responseText = typeof complexityResult.content === "string" ? complexityResult.content.toLowerCase() : JSON.stringify(complexityResult.content).toLowerCase();
+        shouldUseDeepThink = responseText.includes("complex");
+        debug4("CAPTCHA complexity analysis:", responseText, "Using deep think:", shouldUseDeepThink);
+      } catch (error) {
+        debug4("Failed to analyze CAPTCHA complexity:", error);
+      }
+    }
+    const captchaResponse = await this._callInsightCaptcha({
+      deepThink: shouldUseDeepThink
+    });
+    const captchaResult = captchaResponse.content;
+    const usage = captchaResponse.usage;
+    const actualDeepThink = captchaResponse.deepThink || false;
+    if (captchaResult.captchaType === "text") {
+      for (const action of captchaResult.actions) {
+        if (action.type === "click" && action.target) {
+          await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
+        } else if (action.type === "input" && action.value) {
+          if (action.target) {
+            await this.aiInput(action.value, action.target, { deepThink: shouldUseDeepThink });
+          }
+        } else if (action.type === "verify" && action.target) {
+          await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
+        }
+      }
+    } else if (captchaResult.captchaType === "image") {
+      for (const action of captchaResult.actions) {
+        if (action.type === "click") {
+          if (action.coordinates) {
+            const x = action.coordinates[0];
+            const y = action.coordinates[1];
+            await this.aiTap(`element at coordinates (${x}, ${y})`, { deepThink: shouldUseDeepThink });
+          } else if (action.target) {
+            await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
+          }
+        } else if (action.type === "verify" && action.target) {
+          await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
+        }
+      }
+    }
+    await new Promise((resolve2) => setTimeout(resolve2, 3e3));
+    const metadata = {
+      status: "finished",
+      usage,
+      thought: captchaResult.thought
+    };
+    metadata.deepThink = actualDeepThink;
+    if (autoDetectComplexity && !deepThink) {
+      metadata.autoDetectedComplexity = shouldUseDeepThink;
+    }
+    return {
+      result: captchaResult,
+      metadata
+    };
+  }
+  async aiWaitFor(assertion, opt) {
+    const startTime = Date.now();
+    const { executor } = await this.taskExecutor.waitFor(assertion, {
+      timeoutMs: opt?.timeoutMs || 15 * 1e3,
+      checkIntervalMs: opt?.checkIntervalMs || 3 * 1e3,
+      assertion
+    });
+    const metadata = {
+      status: executor.isInErrorState() ? "failed" : "finished",
+      start: startTime,
+      end: Date.now(),
+      totalTime: Date.now() - startTime,
+      thought: executor.latestErrorTask()?.thought,
+      actionDetails: executor.tasks.map((task) => ({
+        type: task.type,
+        subType: task.subType,
+        status: task.status,
+        thought: task.thought
+      }))
+    };
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState()) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask?.error}
+${errorTask?.errorStack}`);
+    }
+    return {
+      result: true,
+      // Successfully waited
+      metadata
+    };
+  }
+  async ai(taskPrompt, type = "action", options) {
+    if (type === "action") {
+      return this.aiAction(taskPrompt);
+    }
+    if (type === "query") {
+      return this.aiQuery(taskPrompt);
+    }
+    if (type === "assert") {
+      return this.aiAssert(taskPrompt);
+    }
+    if (type === "tap") {
+      return this.aiTap(taskPrompt, options);
+    }
+    if (type === "captcha") {
+      return this.aiCaptcha(options);
+    }
+    throw new Error(
+      `Unknown type: ${type}, only support 'action', 'query', 'assert', 'tap', 'captcha'`
+    );
+  }
+  async runYaml(yamlScriptContent) {
+    const startTime = Date.now();
+    const script = parseYamlScript(yamlScriptContent, "yaml", true);
+    const player = new ScriptPlayer(script, async () => {
+      return { agent: this, freeFn: [] };
+    });
+    await player.run();
+    const endTime = Date.now();
+    const metadata = {
+      status: player.status,
+      start: startTime,
+      end: endTime,
+      totalTime: endTime - startTime,
+      tasks: player.taskStatusList.map((task) => ({
+        type: "yaml-task",
+        subType: task.name,
+        status: task.status,
+        error: task.error?.message
+      }))
+    };
+    if (player.status === "error") {
+      const errors = player.taskStatusList.filter((task) => task.status === "error").map((task) => {
+        return `task - ${task.name}: ${task.error?.message}`;
+      }).join("\n");
+      throw new Error(`Error(s) occurred in running yaml script:
+${errors}`);
+    }
+    return {
+      result: player.result,
+      metadata
+    };
+  }
+  async evaluateJavaScript(script) {
+    assert7(
+      this.page.evaluateJavaScript,
+      "evaluateJavaScript is not supported in current agent"
+    );
+    if (this.page.evaluateJavaScript) {
+      return this.page.evaluateJavaScript(script);
+    }
+    throw new Error("evaluateJavaScript is not supported in current agent");
+  }
+  async destroy() {
+    await this.page.destroy();
+  }
+};
+export {
+  PageAgent
+};
+
+//# sourceMappingURL=agent.js.map