misoai-web 1.0.3 → 1.0.4

package/dist/lib/index.js

@@ -0,0 +1,3206 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  PageAgent: () => PageAgent,
+  PlaywrightAgent: () => PlaywrightAgent,
+  PlaywrightAiFixture: () => PlaywrightAiFixture,
+  PuppeteerAgent: () => PuppeteerAgent,
+  ScriptPlayer: () => ScriptPlayer,
+  StaticPageAgent: () => StaticPageAgent,
+  parseContextFromWebPage: () => parseContextFromWebPage,
+  parseYamlScript: () => parseYamlScript
+});
+module.exports = __toCommonJS(src_exports);
+
+// src/common/agent.ts
+var import_misoai_core2 = require("misoai-core");
+var import_js_yaml4 = __toESM(require("js-yaml"));
+
+// src/yaml/player.ts
+var import_node_fs = require("fs");
+var import_node_path = require("path");
+var import_utils = require("misoai-shared/utils");
+var import_common = require("misoai-shared/common");
+var ScriptPlayer = class {
+  constructor(script, setupAgent, onTaskStatusChange) {
+    this.script = script;
+    this.setupAgent = setupAgent;
+    this.onTaskStatusChange = onTaskStatusChange;
+    this.taskStatusList = [];
+    this.status = "init";
+    this.unnamedResultIndex = 0;
+    this.pageAgent = null;
+    this.result = {};
+    if (import_utils.ifInBrowser) {
+      this.output = void 0;
+    } else if (script.target?.output) {
+      this.output = (0, import_node_path.resolve)(process.cwd(), script.target.output);
+    } else {
+      this.output = (0, import_node_path.join)((0, import_common.getMidsceneRunSubDir)("output"), `${process.pid}.json`);
+    }
+    this.taskStatusList = (script.tasks || []).map((task, taskIndex) => ({
+      ...task,
+      index: taskIndex,
+      status: "init",
+      totalSteps: task.flow?.length || 0
+    }));
+  }
+  setResult(key, value) {
+    const keyToUse = key || this.unnamedResultIndex++;
+    if (this.result[keyToUse]) {
+      console.warn(`result key ${keyToUse} already exists, will overwrite`);
+    }
+    this.result[keyToUse] = value;
+    this.flushResult();
+  }
+  setPlayerStatus(status, error) {
+    this.status = status;
+    this.errorInSetup = error;
+  }
+  notifyCurrentTaskStatusChange(taskIndex) {
+    const taskIndexToNotify = typeof taskIndex === "number" ? taskIndex : this.currentTaskIndex;
+    if (typeof taskIndexToNotify !== "number") {
+      return;
+    }
+    const taskStatus = this.taskStatusList[taskIndexToNotify];
+    if (this.onTaskStatusChange) {
+      this.onTaskStatusChange(taskStatus);
+    }
+  }
+  async setTaskStatus(index, statusValue, error) {
+    this.taskStatusList[index].status = statusValue;
+    if (error) {
+      this.taskStatusList[index].error = error;
+    }
+    this.notifyCurrentTaskStatusChange(index);
+  }
+  setTaskIndex(taskIndex) {
+    this.currentTaskIndex = taskIndex;
+  }
+  flushResult() {
+    if (Object.keys(this.result).length && this.output) {
+      const output = (0, import_node_path.resolve)(process.cwd(), this.output);
+      const outputDir = (0, import_node_path.dirname)(output);
+      if (!(0, import_node_fs.existsSync)(outputDir)) {
+        (0, import_node_fs.mkdirSync)(outputDir, { recursive: true });
+      }
+      (0, import_node_fs.writeFileSync)(output, JSON.stringify(this.result, void 0, 2));
+    }
+  }
+  async playTask(taskStatus, agent) {
+    const { flow } = taskStatus;
+    (0, import_utils.assert)(flow, "missing flow in task");
+    for (const flowItemIndex in flow) {
+      const currentStep = Number.parseInt(flowItemIndex, 10);
+      taskStatus.currentStep = currentStep;
+      const flowItem = flow[flowItemIndex];
+      if ("aiAction" in flowItem || "ai" in flowItem) {
+        const actionTask = flowItem;
+        const prompt = actionTask.aiAction || actionTask.ai;
+        (0, import_utils.assert)(prompt, "missing prompt for ai (aiAction)");
+        (0, import_utils.assert)(
+          typeof prompt === "string",
+          "prompt for aiAction must be a string"
+        );
+        await agent.aiAction(prompt, {
+          cacheable: actionTask.cacheable
+        });
+      } else if ("aiAssert" in flowItem) {
+        const assertTask = flowItem;
+        const prompt = assertTask.aiAssert;
+        (0, import_utils.assert)(prompt, "missing prompt for aiAssert");
+        (0, import_utils.assert)(
+          typeof prompt === "string",
+          "prompt for aiAssert must be a string"
+        );
+        await agent.aiAssert(prompt);
+      } else if ("aiQuery" in flowItem) {
+        const queryTask = flowItem;
+        const prompt = queryTask.aiQuery;
+        (0, import_utils.assert)(prompt, "missing prompt for aiQuery");
+        (0, import_utils.assert)(
+          typeof prompt === "string",
+          "prompt for aiQuery must be a string"
+        );
+        const queryResult = await agent.aiQuery(prompt);
+        this.setResult(queryTask.name, queryResult);
+      } else if ("aiNumber" in flowItem) {
+        const numberTask = flowItem;
+        const prompt = numberTask.aiNumber;
+        (0, import_utils.assert)(prompt, "missing prompt for number");
+        (0, import_utils.assert)(
+          typeof prompt === "string",
+          "prompt for number must be a string"
+        );
+        const numberResult = await agent.aiNumber(prompt);
+        this.setResult(numberTask.name, numberResult);
+      } else if ("aiString" in flowItem) {
+        const stringTask = flowItem;
+        const prompt = stringTask.aiString;
+        (0, import_utils.assert)(prompt, "missing prompt for string");
+        (0, import_utils.assert)(
+          typeof prompt === "string",
+          "prompt for string must be a string"
+        );
+        const stringResult = await agent.aiString(prompt);
+        this.setResult(stringTask.name, stringResult);
+      } else if ("aiBoolean" in flowItem) {
+        const booleanTask = flowItem;
+        const prompt = booleanTask.aiBoolean;
+        (0, import_utils.assert)(prompt, "missing prompt for boolean");
+        (0, import_utils.assert)(
+          typeof prompt === "string",
+          "prompt for boolean must be a string"
+        );
+        const booleanResult = await agent.aiBoolean(prompt);
+        this.setResult(booleanTask.name, booleanResult);
+      } else if ("aiLocate" in flowItem) {
+        const locateTask = flowItem;
+        const prompt = locateTask.aiLocate;
+        (0, import_utils.assert)(prompt, "missing prompt for aiLocate");
+        (0, import_utils.assert)(
+          typeof prompt === "string",
+          "prompt for aiLocate must be a string"
+        );
+        const locateResult = await agent.aiLocate(prompt);
+        this.setResult(locateTask.name, locateResult);
+      } else if ("aiWaitFor" in flowItem) {
+        const waitForTask = flowItem;
+        const prompt = waitForTask.aiWaitFor;
+        (0, import_utils.assert)(prompt, "missing prompt for aiWaitFor");
+        (0, import_utils.assert)(
+          typeof prompt === "string",
+          "prompt for aiWaitFor must be a string"
+        );
+        const timeout = waitForTask.timeout;
+        await agent.aiWaitFor(prompt, { timeoutMs: timeout });
+      } else if ("sleep" in flowItem) {
+        const sleepTask = flowItem;
+        const ms = sleepTask.sleep;
+        let msNumber = ms;
+        if (typeof ms === "string") {
+          msNumber = Number.parseInt(ms, 10);
+        }
+        (0, import_utils.assert)(
+          msNumber && msNumber > 0,
+          `ms for sleep must be greater than 0, but got ${ms}`
+        );
+        await new Promise((resolve2) => setTimeout(resolve2, msNumber));
+      } else if ("aiTap" in flowItem) {
+        const tapTask = flowItem;
+        await agent.aiTap(tapTask.aiTap, tapTask);
+      } else if ("aiHover" in flowItem) {
+        const hoverTask = flowItem;
+        await agent.aiHover(hoverTask.aiHover, hoverTask);
+      } else if ("aiInput" in flowItem) {
+        const inputTask = flowItem;
+        await agent.aiInput(inputTask.aiInput, inputTask.locate, inputTask);
+      } else if ("aiKeyboardPress" in flowItem) {
+        const keyboardPressTask = flowItem;
+        await agent.aiKeyboardPress(
+          keyboardPressTask.aiKeyboardPress,
+          keyboardPressTask.locate,
+          keyboardPressTask
+        );
+      } else if ("aiScroll" in flowItem) {
+        const scrollTask = flowItem;
+        await agent.aiScroll(scrollTask, scrollTask.locate, scrollTask);
+      } else if ("javascript" in flowItem) {
+        const evaluateJavaScriptTask = flowItem;
+        const result = await agent.evaluateJavaScript(
+          evaluateJavaScriptTask.javascript
+        );
+        this.setResult(evaluateJavaScriptTask.name, result);
+      } else {
+        throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
+      }
+    }
+    this.reportFile = agent.reportFile;
+  }
+  async run() {
+    const { target, web, android, tasks } = this.script;
+    const webEnv = web || target;
+    const androidEnv = android;
+    const platform = webEnv || androidEnv;
+    this.setPlayerStatus("running");
+    let agent = null;
+    let freeFn = [];
+    try {
+      const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(
+        platform
+      );
+      agent = newAgent;
+      const originalOnTaskStartTip = agent.onTaskStartTip;
+      agent.onTaskStartTip = (tip) => {
+        if (this.status === "running") {
+          this.agentStatusTip = tip;
+        }
+        originalOnTaskStartTip?.(tip);
+      };
+      freeFn = [
+        ...newFreeFn || [],
+        {
+          name: "restore-agent-onTaskStartTip",
+          fn: () => {
+            if (agent) {
+              agent.onTaskStartTip = originalOnTaskStartTip;
+            }
+          }
+        }
+      ];
+    } catch (e) {
+      this.setPlayerStatus("error", e);
+      return;
+    }
+    this.pageAgent = agent;
+    let taskIndex = 0;
+    this.setPlayerStatus("running");
+    let errorFlag = false;
+    while (taskIndex < tasks.length) {
+      const taskStatus = this.taskStatusList[taskIndex];
+      this.setTaskStatus(taskIndex, "running");
+      this.setTaskIndex(taskIndex);
+      try {
+        await this.playTask(taskStatus, this.pageAgent);
+        this.setTaskStatus(taskIndex, "done");
+      } catch (e) {
+        this.setTaskStatus(taskIndex, "error", e);
+        if (taskStatus.continueOnError) {
+        } else {
+          this.reportFile = agent.reportFile;
+          errorFlag = true;
+          break;
+        }
+      }
+      this.reportFile = agent.reportFile;
+      taskIndex++;
+    }
+    if (errorFlag) {
+      this.setPlayerStatus("error");
+    } else {
+      this.setPlayerStatus("done");
+    }
+    this.agentStatusTip = "";
+    for (const fn of freeFn) {
+      try {
+        await fn.fn();
+      } catch (e) {
+      }
+    }
+  }
+};
+
+// src/yaml/builder.ts
+var import_js_yaml = __toESM(require("js-yaml"));
+
+// src/yaml/utils.ts
+var import_utils2 = require("misoai-shared/utils");
+var import_js_yaml2 = __toESM(require("js-yaml"));
+function interpolateEnvVars(content) {
+  return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
+    const value = process.env[envVar.trim()];
+    if (value === void 0) {
+      throw new Error(`Environment variable "${envVar.trim()}" is not defined`);
+    }
+    return value;
+  });
+}
+function parseYamlScript(content, filePath, ignoreCheckingTarget) {
+  let processedContent = content;
+  if (content.indexOf("android") !== -1 && content.match(/deviceId:\s*(\d+)/)) {
+    let matchedDeviceId;
+    processedContent = content.replace(
+      /deviceId:\s*(\d+)/g,
+      (match, deviceId) => {
+        matchedDeviceId = deviceId;
+        return `deviceId: '${deviceId}'`;
+      }
+    );
+    console.warn(
+      `please use string-style deviceId in yaml script, for example: deviceId: "${matchedDeviceId}"`
+    );
+  }
+  const interpolatedContent = interpolateEnvVars(processedContent);
+  const obj = import_js_yaml2.default.load(interpolatedContent, {
+    schema: import_js_yaml2.default.JSON_SCHEMA
+  });
+  const pathTip = filePath ? `, failed to load ${filePath}` : "";
+  const android = typeof obj.android !== "undefined" ? Object.assign({}, obj.android || {}) : void 0;
+  const webConfig = obj.web || obj.target;
+  const web = typeof webConfig !== "undefined" ? Object.assign({}, webConfig || {}) : void 0;
+  if (!ignoreCheckingTarget) {
+    (0, import_utils2.assert)(
+      web || android,
+      `at least one of "target", "web", or "android" properties is required in yaml script${pathTip}`
+    );
+    (0, import_utils2.assert)(
+      web && !android || !web && android,
+      `only one of "target", "web", or "android" properties is allowed in yaml script${pathTip}`
+    );
+    if (web || android) {
+      (0, import_utils2.assert)(
+        typeof web === "object" || typeof android === "object",
+        `property "target/web/android" must be an object${pathTip}`
+      );
+    }
+  }
+  (0, import_utils2.assert)(obj.tasks, `property "tasks" is required in yaml script ${pathTip}`);
+  (0, import_utils2.assert)(
+    Array.isArray(obj.tasks),
+    `property "tasks" must be an array in yaml script, but got ${obj.tasks}`
+  );
+  return obj;
+}
+
+// src/common/agent.ts
+var import_utils11 = require("misoai-core/utils");
+var import_constants2 = require("misoai-shared/constants");
+var import_env2 = require("misoai-shared/env");
+var import_logger4 = require("misoai-shared/logger");
+var import_utils12 = require("misoai-shared/utils");
+
+// src/common/tasks.ts
+var import_misoai_core = require("misoai-core");
+var import_ai_model2 = require("misoai-core/ai-model");
+var import_utils5 = require("misoai-core/utils");
+var import_constants = require("misoai-shared/constants");
+var import_logger = require("misoai-shared/logger");
+var import_utils6 = require("misoai-shared/utils");
+
+// src/common/ui-utils.ts
+function typeStr(task) {
+  return task.subType && task.subType !== "Plan" ? `${task.type} / ${task.subType || ""}` : task.type;
+}
+function getKeyCommands(value) {
+  const keys = Array.isArray(value) ? value : [value];
+  return keys.reduce((acc, k) => {
+    const includeMeta = keys.includes("Meta") || keys.includes("Control");
+    if (includeMeta && (k === "a" || k === "A")) {
+      return acc.concat([{ key: k, command: "SelectAll" }]);
+    }
+    if (includeMeta && (k === "c" || k === "C")) {
+      return acc.concat([{ key: k, command: "Copy" }]);
+    }
+    if (includeMeta && (k === "v" || k === "V")) {
+      return acc.concat([{ key: k, command: "Paste" }]);
+    }
+    return acc.concat([{ key: k }]);
+  }, []);
+}
+function locateParamStr(locate) {
+  if (!locate) {
+    return "";
+  }
+  if (typeof locate === "string") {
+    return locate;
+  }
+  return locate.prompt;
+}
+function scrollParamStr(scrollParam) {
+  if (!scrollParam) {
+    return "";
+  }
+  return `${scrollParam.direction || "down"}, ${scrollParam.scrollType || "once"}, ${scrollParam.distance || "distance-not-set"}`;
+}
+function taskTitleStr(type, prompt) {
+  if (prompt) {
+    return `${type} - ${prompt}`;
+  }
+  return type;
+}
+function paramStr(task) {
+  let value;
+  if (task.type === "Planning") {
+    value = task?.param?.userInstruction;
+  }
+  if (task.type === "Insight") {
+    value = task?.param?.prompt || task?.param?.id || task?.param?.dataDemand || task?.param?.assertion;
+  }
+  if (task.type === "Action") {
+    const locate = task?.locate;
+    const locateStr = locate ? locateParamStr(locate) : "";
+    value = task.thought || "";
+    if (typeof task?.param?.timeMs === "number") {
+      value = `${task?.param?.timeMs}ms`;
+    } else if (typeof task?.param?.scrollType === "string") {
+      value = scrollParamStr(task?.param);
+    } else if (typeof task?.param?.value !== "undefined") {
+      value = task?.param?.value;
+    }
+    if (locateStr) {
+      if (value) {
+        value = `${locateStr} - ${value}`;
+      } else {
+        value = locateStr;
+      }
+    }
+  }
+  if (typeof value === "undefined")
+    return "";
+  return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
+}
+
+// src/common/utils.ts
+var import_ai_model = require("misoai-core/ai-model");
+var import_utils3 = require("misoai-core/utils");
+var import_env = require("misoai-shared/env");
+var import_extractor = require("misoai-shared/extractor");
+var import_img = require("misoai-shared/img");
+var import_utils4 = require("misoai-shared/utils");
+var import_dayjs = __toESM(require("dayjs"));
+
+// src/web-element.ts
+var WebElementInfo = class {
+  constructor({
+    content,
+    rect,
+    // page,
+    locator,
+    id,
+    attributes,
+    indexId,
+    xpaths
+  }) {
+    this.content = content;
+    this.rect = rect;
+    this.center = [
+      Math.floor(rect.left + rect.width / 2),
+      Math.floor(rect.top + rect.height / 2)
+    ];
+    this.locator = locator;
+    this.id = id;
+    this.attributes = attributes;
+    this.indexId = indexId;
+    this.xpaths = xpaths;
+  }
+};
+
+// src/common/utils.ts
+async function parseContextFromWebPage(page, _opt) {
+  (0, import_utils4.assert)(page, "page is required");
+  if (page._forceUsePageContext) {
+    return await page._forceUsePageContext();
+  }
+  const url = await page.url();
+  (0, import_utils3.uploadTestInfoToServer)({ testUrl: url });
+  let screenshotBase64;
+  let tree;
+  await Promise.all([
+    page.screenshotBase64().then((base64) => {
+      screenshotBase64 = base64;
+    }),
+    page.getElementsNodeTree().then(async (treeRoot) => {
+      tree = treeRoot;
+    })
+  ]);
+  const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
+    const { rect, id, content, attributes, locator, indexId } = elementInfo;
+    return new WebElementInfo({
+      rect,
+      locator,
+      id,
+      content,
+      attributes,
+      indexId
+    });
+  });
+  (0, import_utils4.assert)(screenshotBase64, "screenshotBase64 is required");
+  const elementsInfo = (0, import_extractor.treeToList)(webTree);
+  const size = await page.size();
+  if (size.dpr && size.dpr > 1) {
+    screenshotBase64 = await (0, import_img.resizeImgBase64)(screenshotBase64, {
+      width: size.width,
+      height: size.height
+    });
+  }
+  return {
+    content: elementsInfo,
+    tree: webTree,
+    size,
+    screenshotBase64,
+    url
+  };
+}
+function reportFileName(tag = "web") {
+  const reportTagName = (0, import_env.getAIConfig)(import_env.MIDSCENE_REPORT_TAG_NAME);
+  const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss");
+  const uniqueId = (0, import_utils4.uuid)().substring(0, 8);
+  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;
+}
+function printReportMsg(filepath) {
+  (0, import_utils4.logMsg)(`Midscene - report file updated: ${filepath}`);
+}
+function replaceIllegalPathCharsAndSpace(str) {
+  return str.replace(/[/\\:*?"<>| ]/g, "-");
+}
+function forceClosePopup(page, debug7) {
+  page.on("popup", async (popup) => {
+    if (!popup) {
+      console.warn("got a popup event, but the popup is not ready yet, skip");
+      return;
+    }
+    const url = await popup.url();
+    console.log(`Popup opened: ${url}`);
+    if (!popup.isClosed()) {
+      try {
+        await popup.close();
+      } catch (error) {
+        debug7(`failed to close popup ${url}, error: ${error}`);
+      }
+    } else {
+      debug7(`popup is already closed, skip close ${url}`);
+    }
+    if (!page.isClosed()) {
+      try {
+        await page.goto(url);
+      } catch (error) {
+        debug7(`failed to goto ${url}, error: ${error}`);
+      }
+    } else {
+      debug7(`page is already closed, skip goto ${url}`);
+    }
+  });
+}
+function matchElementFromPlan(planLocateParam, tree) {
+  if (!planLocateParam) {
+    return void 0;
+  }
+  if (planLocateParam.id) {
+    return (0, import_extractor.getNodeFromCacheList)(planLocateParam.id);
+  }
+  if (planLocateParam.bbox) {
+    const centerPosition = {
+      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),
+      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2)
+    };
+    let element = (0, import_ai_model.elementByPositionWithElementInfo)(tree, centerPosition);
+    if (!element) {
+      element = (0, import_extractor.generateElementByPosition)(centerPosition);
+    }
+    return element;
+  }
+  return void 0;
+}
+
+// src/common/tasks.ts
+var debug = (0, import_logger.getDebug)("page-task-executor");
+var replanningCountLimit = 10;
+var isAndroidPage = (page) => {
+  return page.pageType === "android";
+};
+var PageTaskExecutor = class {
+  constructor(page, insight, opts) {
+    this.conversationHistory = [];
+    this.page = page;
+    this.insight = insight;
+    this.taskCache = opts.taskCache;
+    this.onTaskStartCallback = opts?.onTaskStart;
+  }
+  async recordScreenshot(timing) {
+    const base64 = await this.page.screenshotBase64();
+    const item = {
+      type: "screenshot",
+      ts: Date.now(),
+      screenshot: base64,
+      timing
+    };
+    return item;
+  }
+  async getElementXpath(pageContext, element) {
+    let elementId = element?.id;
+    if (element?.attributes?.nodeType === import_constants.NodeType.POSITION) {
+      await this.insight.contextRetrieverFn("locate");
+      const info = (0, import_ai_model2.elementByPositionWithElementInfo)(
+        pageContext.tree,
+        {
+          x: element.center[0],
+          y: element.center[1]
+        },
+        {
+          requireStrictDistance: false,
+          filterPositionElements: true
+        }
+      );
+      if (info?.id) {
+        elementId = info.id;
+      } else {
+        debug(
+          "no element id found for position node, will not update cache",
+          element
+        );
+      }
+    }
+    if (!elementId) {
+      return void 0;
+    }
+    try {
+      const result = await this.page.getXpathsById(elementId);
+      return result;
+    } catch (error) {
+      debug("getXpathsById error: ", error);
+    }
+  }
+  prependExecutorWithScreenshot(taskApply, appendAfterExecution = false) {
+    const taskWithScreenshot = {
+      ...taskApply,
+      executor: async (param, context, ...args) => {
+        const recorder = [];
+        const { task } = context;
+        task.recorder = recorder;
+        const shot = await this.recordScreenshot(`before ${task.type}`);
+        recorder.push(shot);
+        const result = await taskApply.executor(param, context, ...args);
+        if (taskApply.type === "Action") {
+          await Promise.all([
+            (async () => {
+              await (0, import_utils5.sleep)(100);
+              if (this.page.waitUntilNetworkIdle) {
+                try {
+                  await this.page.waitUntilNetworkIdle();
+                } catch (error) {
+                }
+              }
+            })(),
+            (0, import_utils5.sleep)(200)
+          ]);
+        }
+        if (appendAfterExecution) {
+          const shot2 = await this.recordScreenshot("after Action");
+          recorder.push(shot2);
+        }
+        return result;
+      }
+    };
+    return taskWithScreenshot;
+  }
+  async convertPlanToExecutable(plans, opts) {
+    const tasks = [];
+    plans.forEach((plan2) => {
+      if (plan2.type === "Locate") {
+        if (plan2.locate === null || plan2.locate?.id === null || plan2.locate?.id === "null") {
+          return;
+        }
+        const taskFind = {
+          type: "Insight",
+          subType: "Locate",
+          param: plan2.locate ? {
+            ...plan2.locate,
+            cacheable: opts?.cacheable
+          } : void 0,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param, taskContext) => {
+            const { task } = taskContext;
+            (0, import_utils6.assert)(
+              param?.prompt || param?.id || param?.bbox,
+              "No prompt or id or position or bbox to locate"
+            );
+            let insightDump;
+            let usage;
+            const dumpCollector = (dump) => {
+              insightDump = dump;
+              usage = dump?.taskInfo?.usage;
+              task.log = {
+                dump: insightDump
+              };
+              task.usage = usage;
+            };
+            this.insight.onceDumpUpdatedFn = dumpCollector;
+            const shotTime = Date.now();
+            const pageContext = await this.insight.contextRetrieverFn("locate");
+            task.pageContext = pageContext;
+            const recordItem = {
+              type: "screenshot",
+              ts: shotTime,
+              screenshot: pageContext.screenshotBase64,
+              timing: "before locate"
+            };
+            task.recorder = [recordItem];
+            let cacheHitFlag = false;
+            const cachePrompt = param.prompt;
+            const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);
+            const xpaths = locateCacheRecord?.cacheContent?.xpaths;
+            let elementFromCache = null;
+            try {
+              if (xpaths?.length && this.taskCache?.isCacheResultUsed && param?.cacheable !== false) {
+                for (let i = 0; i < xpaths.length; i++) {
+                  const element2 = await this.page.getElementInfoByXpath(
+                    xpaths[i]
+                  );
+                  if (element2?.id) {
+                    elementFromCache = element2;
+                    debug("cache hit, prompt: %s", cachePrompt);
+                    cacheHitFlag = true;
+                    debug(
+                      "found a new new element with same xpath, xpath: %s, id: %s",
+                      xpaths[i],
+                      element2?.id
+                    );
+                    break;
+                  }
+                }
+              }
+            } catch (error) {
+              debug("get element info by xpath error: ", error);
+            }
+            const startTime = Date.now();
+            const element = elementFromCache || // try to match element from cache
+            matchElementFromPlan(param, pageContext.tree) || // try to match element from plan
+            (await this.insight.locate(param, {
+              context: pageContext
+            })).element;
+            const aiCost = Date.now() - startTime;
+            let currentXpaths;
+            if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) {
+              const elementXpaths = await this.getElementXpath(
+                pageContext,
+                element
+              );
+              if (elementXpaths?.length) {
+                currentXpaths = elementXpaths;
+                this.taskCache.updateOrAppendCacheRecord(
+                  {
+                    type: "locate",
+                    prompt: cachePrompt,
+                    xpaths: elementXpaths
+                  },
+                  locateCacheRecord
+                );
+              } else {
+                debug(
+                  "no xpaths found, will not update cache",
+                  cachePrompt,
+                  elementXpaths
+                );
+              }
+            }
+            if (!element) {
+              throw new Error(`Element not found: ${param.prompt}`);
+            }
+            return {
+              output: {
+                element
+              },
+              pageContext,
+              cache: {
+                hit: cacheHitFlag,
+                originalXpaths: xpaths,
+                currentXpaths
+              },
+              aiCost
+            };
+          }
+        };
+        tasks.push(taskFind);
+      } else if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
+        const assertPlan = plan2;
+        const taskAssert = {
+          type: "Insight",
+          subType: "Assert",
+          param: assertPlan.param,
+          thought: assertPlan.thought,
+          locate: assertPlan.locate,
+          executor: async (param, taskContext) => {
+            const { task } = taskContext;
+            let insightDump;
+            const dumpCollector = (dump) => {
+              insightDump = dump;
+            };
+            this.insight.onceDumpUpdatedFn = dumpCollector;
+            const assertion = await this.insight.assert(
+              assertPlan.param.assertion
+            );
+            if (!assertion.pass) {
+              if (plan2.type === "Assert") {
+                task.output = assertion;
+                task.log = {
+                  dump: insightDump
+                };
+                throw new Error(
+                  assertion.thought || "Assertion failed without reason"
+                );
+              }
+              task.error = assertion.thought;
+            }
+            return {
+              output: assertion,
+              log: {
+                dump: insightDump
+              },
+              usage: assertion.usage
+            };
+          }
+        };
+        tasks.push(taskAssert);
+      } else if (plan2.type === "Input") {
+        const taskActionInput = {
+          type: "Action",
+          subType: "Input",
+          param: plan2.param,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (taskParam, { element }) => {
+            if (element) {
+              await this.page.clearInput(element);
+              if (!taskParam || !taskParam.value) {
+                return;
+              }
+              await this.page.keyboard.type(taskParam.value);
+            } else {
+              await this.page.keyboard.type(taskParam.value);
+            }
+          }
+        };
+        tasks.push(taskActionInput);
+      } else if (plan2.type === "KeyboardPress") {
+        const taskActionKeyboardPress = {
+          type: "Action",
+          subType: "KeyboardPress",
+          param: plan2.param,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (taskParam) => {
+            const keys = getKeyCommands(taskParam.value);
+            await this.page.keyboard.press(keys);
+          }
+        };
+        tasks.push(taskActionKeyboardPress);
+      } else if (plan2.type === "Tap") {
+        const taskActionTap = {
+          type: "Action",
+          subType: "Tap",
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param, { element }) => {
+            (0, import_utils6.assert)(element, "Element not found, cannot tap");
+            await this.page.mouse.click(element.center[0], element.center[1]);
+          }
+        };
+        tasks.push(taskActionTap);
+      } else if (plan2.type === "Drag") {
+        const taskActionDrag = {
+          type: "Action",
+          subType: "Drag",
+          param: plan2.param,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (taskParam) => {
+            (0, import_utils6.assert)(
+              taskParam?.start_box && taskParam?.end_box,
+              "No start_box or end_box to drag"
+            );
+            await this.page.mouse.drag(taskParam.start_box, taskParam.end_box);
+          }
+        };
+        tasks.push(taskActionDrag);
+      } else if (plan2.type === "Hover") {
+        const taskActionHover = {
+          type: "Action",
+          subType: "Hover",
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param, { element }) => {
+            (0, import_utils6.assert)(element, "Element not found, cannot hover");
+            await this.page.mouse.move(element.center[0], element.center[1]);
+          }
+        };
+        tasks.push(taskActionHover);
+      } else if (plan2.type === "Scroll") {
+        const taskActionScroll = {
+          type: "Action",
+          subType: "Scroll",
+          param: plan2.param,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (taskParam, { element }) => {
+            const startingPoint = element ? {
+              left: element.center[0],
+              top: element.center[1]
+            } : void 0;
+            const scrollToEventName = taskParam?.scrollType;
+            if (scrollToEventName === "untilTop") {
+              await this.page.scrollUntilTop(startingPoint);
+            } else if (scrollToEventName === "untilBottom") {
+              await this.page.scrollUntilBottom(startingPoint);
+            } else if (scrollToEventName === "untilRight") {
+              await this.page.scrollUntilRight(startingPoint);
+            } else if (scrollToEventName === "untilLeft") {
+              await this.page.scrollUntilLeft(startingPoint);
+            } else if (scrollToEventName === "once" || !scrollToEventName) {
+              if (taskParam?.direction === "down" || !taskParam || !taskParam.direction) {
+                await this.page.scrollDown(
+                  taskParam?.distance || void 0,
+                  startingPoint
+                );
+              } else if (taskParam.direction === "up") {
+                await this.page.scrollUp(
+                  taskParam.distance || void 0,
+                  startingPoint
+                );
+              } else if (taskParam.direction === "left") {
+                await this.page.scrollLeft(
+                  taskParam.distance || void 0,
+                  startingPoint
+                );
+              } else if (taskParam.direction === "right") {
+                await this.page.scrollRight(
+                  taskParam.distance || void 0,
+                  startingPoint
+                );
+              } else {
+                throw new Error(
+                  `Unknown scroll direction: ${taskParam.direction}`
+                );
+              }
+              await (0, import_utils5.sleep)(500);
+            } else {
+              throw new Error(
+                `Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
+                  taskParam
+                )}`
+              );
+            }
+          }
+        };
+        tasks.push(taskActionScroll);
+      } else if (plan2.type === "Sleep") {
+        const taskActionSleep = {
+          type: "Action",
+          subType: "Sleep",
+          param: plan2.param,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (taskParam) => {
+            await (0, import_utils5.sleep)(taskParam?.timeMs || 3e3);
+          }
+        };
+        tasks.push(taskActionSleep);
+      } else if (plan2.type === "Error") {
+        const taskActionError = {
+          type: "Action",
+          subType: "Error",
+          param: plan2.param,
+          thought: plan2.thought || plan2.param?.thought,
+          locate: plan2.locate,
+          executor: async () => {
+            throw new Error(
+              plan2?.thought || plan2.param?.thought || "error without thought"
+            );
+          }
+        };
+        tasks.push(taskActionError);
+      } else if (plan2.type === "ExpectedFalsyCondition") {
+        const taskActionFalsyConditionStatement = {
+          type: "Action",
+          subType: "ExpectedFalsyCondition",
+          param: null,
+          thought: plan2.param?.reason,
+          locate: plan2.locate,
+          executor: async () => {
+          }
+        };
+        tasks.push(taskActionFalsyConditionStatement);
+      } else if (plan2.type === "Finished") {
+        const taskActionFinished = {
+          type: "Action",
+          subType: "Finished",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param) => {
+          }
+        };
+        tasks.push(taskActionFinished);
+      } else if (plan2.type === "AndroidHomeButton") {
+        const taskActionAndroidHomeButton = {
+          type: "Action",
+          subType: "AndroidHomeButton",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param) => {
+            (0, import_utils6.assert)(
+              isAndroidPage(this.page),
+              "Cannot use home button on non-Android devices"
+            );
+            await this.page.home();
+          }
+        };
+        tasks.push(taskActionAndroidHomeButton);
+      } else if (plan2.type === "AndroidBackButton") {
+        const taskActionAndroidBackButton = {
+          type: "Action",
+          subType: "AndroidBackButton",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param) => {
+            (0, import_utils6.assert)(
+              isAndroidPage(this.page),
+              "Cannot use back button on non-Android devices"
+            );
+            await this.page.back();
+          }
+        };
+        tasks.push(taskActionAndroidBackButton);
+      } else if (plan2.type === "AndroidRecentAppsButton") {
+        const taskActionAndroidRecentAppsButton = {
+          type: "Action",
+          subType: "AndroidRecentAppsButton",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param) => {
+            (0, import_utils6.assert)(
+              isAndroidPage(this.page),
+              "Cannot use recent apps button on non-Android devices"
+            );
+            await this.page.recentApps();
+          }
+        };
+        tasks.push(taskActionAndroidRecentAppsButton);
+      } else {
+        throw new Error(`Unknown or unsupported task type: ${plan2.type}`);
+      }
+    });
+    const wrappedTasks = tasks.map(
+      (task, index) => {
+        if (task.type === "Action") {
+          return this.prependExecutorWithScreenshot(
+            task,
+            index === tasks.length - 1
+          );
+        }
+        return task;
+      }
+    );
+    return {
+      tasks: wrappedTasks
+    };
+  }
+  async setupPlanningContext(executorContext) {
+    const shotTime = Date.now();
+    const pageContext = await this.insight.contextRetrieverFn("locate");
+    const recordItem = {
+      type: "screenshot",
+      ts: shotTime,
+      screenshot: pageContext.screenshotBase64,
+      timing: "before planning"
+    };
+    executorContext.task.recorder = [recordItem];
+    executorContext.task.pageContext = pageContext;
+    return {
+      pageContext
+    };
+  }
+  async loadYamlFlowAsPlanning(userInstruction, yamlString) {
+    const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userInstruction), {
+      onTaskStart: this.onTaskStartCallback
+    });
+    const task = {
+      type: "Planning",
+      subType: "LoadYaml",
+      locate: null,
+      param: {
+        userInstruction
+      },
+      executor: async (param, executorContext) => {
+        await this.setupPlanningContext(executorContext);
+        return {
+          output: {
+            actions: [],
+            more_actions_needed_by_instruction: false,
+            log: "",
+            yamlString
+          },
+          cache: {
+            hit: true
+          }
+        };
+      }
+    };
+    await taskExecutor.append(task);
+    await taskExecutor.flush();
+    return {
+      executor: taskExecutor
+    };
+  }
+  planningTaskFromPrompt(userInstruction, log, actionContext) {
+    const task = {
+      type: "Planning",
+      subType: "Plan",
+      locate: null,
+      param: {
+        userInstruction,
+        log
+      },
+      executor: async (param, executorContext) => {
+        const startTime = Date.now();
+        const { pageContext } = await this.setupPlanningContext(executorContext);
+        const planResult = await (0, import_misoai_core.plan)(param.userInstruction, {
+          context: pageContext,
+          log: param.log,
+          actionContext,
+          pageType: this.page.pageType
+        });
+        const {
+          actions,
+          log: log2,
+          more_actions_needed_by_instruction,
+          error,
+          usage,
+          rawResponse,
+          sleep: sleep3
+        } = planResult;
+        executorContext.task.log = {
+          ...executorContext.task.log || {},
+          rawResponse
+        };
+        executorContext.task.usage = usage;
+        let stopCollecting = false;
+        let bboxCollected = false;
+        let planParsingError = "";
+        const finalActions = (actions || []).reduce(
+          (acc, planningAction) => {
+            if (stopCollecting) {
+              return acc;
+            }
+            if (planningAction.locate) {
+              if (bboxCollected && planningAction.locate.bbox) {
+                delete planningAction.locate.bbox;
+              }
+              if (planningAction.locate.bbox) {
+                bboxCollected = true;
+              }
+              acc.push({
+                type: "Locate",
+                locate: planningAction.locate,
+                param: null,
+                thought: planningAction.locate.prompt
+              });
+            } else if (["Tap", "Hover", "Input"].includes(planningAction.type)) {
+              planParsingError = `invalid planning response: ${JSON.stringify(planningAction)}`;
+              stopCollecting = true;
+              return acc;
+            }
+            acc.push(planningAction);
+            return acc;
+          },
+          []
+        );
+        if (sleep3) {
+          const timeNow = Date.now();
+          const timeRemaining = sleep3 - (timeNow - startTime);
+          if (timeRemaining > 0) {
+            finalActions.push({
+              type: "Sleep",
+              param: {
+                timeMs: timeRemaining
+              },
+              locate: null
+            });
+          }
+        }
+        if (finalActions.length === 0) {
+          (0, import_utils6.assert)(
+            !more_actions_needed_by_instruction || sleep3,
+            error ? `Failed to plan: ${error}` : planParsingError || "No plan found"
+          );
+        }
+        return {
+          output: {
+            actions: finalActions,
+            more_actions_needed_by_instruction,
+            log: log2,
+            yamlFlow: planResult.yamlFlow
+          },
+          cache: {
+            hit: false
+          },
+          pageContext
+        };
+      }
+    };
+    return task;
+  }
+  planningTaskToGoal(userInstruction) {
+    const task = {
+      type: "Planning",
+      subType: "Plan",
+      locate: null,
+      param: {
+        userInstruction
+      },
+      executor: async (param, executorContext) => {
+        const { pageContext } = await this.setupPlanningContext(executorContext);
+        const imagePayload = await (0, import_ai_model2.resizeImageForUiTars)(
+          pageContext.screenshotBase64,
+          pageContext.size
+        );
+        this.appendConversationHistory({
+          role: "user",
+          content: [
+            {
+              type: "image_url",
+              image_url: {
+                url: imagePayload
+              }
+            }
+          ]
+        });
+        const startTime = Date.now();
+        const planResult = await (0, import_ai_model2.vlmPlanning)({
+          userInstruction: param.userInstruction,
+          conversationHistory: this.conversationHistory,
+          size: pageContext.size
+        });
+        const aiCost = Date.now() - startTime;
+        const { actions, action_summary } = planResult;
+        this.appendConversationHistory({
+          role: "assistant",
+          content: action_summary
+        });
+        return {
+          output: {
+            actions,
+            thought: actions[0]?.thought,
+            actionType: actions[0].type,
+            more_actions_needed_by_instruction: true,
+            log: "",
+            yamlFlow: planResult.yamlFlow
+          },
+          cache: {
+            hit: false
+          },
+          aiCost
+        };
+      }
+    };
+    return task;
+  }
+  async runPlans(title, plans, opts) {
+    const taskExecutor = new import_misoai_core.Executor(title, {
+      onTaskStart: this.onTaskStartCallback
+    });
+    const { tasks } = await this.convertPlanToExecutable(plans, opts);
+    await taskExecutor.append(tasks);
+    const result = await taskExecutor.flush();
+    return {
+      output: result,
+      executor: taskExecutor
+    };
+  }
+  async action(userPrompt, actionContext, opts) {
+    const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
+      onTaskStart: this.onTaskStartCallback
+    });
+    let planningTask = this.planningTaskFromPrompt(userPrompt, void 0, actionContext);
+    let replanCount = 0;
+    const logList = [];
+    const yamlFlow = [];
+    while (planningTask) {
+      if (replanCount > replanningCountLimit) {
+        const errorMsg = "Replanning too many times, please split the task into multiple steps";
+        return this.appendErrorPlan(taskExecutor, errorMsg);
+      }
+      await taskExecutor.append(planningTask);
+      const planResult = await taskExecutor.flush();
+      if (taskExecutor.isInErrorState()) {
+        return {
+          output: planResult,
+          executor: taskExecutor
+        };
+      }
+      const plans = planResult.actions || [];
+      yamlFlow.push(...planResult.yamlFlow || []);
+      let executables;
+      try {
+        executables = await this.convertPlanToExecutable(plans, opts);
+        taskExecutor.append(executables.tasks);
+      } catch (error) {
+        return this.appendErrorPlan(
+          taskExecutor,
+          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(
+            plans
+          )}`
+        );
+      }
+      await taskExecutor.flush();
+      if (taskExecutor.isInErrorState()) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      if (planResult?.log) {
+        logList.push(planResult.log);
+      }
+      if (!planResult.more_actions_needed_by_instruction) {
+        planningTask = null;
+        break;
+      }
+      planningTask = this.planningTaskFromPrompt(
+        userPrompt,
+        logList.length > 0 ? `- ${logList.join("\n- ")}` : void 0,
+        actionContext
+      );
+      replanCount++;
+    }
+    return {
+      output: {
+        yamlFlow
+      },
+      executor: taskExecutor
+    };
+  }
+  async actionToGoal(userPrompt, opts) {
+    const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
+      onTaskStart: this.onTaskStartCallback
+    });
+    this.conversationHistory = [];
+    const isCompleted = false;
+    let currentActionNumber = 0;
+    const maxActionNumber = 40;
+    const yamlFlow = [];
+    while (!isCompleted && currentActionNumber < maxActionNumber) {
+      currentActionNumber++;
+      const planningTask = this.planningTaskToGoal(userPrompt);
+      await taskExecutor.append(planningTask);
+      const output = await taskExecutor.flush();
+      if (taskExecutor.isInErrorState()) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      const plans = output.actions;
+      yamlFlow.push(...output.yamlFlow || []);
+      let executables;
+      try {
+        executables = await this.convertPlanToExecutable(plans, opts);
+        taskExecutor.append(executables.tasks);
+      } catch (error) {
+        return this.appendErrorPlan(
+          taskExecutor,
+          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(
+            plans
+          )}`
+        );
+      }
+      await taskExecutor.flush();
+      if (taskExecutor.isInErrorState()) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      if (plans[0].type === "Finished") {
+        break;
+      }
+    }
+    return {
+      output: {
+        yamlFlow
+      },
+      executor: taskExecutor
+    };
+  }
+  async createTypeQueryTask(type, demand) {
+    const taskExecutor = new import_misoai_core.Executor(
+      taskTitleStr(
+        type,
+        typeof demand === "string" ? demand : JSON.stringify(demand)
+      ),
+      {
+        onTaskStart: this.onTaskStartCallback
+      }
+    );
+    const queryTask = {
+      type: "Insight",
+      subType: type,
+      locate: null,
+      param: {
+        dataDemand: demand
+        // for user param presentation in report right sidebar
+      },
+      executor: async (param) => {
+        let insightDump;
+        const dumpCollector = (dump) => {
+          insightDump = dump;
+        };
+        this.insight.onceDumpUpdatedFn = dumpCollector;
+        const ifTypeRestricted = type !== "Query";
+        let demandInput = demand;
+        if (ifTypeRestricted) {
+          demandInput = {
+            result: `${type}, ${demand}`
+          };
+        }
+        const { data, usage } = await this.insight.extract(demandInput);
+        let outputResult = data;
+        if (ifTypeRestricted) {
+          (0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
+          outputResult = data.result;
+        }
+        return {
+          output: outputResult,
+          log: { dump: insightDump },
+          usage
+        };
+      }
+    };
+    await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));
+    const output = await taskExecutor.flush();
+    return {
+      output,
+      executor: taskExecutor
+    };
+  }
+  async query(demand) {
+    return this.createTypeQueryTask("Query", demand);
+  }
+  async boolean(prompt) {
+    return this.createTypeQueryTask("Boolean", prompt);
+  }
+  async number(prompt) {
+    return this.createTypeQueryTask("Number", prompt);
+  }
+  async string(prompt) {
+    return this.createTypeQueryTask("String", prompt);
+  }
+  async assert(assertion) {
+    const description = `assert: ${assertion}`;
+    const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Assert", description), {
+      onTaskStart: this.onTaskStartCallback
+    });
+    const assertionPlan = {
+      type: "Assert",
+      param: {
+        assertion
+      },
+      locate: null
+    };
+    const { tasks } = await this.convertPlanToExecutable([assertionPlan]);
+    await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));
+    const output = await taskExecutor.flush();
+    return {
+      output,
+      executor: taskExecutor
+    };
+  }
+  /**
+   * Append a message to the conversation history
+   * For user messages with images:
+   * - Keep max 4 user image messages in history
+   * - Remove oldest user image message when limit reached
+   * For assistant messages:
+   * - Simply append to history
+   * @param conversationHistory Message to append
+   */
+  appendConversationHistory(conversationHistory) {
+    if (conversationHistory.role === "user") {
+      const userImgItems = this.conversationHistory.filter(
+        (item) => item.role === "user"
+      );
+      if (userImgItems.length >= 4 && conversationHistory.role === "user") {
+        const firstUserImgIndex = this.conversationHistory.findIndex(
+          (item) => item.role === "user"
+        );
+        if (firstUserImgIndex >= 0) {
+          this.conversationHistory.splice(firstUserImgIndex, 1);
+        }
+      }
+    }
+    this.conversationHistory.push(conversationHistory);
+  }
+  async appendErrorPlan(taskExecutor, errorMsg) {
+    const errorPlan = {
+      type: "Error",
+      param: {
+        thought: errorMsg
+      },
+      locate: null
+    };
+    const { tasks } = await this.convertPlanToExecutable([errorPlan]);
+    await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));
+    await taskExecutor.flush();
+    return {
+      output: void 0,
+      executor: taskExecutor
+    };
+  }
+  async waitFor(assertion, opt) {
+    const description = `waitFor: ${assertion}`;
+    const taskExecutor = new import_misoai_core.Executor(taskTitleStr("WaitFor", description), {
+      onTaskStart: this.onTaskStartCallback
+    });
+    const { timeoutMs, checkIntervalMs } = opt;
+    (0, import_utils6.assert)(assertion, "No assertion for waitFor");
+    (0, import_utils6.assert)(timeoutMs, "No timeoutMs for waitFor");
+    (0, import_utils6.assert)(checkIntervalMs, "No checkIntervalMs for waitFor");
+    const overallStartTime = Date.now();
+    let startTime = Date.now();
+    let errorThought = "";
+    while (Date.now() - overallStartTime < timeoutMs) {
+      startTime = Date.now();
+      const assertPlan = {
+        type: "AssertWithoutThrow",
+        param: {
+          assertion
+        },
+        locate: null
+      };
+      const { tasks: assertTasks } = await this.convertPlanToExecutable([
+        assertPlan
+      ]);
+      await taskExecutor.append(
+        this.prependExecutorWithScreenshot(assertTasks[0])
+      );
+      const output = await taskExecutor.flush();
+      if (output?.pass) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      errorThought = output?.thought || `unknown error when waiting for assertion: ${assertion}`;
+      const now = Date.now();
+      if (now - startTime < checkIntervalMs) {
+        const timeRemaining = checkIntervalMs - (now - startTime);
+        const sleepPlan = {
+          type: "Sleep",
+          param: {
+            timeMs: timeRemaining
+          },
+          locate: null
+        };
+        const { tasks: sleepTasks } = await this.convertPlanToExecutable([
+          sleepPlan
+        ]);
+        await taskExecutor.append(
+          this.prependExecutorWithScreenshot(sleepTasks[0])
+        );
+        await taskExecutor.flush();
+      }
+    }
+    return this.appendErrorPlan(
+      taskExecutor,
+      `waitFor timeout: ${errorThought}`
+    );
+  }
+};
+
+// src/common/plan-builder.ts
+var import_logger2 = require("misoai-shared/logger");
+var import_utils8 = require("misoai-shared/utils");
+var debug2 = (0, import_logger2.getDebug)("plan-builder");
+function buildPlans(type, locateParam, param) {
+  let returnPlans = [];
+  const locatePlan = locateParam ? {
+    type: "Locate",
+    locate: locateParam,
+    param: locateParam,
+    thought: ""
+  } : null;
+  if (type === "Tap" || type === "Hover") {
+    (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
+    (0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
+    const tapPlan = {
+      type,
+      param: null,
+      thought: "",
+      locate: locateParam
+    };
+    returnPlans = [locatePlan, tapPlan];
+  }
+  if (type === "Input" || type === "KeyboardPress") {
+    if (type === "Input") {
+      (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
+    }
+    (0, import_utils8.assert)(param, `missing param for action "${type}"`);
+    const inputPlan = {
+      type,
+      param,
+      thought: "",
+      locate: locateParam
+    };
+    if (locatePlan) {
+      returnPlans = [locatePlan, inputPlan];
+    } else {
+      returnPlans = [inputPlan];
+    }
+  }
+  if (type === "Scroll") {
+    (0, import_utils8.assert)(param, `missing param for action "${type}"`);
+    const scrollPlan = {
+      type,
+      param,
+      thought: "",
+      locate: locateParam
+    };
+    if (locatePlan) {
+      returnPlans = [locatePlan, scrollPlan];
+    } else {
+      returnPlans = [scrollPlan];
+    }
+  }
+  if (type === "Sleep") {
+    (0, import_utils8.assert)(param, `missing param for action "${type}"`);
+    const sleepPlan = {
+      type,
+      param,
+      thought: "",
+      locate: null
+    };
+    returnPlans = [sleepPlan];
+  }
+  if (type === "Locate") {
+    (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
+    const locatePlan2 = {
+      type,
+      param: locateParam,
+      locate: locateParam,
+      thought: ""
+    };
+    returnPlans = [locatePlan2];
+  }
+  if (returnPlans) {
+    debug2("buildPlans", returnPlans);
+    return returnPlans;
+  }
+  throw new Error(`Not supported type: ${type}`);
+}
+
+// src/common/task-cache.ts
+var import_node_assert = __toESM(require("assert"));
+var import_node_fs2 = require("fs");
+var import_node_path2 = require("path");
+var import_common2 = require("misoai-shared/common");
+var import_logger3 = require("misoai-shared/logger");
+var import_utils9 = require("misoai-shared/utils");
+var import_js_yaml3 = __toESM(require("js-yaml"));
+var import_semver = __toESM(require("semver"));
+
+// package.json
+var version = "1.0.4";
+
+// src/common/task-cache.ts
+var debug3 = (0, import_logger3.getDebug)("cache");
+var lowestSupportedMidsceneVersion = "0.16.10";
+var cacheFileExt = ".cache.yaml";
+var TaskCache = class {
+  // Track matched records
+  constructor(cacheId, isCacheResultUsed, cacheFilePath) {
+    this.matchedCacheIndices = /* @__PURE__ */ new Set();
+    (0, import_node_assert.default)(cacheId, "cacheId is required");
+    this.cacheId = replaceIllegalPathCharsAndSpace(cacheId);
+    this.cacheFilePath = import_utils9.ifInBrowser ? void 0 : cacheFilePath || (0, import_node_path2.join)((0, import_common2.getMidsceneRunSubDir)("cache"), `${this.cacheId}${cacheFileExt}`);
+    this.isCacheResultUsed = isCacheResultUsed;
+    let cacheContent;
+    if (this.cacheFilePath) {
+      cacheContent = this.loadCacheFromFile();
+    }
+    if (!cacheContent) {
+      cacheContent = {
+        midsceneVersion: version,
+        cacheId: this.cacheId,
+        caches: []
+      };
+    }
+    this.cache = cacheContent;
+    this.cacheOriginalLength = this.cache.caches.length;
+  }
+  matchCache(prompt, type) {
+    for (let i = 0; i < this.cacheOriginalLength; i++) {
+      const item = this.cache.caches[i];
+      const key = `${type}:${prompt}:${i}`;
+      if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
+        this.matchedCacheIndices.add(key);
+        debug3(
+          "cache found and marked as used, type: %s, prompt: %s, index: %d",
+          type,
+          prompt,
+          i
+        );
+        return {
+          cacheContent: item,
+          updateFn: (cb) => {
+            debug3(
+              "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
+              type,
+              prompt,
+              i
+            );
+            cb(item);
+            debug3(
+              "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
+              type,
+              prompt,
+              i
+            );
+            this.flushCacheToFile();
+          }
+        };
+      }
+    }
+    debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
+    return void 0;
+  }
+  matchPlanCache(prompt) {
+    return this.matchCache(prompt, "plan");
+  }
+  matchLocateCache(prompt) {
+    return this.matchCache(prompt, "locate");
+  }
+  appendCache(cache) {
+    debug3("will append cache", cache);
+    this.cache.caches.push(cache);
+    this.flushCacheToFile();
+  }
+  loadCacheFromFile() {
+    const cacheFile = this.cacheFilePath;
+    (0, import_node_assert.default)(cacheFile, "cache file path is required");
+    if (!(0, import_node_fs2.existsSync)(cacheFile)) {
+      debug3("no cache file found, path: %s", cacheFile);
+      return void 0;
+    }
+    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, ".json");
+    if ((0, import_node_fs2.existsSync)(jsonTypeCacheFile) && this.isCacheResultUsed) {
+      console.warn(
+        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`
+      );
+      return void 0;
+    }
+    try {
+      const data = (0, import_node_fs2.readFileSync)(cacheFile, "utf8");
+      const jsonData = import_js_yaml3.default.load(data);
+      if (!version) {
+        debug3("no midscene version info, will not read cache from file");
+        return void 0;
+      }
+      if (import_semver.default.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) && !jsonData.midsceneVersion.includes("beta")) {
+        console.warn(
+          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.
+Please delete the existing cache and rebuild it. Sorry for the inconvenience.
+cache file: ${cacheFile}`
+        );
+        return void 0;
+      }
+      debug3(
+        "cache loaded from file, path: %s, cache version: %s, record length: %s",
+        cacheFile,
+        jsonData.midsceneVersion,
+        jsonData.caches.length
+      );
+      jsonData.midsceneVersion = version;
+      return jsonData;
+    } catch (err) {
+      debug3(
+        "cache file exists but load failed, path: %s, error: %s",
+        cacheFile,
+        err
+      );
+      return void 0;
+    }
+  }
+  flushCacheToFile() {
+    if (!version) {
+      debug3("no midscene version info, will not write cache to file");
+      return;
+    }
+    if (!this.cacheFilePath) {
+      debug3("no cache file path, will not write cache to file");
+      return;
+    }
+    try {
+      const yamlData = import_js_yaml3.default.dump(this.cache);
+      (0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
+    } catch (err) {
+      debug3(
+        "write cache to file failed, path: %s, error: %s",
+        this.cacheFilePath,
+        err
+      );
+    }
+  }
+  updateOrAppendCacheRecord(newRecord, cachedRecord) {
+    if (cachedRecord) {
+      if (newRecord.type === "plan") {
+        cachedRecord.updateFn((cache) => {
+          cache.yamlWorkflow = newRecord.yamlWorkflow;
+        });
+      } else {
+        cachedRecord.updateFn((cache) => {
+          cache.xpaths = newRecord.xpaths;
+        });
+      }
+    } else {
+      this.appendCache(newRecord);
+    }
+  }
+};
+
+// src/common/agent.ts
+var debug4 = (0, import_logger4.getDebug)("web-integration");
+var distanceOfTwoPoints = (p1, p2) => {
+  const [x1, y1] = p1;
+  const [x2, y2] = p2;
+  return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));
+};
+var includedInRect = (point, rect) => {
+  const [x, y] = point;
+  const { left, top, width, height } = rect;
+  return x >= left && x <= left + width && y >= top && y <= top + height;
+};
+var PageAgent = class {
+  constructor(page, opts) {
+    /**
+     * If true, the agent will not perform any actions
+     */
+    this.dryMode = false;
+    this.page = page;
+    this.opts = Object.assign(
+      {
+        generateReport: true,
+        autoPrintReportMsg: true,
+        groupName: "Midscene Report",
+        groupDescription: ""
+      },
+      opts || {}
+    );
+    if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
+      this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || import_constants2.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
+      this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || import_constants2.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
+    }
+    this.onTaskStartTip = this.opts.onTaskStartTip;
+    this.insight = new import_misoai_core2.Insight(
+      async (action) => {
+        return this.getUIContext(action);
+      }
+    );
+    if (opts?.cacheId && this.page.pageType !== "android") {
+      this.taskCache = new TaskCache(
+        opts.cacheId,
+        (0, import_env2.getAIConfigInBoolean)("MIDSCENE_CACHE")
+        // if we should use cache to match the element
+      );
+    }
+    this.taskExecutor = new PageTaskExecutor(this.page, this.insight, {
+      taskCache: this.taskCache,
+      onTaskStart: this.callbackOnTaskStartTip.bind(this)
+    });
+    this.dump = this.resetDump();
+    this.reportFileName = reportFileName(
+      opts?.testId || this.page.pageType || "web"
+    );
+  }
+  async getUIContext(action) {
+    if (action && (action === "extract" || action === "assert" || action === "captcha")) {
+      return await parseContextFromWebPage(this.page, {
+        ignoreMarker: true
+      });
+    }
+    return await parseContextFromWebPage(this.page, {
+      ignoreMarker: !!(0, import_env2.vlLocateMode)()
+    });
+  }
+  // Helper method to call the insight.captcha method
+  async _callInsightCaptcha(options) {
+    const context = await this.getUIContext();
+    if (this.page.url) {
+      const url = await this.page.url();
+      context.url = url;
+    }
+    return this.insight.captcha(context, options);
+  }
+  async setAIActionContext(prompt) {
+    this.opts.aiActionContext = prompt;
+  }
+  resetDump() {
+    this.dump = {
+      groupName: this.opts.groupName,
+      groupDescription: this.opts.groupDescription,
+      executions: []
+    };
+    return this.dump;
+  }
+  appendExecutionDump(execution) {
+    const currentDump = this.dump;
+    currentDump.executions.push(execution);
+  }
+  dumpDataString() {
+    this.dump.groupName = this.opts.groupName;
+    this.dump.groupDescription = this.opts.groupDescription;
+    return (0, import_utils11.stringifyDumpData)(this.dump);
+  }
+  reportHTMLString() {
+    return (0, import_utils11.reportHTMLContent)(this.dumpDataString());
+  }
+  writeOutActionDumps() {
+    const { generateReport, autoPrintReportMsg } = this.opts;
+    this.reportFile = (0, import_utils11.writeLogFile)({
+      fileName: this.reportFileName,
+      fileExt: import_utils11.groupedActionDumpFileExt,
+      fileContent: this.dumpDataString(),
+      type: "dump",
+      generateReport
+    });
+    debug4("writeOutActionDumps", this.reportFile);
+    if (generateReport && autoPrintReportMsg && this.reportFile) {
+      printReportMsg(this.reportFile);
+    }
+  }
+  async callbackOnTaskStartTip(task) {
+    const param = paramStr(task);
+    const tip = param ? `${typeStr(task)} - ${param}` : typeStr(task);
+    if (this.onTaskStartTip) {
+      await this.onTaskStartTip(tip);
+    }
+  }
+  afterTaskRunning(executor, doNotThrowError = false) {
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState() && !doNotThrowError) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask?.error}`);
+    }
+    const lastTask = executor.tasks[executor.tasks.length - 1];
+    const allThoughts = executor.tasks.filter((task) => task.thought).map((task) => task.thought);
+    const allLocates = executor.tasks.filter((task) => task.locate).map((task) => task.locate);
+    const allPlans = executor.tasks.filter((task) => task.param?.plans).map((task) => task.param?.plans);
+    const planningTasks = executor.tasks.filter((task) => task.type === "Planning");
+    const insightTasks = executor.tasks.filter((task) => task.type === "Insight");
+    const actionTasks = executor.tasks.filter((task) => task.type === "Action");
+    const planning = planningTasks.length > 0 ? {
+      type: "Planning",
+      description: `Planning for task execution`,
+      steps: planningTasks.map((task) => task.thought || "Planning step")
+    } : void 0;
+    const insight = insightTasks.length > 0 ? {
+      type: "Insight",
+      description: `Insight for task execution`,
+      elements: insightTasks.map((task) => task.thought || "Insight element")
+    } : void 0;
+    const action = actionTasks.length > 0 ? {
+      type: "Action",
+      description: `Action for task execution`,
+      result: lastTask?.output
+    } : void 0;
+    const actionDetails = executor.tasks.map((task) => ({
+      type: task.type,
+      subType: task.subType,
+      status: task.status,
+      thought: task.thought
+    }));
+    const metadata = {
+      status: lastTask?.status,
+      start: lastTask?.timing?.start,
+      end: lastTask?.timing?.end,
+      totalTime: lastTask?.timing?.cost,
+      cache: lastTask?.cache,
+      usage: lastTask?.usage,
+      thought: allThoughts.length > 0 ? allThoughts.join("\n") : lastTask?.thought,
+      locate: allLocates.length > 0 ? allLocates : lastTask?.locate,
+      plan: allPlans.length > 0 ? allPlans : lastTask?.param?.plans,
+      // Add planning, insight, and action information
+      planning,
+      insight,
+      action,
+      actionDetails,
+      // Include raw tasks for debugging
+      tasks: executor.tasks.map((task) => ({
+        type: task.type,
+        subType: task.subType,
+        status: task.status,
+        thought: task.thought,
+        locate: task.locate,
+        timing: task.timing,
+        usage: task.usage,
+        cache: task.cache,
+        error: task.error
+      }))
+    };
+    return metadata;
+  }
+  buildDetailedLocateParam(locatePrompt, opt) {
+    (0, import_utils12.assert)(locatePrompt, "missing locate prompt");
+    if (typeof opt === "object") {
+      const prompt = opt.prompt ?? locatePrompt;
+      const deepThink = opt.deepThink ?? false;
+      const cacheable = opt.cacheable ?? true;
+      return {
+        prompt,
+        deepThink,
+        cacheable
+      };
+    }
+    return {
+      prompt: locatePrompt
+    };
+  }
+  async aiTap(locatePrompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(
+      locatePrompt,
+      opt
+    );
+    const plans = buildPlans("Tap", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Tap", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiHover(locatePrompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(
+      locatePrompt,
+      opt
+    );
+    const plans = buildPlans("Hover", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Hover", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiInput(value, locatePrompt, opt) {
+    (0, import_utils12.assert)(
+      typeof value === "string",
+      "input value must be a string, use empty string if you want to clear the input"
+    );
+    (0, import_utils12.assert)(locatePrompt, "missing locate prompt for input");
+    const detailedLocateParam = this.buildDetailedLocateParam(
+      locatePrompt,
+      opt
+    );
+    const plans = buildPlans("Input", detailedLocateParam, {
+      value
+    });
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Input", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiKeyboardPress(keyName, locatePrompt, opt) {
+    (0, import_utils12.assert)(keyName, "missing keyName for keyboard press");
+    const detailedLocateParam = locatePrompt ? this.buildDetailedLocateParam(locatePrompt, opt) : void 0;
+    const plans = buildPlans("KeyboardPress", detailedLocateParam, {
+      value: keyName
+    });
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("KeyboardPress", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiScroll(scrollParam, locatePrompt, opt) {
+    const detailedLocateParam = locatePrompt ? this.buildDetailedLocateParam(locatePrompt, opt) : void 0;
+    const plans = buildPlans("Scroll", detailedLocateParam, scrollParam);
+    const paramInTitle = locatePrompt ? `${locateParamStr(detailedLocateParam)} - ${scrollParamStr(scrollParam)}` : scrollParamStr(scrollParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Scroll", paramInTitle),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiAction(taskPrompt, opt) {
+    try {
+      const aiModel = await import("misoai-core/ai-model");
+      const contextStore = aiModel.getContextStore();
+      const processedPrompt = contextStore.replaceAllReferences(taskPrompt, "action");
+      contextStore.addStep({
+        type: "action",
+        summary: `Action: ${processedPrompt}`,
+        prompt: processedPrompt
+      });
+      taskPrompt = processedPrompt;
+    } catch (error) {
+      debug4("Context store not available:", error);
+    }
+    const cacheable = opt?.cacheable;
+    const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
+    const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
+    if (matchedCache && this.taskCache?.isCacheResultUsed) {
+      const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
+        taskPrompt,
+        matchedCache.cacheContent?.yamlWorkflow
+      );
+      const metadata2 = this.afterTaskRunning(executor2);
+      debug4("matched cache, will call .runYaml to run the action");
+      const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
+      const result = await this.runYaml(yaml5);
+      return {
+        result: result.result,
+        metadata: metadata2
+      };
+    }
+    const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {
+      cacheable
+    }));
+    if (this.taskCache && output?.yamlFlow && cacheable !== false) {
+      const yamlContent = {
+        tasks: [
+          {
+            name: taskPrompt,
+            flow: output.yamlFlow
+          }
+        ]
+      };
+      const yamlFlowStr = import_js_yaml4.default.dump(yamlContent);
+      this.taskCache.updateOrAppendCacheRecord(
+        {
+          type: "plan",
+          prompt: taskPrompt,
+          yamlWorkflow: yamlFlowStr
+        },
+        matchedCache
+      );
+    }
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiQuery(demand) {
+    let processedDemand = demand;
+    let storageKey;
+    try {
+      const aiModel = await import("misoai-core/ai-model");
+      const contextStore = aiModel.getContextStore();
+      if (typeof demand === "string") {
+        const storageInstruction = contextStore.parseStorageInstruction(demand);
+        if (storageInstruction) {
+          storageKey = storageInstruction.key;
+          processedDemand = storageInstruction.cleanText;
+          contextStore._pendingAliases = storageInstruction.aliases;
+        } else {
+          const storageMatch = demand.match(/store\s+(?:as\s+)?(\w+)/i);
+          if (storageMatch) {
+            storageKey = storageMatch[1];
+            processedDemand = demand.replace(/,?\s*store\s+(?:as\s+)?\w+/i, "").trim();
+          }
+        }
+      }
+    } catch (error) {
+      debug4("Context store not available:", error);
+    }
+    const { output, executor } = await this.taskExecutor.query(processedDemand);
+    if (storageKey && output) {
+      try {
+        const aiModel = await import("misoai-core/ai-model");
+        const contextStore = aiModel.getContextStore();
+        const pendingAliases = contextStore._pendingAliases;
+        if (pendingAliases) {
+          contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
+          delete contextStore._pendingAliases;
+        } else {
+          contextStore.storeData(storageKey, output);
+        }
+        contextStore.addStep({
+          type: "query",
+          summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
+          data: output,
+          prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
+        });
+      } catch (error) {
+        debug4("Failed to store query result:", error);
+      }
+    } else {
+      try {
+        const aiModel = await import("misoai-core/ai-model");
+        const contextStore = aiModel.getContextStore();
+        contextStore.addStep({
+          type: "query",
+          summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
+          data: output,
+          prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
+        });
+      } catch (error) {
+        debug4("Failed to add query step:", error);
+      }
+    }
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiBoolean(prompt) {
+    const { output, executor } = await this.taskExecutor.boolean(prompt);
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiNumber(prompt) {
+    const { output, executor } = await this.taskExecutor.number(prompt);
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async aiString(prompt) {
+    const { output, executor } = await this.taskExecutor.string(prompt);
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
+  async describeElementAtPoint(center, opt) {
+    const { verifyPrompt = true, retryLimit = 3 } = opt || {};
+    let success = false;
+    let retryCount = 0;
+    let resultPrompt = "";
+    let deepThink = opt?.deepThink || false;
+    let verifyResult;
+    while (!success && retryCount < retryLimit) {
+      if (retryCount >= 2) {
+        deepThink = true;
+      }
+      debug4(
+        "aiDescribe",
+        center,
+        "verifyPrompt",
+        verifyPrompt,
+        "retryCount",
+        retryCount,
+        "deepThink",
+        deepThink
+      );
+      const text = await this.insight.describe(center, { deepThink });
+      debug4("aiDescribe text", text);
+      (0, import_utils12.assert)(text.description, `failed to describe element at [${center}]`);
+      resultPrompt = text.description;
+      verifyResult = await this.verifyLocator(
+        resultPrompt,
+        deepThink ? { deepThink: true } : void 0,
+        center,
+        opt
+      );
+      if (verifyResult.pass) {
+        success = true;
+      } else {
+        retryCount++;
+      }
+    }
+    return {
+      prompt: resultPrompt,
+      deepThink,
+      verifyResult
+    };
+  }
+  async verifyLocator(prompt, locateOpt, expectCenter, verifyLocateOption) {
+    debug4("verifyLocator", prompt, locateOpt, expectCenter, verifyLocateOption);
+    const locateResult = await this.aiLocate(prompt, locateOpt);
+    const { center: verifyCenter, rect: verifyRect } = locateResult.result;
+    const distance = distanceOfTwoPoints(expectCenter, verifyCenter);
+    const included = includedInRect(expectCenter, verifyRect);
+    const pass = distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || included;
+    const verifyResult = {
+      pass,
+      rect: verifyRect,
+      center: verifyCenter,
+      centerDistance: distance
+    };
+    debug4("aiDescribe verifyResult", verifyResult);
+    return verifyResult;
+  }
+  async aiLocate(prompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);
+    const plans = buildPlans("Locate", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    const { element } = output;
+    const result = {
+      rect: element?.rect,
+      center: element?.center
+    };
+    return {
+      result,
+      metadata
+    };
+  }
+  async aiAssert(assertion, msg, opt) {
+    let processedAssertion = assertion;
+    try {
+      const aiModel = await import("misoai-core/ai-model");
+      const contextStore = aiModel.getContextStore();
+      processedAssertion = contextStore.replaceAllReferences(assertion, "assertion");
+      contextStore.addStep({
+        type: "assertion",
+        summary: `Assertion: ${processedAssertion}`,
+        prompt: processedAssertion
+      });
+    } catch (error) {
+      debug4("Context store not available:", error);
+    }
+    let currentUrl = "";
+    if (this.page.url) {
+      try {
+        currentUrl = await this.page.url();
+      } catch (e) {
+      }
+    }
+    const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${processedAssertion}` : processedAssertion;
+    const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
+    const metadata = this.afterTaskRunning(executor, true);
+    if (output && opt?.keepRawResponse) {
+      return {
+        result: output,
+        metadata
+      };
+    }
+    if (!output?.pass) {
+      const errMsg = msg || `Assertion failed: ${assertion}`;
+      const reasonMsg = `Reason: ${output?.thought || executor.latestErrorTask()?.error || "(no_reason)"}`;
+      throw new Error(`${errMsg}
+${reasonMsg}`);
+    }
+    return {
+      result: true,
+      metadata
+    };
+  }
+  async aiCaptcha(options) {
+    const { deepThink = false, autoDetectComplexity = true } = options || {};
+    let shouldUseDeepThink = deepThink;
+    if (autoDetectComplexity && !deepThink) {
+      const context = await this.getUIContext();
+      const { screenshotBase64 } = context;
+      try {
+        const complexityAnalysisPrompt = `
+Analyze this screenshot and determine if it contains a complex CAPTCHA that would benefit from deep thinking.
+A complex CAPTCHA typically has one or more of these characteristics:
+- Distorted or overlapping text that is hard to read
+- Multiple images that need to be selected based on a specific criteria
+- Puzzles that require spatial reasoning
+- Multiple steps or verification methods
+- Small or hard-to-distinguish elements
+
+Return only "complex" or "simple" based on your analysis.
+`;
+        const complexityMsgs = [
+          { role: "system", content: "You are an AI assistant that analyzes screenshots to determine CAPTCHA complexity." },
+          {
+            role: "user",
+            content: [
+              {
+                type: "image_url",
+                image_url: {
+                  url: screenshotBase64,
+                  detail: "high"
+                }
+              },
+              {
+                type: "text",
+                text: complexityAnalysisPrompt
+              }
+            ]
+          }
+        ];
+        const complexityResult = await this.insight.aiVendorFn(
+          complexityMsgs,
+          { type: "extract_data" }
+        );
+        const responseText = typeof complexityResult.content === "string" ? complexityResult.content.toLowerCase() : JSON.stringify(complexityResult.content).toLowerCase();
+        shouldUseDeepThink = responseText.includes("complex");
+        debug4("CAPTCHA complexity analysis:", responseText, "Using deep think:", shouldUseDeepThink);
+      } catch (error) {
+        debug4("Failed to analyze CAPTCHA complexity:", error);
+      }
+    }
+    const captchaResponse = await this._callInsightCaptcha({
+      deepThink: shouldUseDeepThink
+    });
+    const captchaResult = captchaResponse.content;
+    const usage = captchaResponse.usage;
+    const actualDeepThink = captchaResponse.deepThink || false;
+    if (captchaResult.captchaType === "text") {
+      for (const action of captchaResult.actions) {
+        if (action.type === "click" && action.target) {
+          await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
+        } else if (action.type === "input" && action.value) {
+          if (action.target) {
+            await this.aiInput(action.value, action.target, { deepThink: shouldUseDeepThink });
+          }
+        } else if (action.type === "verify" && action.target) {
+          await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
+        }
+      }
+    } else if (captchaResult.captchaType === "image") {
+      for (const action of captchaResult.actions) {
+        if (action.type === "click") {
+          if (action.coordinates) {
+            const x = action.coordinates[0];
+            const y = action.coordinates[1];
+            await this.aiTap(`element at coordinates (${x}, ${y})`, { deepThink: shouldUseDeepThink });
+          } else if (action.target) {
+            await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
+          }
+        } else if (action.type === "verify" && action.target) {
+          await this.aiTap(action.target, { deepThink: shouldUseDeepThink });
+        }
+      }
+    }
+    await new Promise((resolve2) => setTimeout(resolve2, 3e3));
+    const metadata = {
+      status: "finished",
+      usage,
+      thought: captchaResult.thought
+    };
+    metadata.deepThink = actualDeepThink;
+    if (autoDetectComplexity && !deepThink) {
+      metadata.autoDetectedComplexity = shouldUseDeepThink;
+    }
+    return {
+      result: captchaResult,
+      metadata
+    };
+  }
+  async aiWaitFor(assertion, opt) {
+    const startTime = Date.now();
+    const { executor } = await this.taskExecutor.waitFor(assertion, {
+      timeoutMs: opt?.timeoutMs || 15 * 1e3,
+      checkIntervalMs: opt?.checkIntervalMs || 3 * 1e3,
+      assertion
+    });
+    const metadata = {
+      status: executor.isInErrorState() ? "failed" : "finished",
+      start: startTime,
+      end: Date.now(),
+      totalTime: Date.now() - startTime,
+      thought: executor.latestErrorTask()?.thought,
+      actionDetails: executor.tasks.map((task) => ({
+        type: task.type,
+        subType: task.subType,
+        status: task.status,
+        thought: task.thought
+      }))
+    };
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState()) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask?.error}
+${errorTask?.errorStack}`);
+    }
+    return {
+      result: true,
+      // Successfully waited
+      metadata
+    };
+  }
+  async ai(taskPrompt, type = "action", options) {
+    if (type === "action") {
+      return this.aiAction(taskPrompt);
+    }
+    if (type === "query") {
+      return this.aiQuery(taskPrompt);
+    }
+    if (type === "assert") {
+      return this.aiAssert(taskPrompt);
+    }
+    if (type === "tap") {
+      return this.aiTap(taskPrompt, options);
+    }
+    if (type === "captcha") {
+      return this.aiCaptcha(options);
+    }
+    throw new Error(
+      `Unknown type: ${type}, only support 'action', 'query', 'assert', 'tap', 'captcha'`
+    );
+  }
+  async runYaml(yamlScriptContent) {
+    const startTime = Date.now();
+    const script = parseYamlScript(yamlScriptContent, "yaml", true);
+    const player = new ScriptPlayer(script, async () => {
+      return { agent: this, freeFn: [] };
+    });
+    await player.run();
+    const endTime = Date.now();
+    const metadata = {
+      status: player.status,
+      start: startTime,
+      end: endTime,
+      totalTime: endTime - startTime,
+      tasks: player.taskStatusList.map((task) => ({
+        type: "yaml-task",
+        subType: task.name,
+        status: task.status,
+        error: task.error?.message
+      }))
+    };
+    if (player.status === "error") {
+      const errors = player.taskStatusList.filter((task) => task.status === "error").map((task) => {
+        return `task - ${task.name}: ${task.error?.message}`;
+      }).join("\n");
+      throw new Error(`Error(s) occurred in running yaml script:
+${errors}`);
+    }
+    return {
+      result: player.result,
+      metadata
+    };
+  }
+  async evaluateJavaScript(script) {
+    (0, import_utils12.assert)(
+      this.page.evaluateJavaScript,
+      "evaluateJavaScript is not supported in current agent"
+    );
+    if (this.page.evaluateJavaScript) {
+      return this.page.evaluateJavaScript(script);
+    }
+    throw new Error("evaluateJavaScript is not supported in current agent");
+  }
+  async destroy() {
+    await this.page.destroy();
+  }
+};
+
+// src/puppeteer/base-page.ts
+var import_utils15 = require("misoai-core/utils");
+var import_constants3 = require("misoai-shared/constants");
+var import_extractor2 = require("misoai-shared/extractor");
+var import_fs = require("misoai-shared/fs");
+var import_logger5 = require("misoai-shared/logger");
+var import_utils16 = require("misoai-shared/utils");
+var debugPage = (0, import_logger5.getDebug)("web:page");
+var Page = class {
+  constructor(underlyingPage, pageType, opts) {
+    this.everMoved = false;
+    this.underlyingPage = underlyingPage;
+    this.pageType = pageType;
+    this.waitForNavigationTimeout = opts?.waitForNavigationTimeout || import_constants3.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
+  }
+  async evaluate(pageFunction, arg) {
+    let result;
+    debugPage("evaluate function begin");
+    if (this.pageType === "puppeteer") {
+      result = await this.underlyingPage.evaluate(
+        pageFunction,
+        arg
+      );
+    } else {
+      result = await this.underlyingPage.evaluate(
+        pageFunction,
+        arg
+      );
+    }
+    debugPage("evaluate function end");
+    return result;
+  }
+  async evaluateJavaScript(script) {
+    return this.evaluate(script);
+  }
+  async waitForNavigation() {
+    if (this.pageType === "puppeteer" || this.pageType === "playwright") {
+      debugPage("waitForNavigation begin");
+      debugPage(`waitForNavigation timeout: ${this.waitForNavigationTimeout}`);
+      try {
+        await this.underlyingPage.waitForSelector("html", {
+          timeout: this.waitForNavigationTimeout
+        });
+      } catch (error) {
+        console.warn(
+          "[midscene:warning] Waiting for the navigation has timed out, but Midscene will continue execution. Please check https://midscenejs.com/faq.html#customize-the-network-timeout for more information on customizing the network timeout"
+        );
+      }
+      debugPage("waitForNavigation end");
+    }
+  }
+  // @deprecated
+  async getElementsInfo() {
+    await this.waitForNavigation();
+    debugPage("getElementsInfo begin");
+    const tree = await this.getElementsNodeTree();
+    debugPage("getElementsInfo end");
+    return (0, import_extractor2.treeToList)(tree);
+  }
+  async getXpathsById(id) {
+    const elementInfosScriptContent = (0, import_fs.getElementInfosScriptContent)();
+    return this.evaluateJavaScript(
+      `${elementInfosScriptContent}midscene_element_inspector.getXpathsById('${id}')`
+    );
+  }
+  async getElementInfoByXpath(xpath) {
+    const elementInfosScriptContent = (0, import_fs.getElementInfosScriptContent)();
+    return this.evaluateJavaScript(
+      `${elementInfosScriptContent}midscene_element_inspector.getElementInfoByXpath('${xpath}')`
+    );
+  }
+  async getElementsNodeTree() {
+    await this.waitForNavigation();
+    const scripts = await (0, import_fs.getExtraReturnLogic)(true);
+    (0, import_utils16.assert)(scripts, "scripts should be set before writing report in browser");
+    const captureElementSnapshot = await this.evaluate(scripts);
+    return captureElementSnapshot;
+  }
+  async size() {
+    if (this.viewportSize)
+      return this.viewportSize;
+    const sizeInfo = await this.evaluate(() => {
+      return {
+        width: document.documentElement.clientWidth,
+        height: document.documentElement.clientHeight,
+        dpr: window.devicePixelRatio
+      };
+    });
+    this.viewportSize = sizeInfo;
+    return sizeInfo;
+  }
+  async screenshotBase64() {
+    const imgType = "jpeg";
+    const quality = 90;
+    await this.waitForNavigation();
+    debugPage("screenshotBase64 begin");
+    let base64;
+    if (this.pageType === "puppeteer") {
+      const result = await this.underlyingPage.screenshot({
+        type: imgType,
+        quality,
+        encoding: "base64"
+      });
+      base64 = `data:image/jpeg;base64,${result}`;
+    } else if (this.pageType === "playwright") {
+      const buffer = await this.underlyingPage.screenshot({
+        type: imgType,
+        quality,
+        timeout: 10 * 1e3
+      });
+      base64 = `data:image/jpeg;base64,${buffer.toString("base64")}`;
+    } else {
+      throw new Error("Unsupported page type for screenshot");
+    }
+    debugPage("screenshotBase64 end");
+    return base64;
+  }
+  async url() {
+    return this.underlyingPage.url();
+  }
+  get mouse() {
+    return {
+      click: async (x, y, options) => {
+        await this.mouse.move(x, y);
+        this.underlyingPage.mouse.click(x, y, {
+          button: options?.button || "left",
+          count: options?.count || 1
+        });
+      },
+      wheel: async (deltaX, deltaY) => {
+        if (this.pageType === "puppeteer") {
+          await this.underlyingPage.mouse.wheel({
+            deltaX,
+            deltaY
+          });
+        } else if (this.pageType === "playwright") {
+          await this.underlyingPage.mouse.wheel(
+            deltaX,
+            deltaY
+          );
+        }
+      },
+      move: async (x, y) => {
+        this.everMoved = true;
+        return this.underlyingPage.mouse.move(x, y);
+      },
+      drag: async (from, to) => {
+        if (this.pageType === "puppeteer") {
+          await this.underlyingPage.mouse.drag(
+            {
+              x: from.x,
+              y: from.y
+            },
+            {
+              x: to.x,
+              y: to.y
+            }
+          );
+        } else if (this.pageType === "playwright") {
+          await this.underlyingPage.mouse.move(
+            from.x,
+            from.y
+          );
+          await this.underlyingPage.mouse.down();
+          await this.underlyingPage.mouse.move(to.x, to.y);
+          await this.underlyingPage.mouse.up();
+        }
+      }
+    };
+  }
+  get keyboard() {
+    return {
+      type: async (text) => this.underlyingPage.keyboard.type(text, { delay: 80 }),
+      press: async (action) => {
+        const keys = Array.isArray(action) ? action : [action];
+        for (const k of keys) {
+          const commands = k.command ? [k.command] : [];
+          await this.underlyingPage.keyboard.down(k.key, { commands });
+        }
+        for (const k of [...keys].reverse()) {
+          await this.underlyingPage.keyboard.up(k.key);
+        }
+      },
+      down: async (key) => {
+        this.underlyingPage.keyboard.down(key);
+      },
+      up: async (key) => {
+        this.underlyingPage.keyboard.up(key);
+      }
+    };
+  }
+  async clearInput(element) {
+    if (!element) {
+      console.warn("No element to clear input");
+      return;
+    }
+    const isMac = process.platform === "darwin";
+    if (isMac) {
+      if (this.pageType === "puppeteer") {
+        await this.mouse.click(element.center[0], element.center[1], {
+          count: 3
+        });
+      } else {
+        await this.mouse.click(element.center[0], element.center[1]);
+        await this.underlyingPage.keyboard.down("Meta");
+        await this.underlyingPage.keyboard.press("a");
+        await this.underlyingPage.keyboard.up("Meta");
+      }
+    } else {
+      await this.mouse.click(element.center[0], element.center[1]);
+      await this.underlyingPage.keyboard.down("Control");
+      await this.underlyingPage.keyboard.press("a");
+      await this.underlyingPage.keyboard.up("Control");
+    }
+    await (0, import_utils15.sleep)(100);
+    await this.keyboard.press([{ key: "Backspace" }]);
+  }
+  async moveToPointBeforeScroll(point) {
+    if (point) {
+      await this.mouse.move(point.left, point.top);
+    } else if (!this.everMoved) {
+      const size = await this.size();
+      const targetX = Math.floor(size.width / 2);
+      const targetY = Math.floor(size.height / 2);
+      await this.mouse.move(targetX, targetY);
+    }
+  }
+  async scrollUntilTop(startingPoint) {
+    await this.moveToPointBeforeScroll(startingPoint);
+    return this.mouse.wheel(0, -9999999);
+  }
+  async scrollUntilBottom(startingPoint) {
+    await this.moveToPointBeforeScroll(startingPoint);
+    return this.mouse.wheel(0, 9999999);
+  }
+  async scrollUntilLeft(startingPoint) {
+    await this.moveToPointBeforeScroll(startingPoint);
+    return this.mouse.wheel(-9999999, 0);
+  }
+  async scrollUntilRight(startingPoint) {
+    await this.moveToPointBeforeScroll(startingPoint);
+    return this.mouse.wheel(9999999, 0);
+  }
+  async scrollUp(distance, startingPoint) {
+    const innerHeight = await this.evaluate(() => window.innerHeight);
+    const scrollDistance = distance || innerHeight * 0.7;
+    await this.moveToPointBeforeScroll(startingPoint);
+    return this.mouse.wheel(0, -scrollDistance);
+  }
+  async scrollDown(distance, startingPoint) {
+    const innerHeight = await this.evaluate(() => window.innerHeight);
+    const scrollDistance = distance || innerHeight * 0.7;
+    await this.moveToPointBeforeScroll(startingPoint);
+    return this.mouse.wheel(0, scrollDistance);
+  }
+  async scrollLeft(distance, startingPoint) {
+    const innerWidth = await this.evaluate(() => window.innerWidth);
+    const scrollDistance = distance || innerWidth * 0.7;
+    await this.moveToPointBeforeScroll(startingPoint);
+    return this.mouse.wheel(-scrollDistance, 0);
+  }
+  async scrollRight(distance, startingPoint) {
+    const innerWidth = await this.evaluate(() => window.innerWidth);
+    const scrollDistance = distance || innerWidth * 0.7;
+    await this.moveToPointBeforeScroll(startingPoint);
+    return this.mouse.wheel(scrollDistance, 0);
+  }
+  async navigate(url) {
+    if (this.pageType === "puppeteer") {
+      await this.underlyingPage.goto(url);
+    } else if (this.pageType === "playwright") {
+      await this.underlyingPage.goto(url);
+    } else {
+      throw new Error("Unsupported page type for navigate");
+    }
+  }
+  async destroy() {
+  }
+};
+
+// src/playwright/page.ts
+var WebPage = class extends Page {
+  constructor(page) {
+    super(page, "playwright");
+  }
+};
+
+// src/playwright/ai-fixture.ts
+var import_node_crypto = require("crypto");
+var import_logger6 = require("misoai-shared/logger");
+var import_test = require("@playwright/test");
+var debugPage2 = (0, import_logger6.getDebug)("web:playwright:ai-fixture");
+var groupAndCaseForTest = (testInfo) => {
+  let taskFile;
+  let taskTitle;
+  const titlePath = [...testInfo.titlePath];
+  if (titlePath.length > 1) {
+    taskFile = titlePath.shift() || "unnamed";
+    taskTitle = titlePath.join("__");
+  } else if (titlePath.length === 1) {
+    taskTitle = titlePath[0];
+    taskFile = `${taskTitle}`;
+  } else {
+    taskTitle = "unnamed";
+    taskFile = "unnamed";
+  }
+  const taskTitleWithRetry = `${taskTitle}${testInfo.retry ? `(retry #${testInfo.retry})` : ""}`;
+  return {
+    file: taskFile,
+    id: replaceIllegalPathCharsAndSpace(`${taskFile}(${taskTitle})`),
+    title: replaceIllegalPathCharsAndSpace(taskTitleWithRetry)
+  };
+};
+var midsceneAgentKeyId = "_midsceneAgentId";
+var midsceneDumpAnnotationId = "MIDSCENE_DUMP_ANNOTATION";
+var PlaywrightAiFixture = (options) => {
+  const { forceSameTabNavigation = true, waitForNetworkIdleTimeout = 1e3 } = options ?? {};
+  const pageAgentMap = {};
+  const createOrReuseAgentForPage = (page, testInfo, opts) => {
+    let idForPage = page[midsceneAgentKeyId];
+    if (!idForPage) {
+      idForPage = (0, import_node_crypto.randomUUID)();
+      page[midsceneAgentKeyId] = idForPage;
+      const { testId } = testInfo;
+      const { file, id, title } = groupAndCaseForTest(testInfo);
+      pageAgentMap[idForPage] = new PlaywrightAgent(page, {
+        testId: `playwright-${testId}-${idForPage}`,
+        forceSameTabNavigation,
+        cacheId: id,
+        groupName: title,
+        groupDescription: file,
+        generateReport: false,
+        // we will generate it in the reporter
+        ...opts
+      });
+    }
+    return pageAgentMap[idForPage];
+  };
+  async function generateAiFunction(options2) {
+    const { page, testInfo, use, aiActionType } = options2;
+    const agent = createOrReuseAgentForPage(page, testInfo);
+    await use(async (taskPrompt, ...args) => {
+      return new Promise((resolve2, reject) => {
+        import_test.test.step(`ai-${aiActionType} - ${JSON.stringify(taskPrompt)}`, async () => {
+          try {
+            debugPage2(
+              `waitForNetworkIdle timeout: ${waitForNetworkIdleTimeout}`
+            );
+            await agent.waitForNetworkIdle(waitForNetworkIdleTimeout);
+          } catch (error) {
+            console.warn(
+              "[midscene:warning] Waiting for network idle has timed out, but Midscene will continue execution. Please check https://midscenejs.com/faq.html#customize-the-network-timeout for more information on customizing the network timeout"
+            );
+          }
+          try {
+            const result = await agent[aiActionType](
+              taskPrompt,
+              ...args || []
+            );
+            resolve2(result);
+          } catch (error) {
+            reject(error);
+          }
+        });
+      });
+    });
+    updateDumpAnnotation(testInfo, agent.dumpDataString());
+  }
+  const updateDumpAnnotation = (test2, dump) => {
+    const currentAnnotation = test2.annotations.find((item) => {
+      return item.type === midsceneDumpAnnotationId;
+    });
+    if (currentAnnotation) {
+      currentAnnotation.description = dump;
+    } else {
+      test2.annotations.push({
+        type: midsceneDumpAnnotationId,
+        description: dump
+      });
+    }
+  };
+  return {
+    agentForPage: async ({ page }, use, testInfo) => {
+      await use(
+        async (propsPage, opts) => {
+          const agent = createOrReuseAgentForPage(
+            propsPage || page,
+            testInfo,
+            opts
+          );
+          return agent;
+        }
+      );
+    },
+    ai: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "ai"
+      });
+    },
+    aiAction: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiAction"
+      });
+    },
+    aiTap: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiTap"
+      });
+    },
+    aiHover: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiHover"
+      });
+    },
+    aiInput: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiInput"
+      });
+    },
+    aiKeyboardPress: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiKeyboardPress"
+      });
+    },
+    aiScroll: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiScroll"
+      });
+    },
+    aiQuery: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiQuery"
+      });
+    },
+    aiAssert: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiAssert"
+      });
+    },
+    aiWaitFor: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiWaitFor"
+      });
+    },
+    aiLocate: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiLocate"
+      });
+    },
+    aiNumber: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiNumber"
+      });
+    },
+    aiString: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiString"
+      });
+    },
+    aiBoolean: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiBoolean"
+      });
+    }
+  };
+};
+
+// src/playwright/index.ts
+var import_env3 = require("misoai-shared/env");
+var import_logger7 = require("misoai-shared/logger");
+var debug5 = (0, import_logger7.getDebug)("playwright:agent");
+var PlaywrightAgent = class extends PageAgent {
+  constructor(page, opts) {
+    const webPage = new WebPage(page);
+    super(webPage, opts);
+    const { forceSameTabNavigation = true } = opts ?? {};
+    if (forceSameTabNavigation) {
+      forceClosePopup(page, debug5);
+    }
+  }
+  async waitForNetworkIdle(timeout = 1e3) {
+    await this.page.underlyingPage.waitForLoadState("networkidle", { timeout });
+  }
+};
+
+// src/puppeteer/index.ts
+var import_logger8 = require("misoai-shared/logger");
+
+// src/puppeteer/page.ts
+var import_constants4 = require("misoai-shared/constants");
+var WebPage2 = class extends Page {
+  constructor(page, opts) {
+    super(page, "puppeteer");
+    const {
+      waitForNavigationTimeout = import_constants4.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,
+      waitForNetworkIdleTimeout = import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT
+    } = opts ?? {};
+    this.waitForNavigationTimeout = waitForNavigationTimeout;
+    this.waitForNetworkIdleTimeout = waitForNetworkIdleTimeout;
+  }
+  async waitUntilNetworkIdle(options) {
+    await this.underlyingPage.waitForNetworkIdle({
+      idleTime: options?.idleTime || import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME,
+      concurrency: options?.concurrency || import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY,
+      timeout: options?.timeout || this.waitForNetworkIdleTimeout
+    });
+  }
+};
+
+// src/puppeteer/index.ts
+var import_env4 = require("misoai-shared/env");
+var debug6 = (0, import_logger8.getDebug)("puppeteer:agent");
+var PuppeteerAgent = class extends PageAgent {
+  constructor(page, opts) {
+    const webPage = new WebPage2(page);
+    super(webPage, opts);
+    const { forceSameTabNavigation = true } = opts ?? {};
+    if (forceSameTabNavigation) {
+      forceClosePopup(page, debug6);
+    }
+  }
+};
+
+// src/playground/agent.ts
+var StaticPageAgent = class extends PageAgent {
+  constructor(page) {
+    super(page, {});
+    this.dryMode = true;
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  PageAgent,
+  PlaywrightAgent,
+  PlaywrightAiFixture,
+  PuppeteerAgent,
+  ScriptPlayer,
+  StaticPageAgent,
+  parseContextFromWebPage,
+  parseYamlScript
+});
+
+//# sourceMappingURL=index.js.map