misoai-web 1.0.3 → 1.0.4

package/dist/es/ui-utils.js

@@ -0,0 +1,106 @@
+// src/common/ui-utils.ts
+function typeStr(task) {
+  return task.subType && task.subType !== "Plan" ? `${task.type} / ${task.subType || ""}` : task.type;
+}
+function getKeyCommands(value) {
+  const keys = Array.isArray(value) ? value : [value];
+  return keys.reduce((acc, k) => {
+    const includeMeta = keys.includes("Meta") || keys.includes("Control");
+    if (includeMeta && (k === "a" || k === "A")) {
+      return acc.concat([{ key: k, command: "SelectAll" }]);
+    }
+    if (includeMeta && (k === "c" || k === "C")) {
+      return acc.concat([{ key: k, command: "Copy" }]);
+    }
+    if (includeMeta && (k === "v" || k === "V")) {
+      return acc.concat([{ key: k, command: "Paste" }]);
+    }
+    return acc.concat([{ key: k }]);
+  }, []);
+}
+function locateParamStr(locate) {
+  if (!locate) {
+    return "";
+  }
+  if (typeof locate === "string") {
+    return locate;
+  }
+  return locate.prompt;
+}
+function scrollParamStr(scrollParam) {
+  if (!scrollParam) {
+    return "";
+  }
+  return `${scrollParam.direction || "down"}, ${scrollParam.scrollType || "once"}, ${scrollParam.distance || "distance-not-set"}`;
+}
+function taskTitleStr(type, prompt) {
+  if (prompt) {
+    return `${type} - ${prompt}`;
+  }
+  return type;
+}
+function paramStr(task) {
+  let value;
+  if (task.type === "Planning") {
+    value = task?.param?.userInstruction;
+  }
+  if (task.type === "Insight") {
+    value = task?.param?.prompt || task?.param?.id || task?.param?.dataDemand || task?.param?.assertion;
+  }
+  if (task.type === "Action") {
+    const locate = task?.locate;
+    const locateStr = locate ? locateParamStr(locate) : "";
+    value = task.thought || "";
+    if (typeof task?.param?.timeMs === "number") {
+      value = `${task?.param?.timeMs}ms`;
+    } else if (typeof task?.param?.scrollType === "string") {
+      value = scrollParamStr(task?.param);
+    } else if (typeof task?.param?.value !== "undefined") {
+      value = task?.param?.value;
+    }
+    if (locateStr) {
+      if (value) {
+        value = `${locateStr} - ${value}`;
+      } else {
+        value = locateStr;
+      }
+    }
+  }
+  if (typeof value === "undefined")
+    return "";
+  return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
+}
+var limitOpenNewTabScript = `
+if (!window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__) {
+  window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__ = true;
+
+  // Intercept the window.open method (only once)
+  window.open = function(url) {
+    console.log('Blocked window.open:', url);
+    window.location.href = url;
+    return null;
+  };
+
+  // Block all a tag clicks with target="_blank" (only once)
+  document.addEventListener('click', function(e) {
+    const target = e.target.closest('a');
+    if (target && target.target === '_blank') {
+      e.preventDefault();
+      console.log('Blocked new tab:', target.href);
+      window.location.href = target.href;
+      target.removeAttribute('target');
+    }
+  }, true);
+}
+`;
+export {
+  getKeyCommands,
+  limitOpenNewTabScript,
+  locateParamStr,
+  paramStr,
+  scrollParamStr,
+  taskTitleStr,
+  typeStr
+};
+
+//# sourceMappingURL=ui-utils.js.map

package/dist/es/ui-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"mappings":";AAWO,SAAS,QAAQ,MAAqB;AAC3C,SAAO,KAAK,WAAW,KAAK,YAAY,SACpC,GAAG,KAAK,IAAI,MAAM,KAAK,WAAW,EAAE,KACpC,KAAK;AACX;AAEO,SAAS,eACd,OAC0C;AAE1C,QAAM,OAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC,KAAK;AAIlD,SAAO,KAAK,OAAO,CAAC,KAA+C,MAAM;AACvE,UAAM,cAAc,KAAK,SAAS,MAAM,KAAK,KAAK,SAAS,SAAS;AACpE,QAAI,gBAAgB,MAAM,OAAO,MAAM,MAAM;AAC3C,aAAO,IAAI,OAAO,CAAC,EAAE,KAAK,GAAG,SAAS,YAAY,CAAC,CAAC;AAAA,IACtD;AACA,QAAI,gBAAgB,MAAM,OAAO,MAAM,MAAM;AAC3C,aAAO,IAAI,OAAO,CAAC,EAAE,KAAK,GAAG,SAAS,OAAO,CAAC,CAAC;AAAA,IACjD;AACA,QAAI,gBAAgB,MAAM,OAAO,MAAM,MAAM;AAC3C,aAAO,IAAI,OAAO,CAAC,EAAE,KAAK,GAAG,SAAS,QAAQ,CAAC,CAAC;AAAA,IAClD;AACA,WAAO,IAAI,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAAA,EAChC,GAAG,CAAC,CAAC;AACP;AAEO,SAAS,eAAe,QAA8B;AAC3D,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,WAAW,UAAU;AAC9B,WAAO;AAAA,EACT;AAEA,SAAO,OAAO;AAChB;AAEO,SAAS,eAAe,aAAyC;AACtE,MAAI,CAAC,aAAa;AAChB,WAAO;AAAA,EACT;AACA,SAAO,GAAG,YAAY,aAAa,MAAM,KAAK,YAAY,cAAc,MAAM,KAAK,YAAY,YAAY,kBAAkB;AAC/H;AAEO,SAAS,aACd,MAcA,QACA;AACA,MAAI,QAAQ;AACV,WAAO,GAAG,IAAI,MAAM,MAAM;AAAA,EAC5B;AACA,SAAO;AACT;AAEO,SAAS,SAAS,MAAqB;AAC5C,MAAI;AACJ,MAAI,KAAK,SAAS,YAAY;AAC5B,YAAS,MAAgC,OAAO;AAAA,EAClD;AAEA,MAAI,KAAK,SAAS,WAAW;AAC3B,YACG,MAAqC,OAAO,UAC5C,MAAqC,OAAO,MAC5C,MAAoC,OAAO,cAC3C,MAAwC,OAAO;AAAA,EACpD;AAEA,MAAI,KAAK,SAAS,UAAU;AAC1B,UAAM,SAAU,MAA8B;AAC9C,UAAM,YAAY,SAAS,eAAe,MAAM,IAAI;AAEpD,YAAQ,KAAK,WAAW;AACxB,QAAI,OAAQ,MAA8B,OAAO,WAAW,UAAU;AACpE,cAAQ,GAAI,MAA8B,OAAO,MAAM;AAAA,IACzD,WACE,OAAQ,MAA8B,OAAO,eAAe,UAC5D;AACA,cAAQ,eAAgB,MAA8B,KAAK;AAAA,IAC7D,WACE,OAAQ,MAA8B,OAAO,UAAU,aACvD;AACA,cAAS,MAA8B,OAAO;AAAA,IAChD;AAEA,QAAI,WAAW;AACb,UAAI,OAAO;AACT,gBAAQ,GAAG,SAAS,MAAM,KAAK;AAAA,MACjC,OAAO;AACL,gBAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,UAAU;AAAa,WAAO;AACzC,SAAO,OAAO,UAAU,WACpB,QACA,KAAK,UAAU,OAAO,QAAW,CAAC;AACxC;AAEO,IAAM,wBAAwB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA","names":[],"ignoreList":[],"sources":["../../src/common/ui-utils.ts"],"sourcesContent":["import type {\r\n  DetailedLocateParam,\r\n  ExecutionTask,\r\n  ExecutionTaskAction,\r\n  ExecutionTaskInsightAssertion,\r\n  ExecutionTaskInsightLocate,\r\n  ExecutionTaskInsightQuery,\r\n  ExecutionTaskPlanning,\r\n  PlanningActionParamScroll,\r\n} from 'misoai-core';\r\n\r\nexport function typeStr(task: ExecutionTask) {\r\n  return task.subType && task.subType !== 'Plan'\r\n    ? `${task.type} / ${task.subType || ''}`\r\n    : task.type;\r\n}\r\n\r\nexport function getKeyCommands(\r\n  value: string | string[],\r\n): Array<{ key: string; command?: string }> {\r\n  // Ensure value is an array of keys\r\n  const keys = Array.isArray(value) ? value : [value];\r\n\r\n  // Process each key to attach a corresponding command if needed, based on the presence of 'Meta' or 'Control' in the keys array.\r\n  // ref: https://github.com/puppeteer/puppeteer/pull/9357/files#diff-32cf475237b000f980eb214a0a823e45a902bddb7d2426d677cae96397aa0ae4R94\r\n  return keys.reduce((acc: Array<{ key: string; command?: string }>, k) => {\r\n    const includeMeta = keys.includes('Meta') || keys.includes('Control');\r\n    if (includeMeta && (k === 'a' || k === 'A')) {\r\n      return acc.concat([{ key: k, command: 'SelectAll' }]);\r\n    }\r\n    if (includeMeta && (k === 'c' || k === 'C')) {\r\n      return acc.concat([{ key: k, command: 'Copy' }]);\r\n    }\r\n    if (includeMeta && (k === 'v' || k === 'V')) {\r\n      return acc.concat([{ key: k, command: 'Paste' }]);\r\n    }\r\n    return acc.concat([{ key: k }]);\r\n  }, []);\r\n}\r\n\r\nexport function locateParamStr(locate?: DetailedLocateParam) {\r\n  if (!locate) {\r\n    return '';\r\n  }\r\n\r\n  if (typeof locate === 'string') {\r\n    return locate;\r\n  }\r\n\r\n  return locate.prompt;\r\n}\r\n\r\nexport function scrollParamStr(scrollParam?: PlanningActionParamScroll) {\r\n  if (!scrollParam) {\r\n    return '';\r\n  }\r\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\r\n}\r\n\r\nexport function taskTitleStr(\r\n  type:\r\n    | 'Tap'\r\n    | 'Hover'\r\n    | 'Input'\r\n    | 'KeyboardPress'\r\n    | 'Scroll'\r\n    | 'Action'\r\n    | 'Query'\r\n    | 'Assert'\r\n    | 'WaitFor'\r\n    | 'Locate'\r\n    | 'Boolean'\r\n    | 'Number'\r\n    | 'String',\r\n  prompt: string,\r\n) {\r\n  if (prompt) {\r\n    return `${type} - ${prompt}`;\r\n  }\r\n  return type;\r\n}\r\n\r\nexport function paramStr(task: ExecutionTask) {\r\n  let value: string | undefined | object;\r\n  if (task.type === 'Planning') {\r\n    value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\r\n  }\r\n\r\n  if (task.type === 'Insight') {\r\n    value =\r\n      (task as ExecutionTaskInsightLocate)?.param?.prompt ||\r\n      (task as ExecutionTaskInsightLocate)?.param?.id ||\r\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\r\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\r\n  }\r\n\r\n  if (task.type === 'Action') {\r\n    const locate = (task as ExecutionTaskAction)?.locate;\r\n    const locateStr = locate ? locateParamStr(locate) : '';\r\n\r\n    value = task.thought || '';\r\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\r\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\r\n    } else if (\r\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\r\n    ) {\r\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\r\n    } else if (\r\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\r\n    ) {\r\n      value = (task as ExecutionTaskAction)?.param?.value;\r\n    }\r\n\r\n    if (locateStr) {\r\n      if (value) {\r\n        value = `${locateStr} - ${value}`;\r\n      } else {\r\n        value = locateStr;\r\n      }\r\n    }\r\n  }\r\n\r\n  if (typeof value === 'undefined') return '';\r\n  return typeof value === 'string'\r\n    ? value\r\n    : JSON.stringify(value, undefined, 2);\r\n}\r\n\r\nexport const limitOpenNewTabScript = `\r\nif (!window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__) {\r\n  window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__ = true;\r\n\r\n  // Intercept the window.open method (only once)\r\n  window.open = function(url) {\r\n    console.log('Blocked window.open:', url);\r\n    window.location.href = url;\r\n    return null;\r\n  };\r\n\r\n  // Block all a tag clicks with target=\"_blank\" (only once)\r\n  document.addEventListener('click', function(e) {\r\n    const target = e.target.closest('a');\r\n    if (target && target.target === '_blank') {\r\n      e.preventDefault();\r\n      console.log('Blocked new tab:', target.href);\r\n      window.location.href = target.href;\r\n      target.removeAttribute('target');\r\n    }\r\n  }, true);\r\n}\r\n`;\r\n"]}

package/dist/es/utils.js

@@ -0,0 +1,197 @@
+// src/common/utils.ts
+import { elementByPositionWithElementInfo } from "misoai-core/ai-model";
+import { uploadTestInfoToServer } from "misoai-core/utils";
+import { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from "misoai-shared/env";
+import {
+  generateElementByPosition,
+  getNodeFromCacheList,
+  traverseTree,
+  treeToList
+} from "misoai-shared/extractor";
+import { resizeImgBase64 } from "misoai-shared/img";
+import { assert, logMsg, uuid } from "misoai-shared/utils";
+import dayjs from "dayjs";
+
+// src/web-element.ts
+var WebElementInfo = class {
+  constructor({
+    content,
+    rect,
+    // page,
+    locator,
+    id,
+    attributes,
+    indexId,
+    xpaths
+  }) {
+    this.content = content;
+    this.rect = rect;
+    this.center = [
+      Math.floor(rect.left + rect.width / 2),
+      Math.floor(rect.top + rect.height / 2)
+    ];
+    this.locator = locator;
+    this.id = id;
+    this.attributes = attributes;
+    this.indexId = indexId;
+    this.xpaths = xpaths;
+  }
+};
+
+// src/common/utils.ts
+async function parseContextFromWebPage(page, _opt) {
+  assert(page, "page is required");
+  if (page._forceUsePageContext) {
+    return await page._forceUsePageContext();
+  }
+  const url = await page.url();
+  uploadTestInfoToServer({ testUrl: url });
+  let screenshotBase64;
+  let tree;
+  await Promise.all([
+    page.screenshotBase64().then((base64) => {
+      screenshotBase64 = base64;
+    }),
+    page.getElementsNodeTree().then(async (treeRoot) => {
+      tree = treeRoot;
+    })
+  ]);
+  const webTree = traverseTree(tree, (elementInfo) => {
+    const { rect, id, content, attributes, locator, indexId } = elementInfo;
+    return new WebElementInfo({
+      rect,
+      locator,
+      id,
+      content,
+      attributes,
+      indexId
+    });
+  });
+  assert(screenshotBase64, "screenshotBase64 is required");
+  const elementsInfo = treeToList(webTree);
+  const size = await page.size();
+  if (size.dpr && size.dpr > 1) {
+    screenshotBase64 = await resizeImgBase64(screenshotBase64, {
+      width: size.width,
+      height: size.height
+    });
+  }
+  return {
+    content: elementsInfo,
+    tree: webTree,
+    size,
+    screenshotBase64,
+    url
+  };
+}
+function reportFileName(tag = "web") {
+  const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);
+  const dateTimeInFileName = dayjs().format("YYYY-MM-DD_HH-mm-ss");
+  const uniqueId = uuid().substring(0, 8);
+  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;
+}
+function printReportMsg(filepath) {
+  logMsg(`Midscene - report file updated: ${filepath}`);
+}
+function getCurrentExecutionFile(trace) {
+  const error = new Error();
+  const stackTrace = trace || error.stack;
+  const pkgDir = process.cwd() || "";
+  if (stackTrace) {
+    const stackLines = stackTrace.split("\n");
+    for (const line of stackLines) {
+      if (line.includes(".spec.") || line.includes(".test.") || line.includes(".ts") || line.includes(".js")) {
+        const match = line.match(/(?:at\s+)?(.*?\.(?:spec|test)\.[jt]s)/);
+        if (match?.[1]) {
+          const targetFileName = match[1].replace(pkgDir, "").trim().replace("at ", "");
+          return targetFileName;
+        }
+      }
+    }
+  }
+  return false;
+}
+var testFileIndex = /* @__PURE__ */ new Map();
+function generateCacheId(fileName) {
+  let taskFile = fileName || getCurrentExecutionFile();
+  if (!taskFile) {
+    taskFile = uuid();
+    console.warn(
+      "Midscene - using random UUID for cache id. Cache may be invalid."
+    );
+  }
+  if (testFileIndex.has(taskFile)) {
+    const currentIndex = testFileIndex.get(taskFile);
+    if (currentIndex !== void 0) {
+      testFileIndex.set(taskFile, currentIndex + 1);
+    }
+  } else {
+    testFileIndex.set(taskFile, 1);
+  }
+  return `${taskFile}-${testFileIndex.get(taskFile)}`;
+}
+var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
+function replaceIllegalPathCharsAndSpace(str) {
+  return str.replace(/[/\\:*?"<>| ]/g, "-");
+}
+function forceClosePopup(page, debug) {
+  page.on("popup", async (popup) => {
+    if (!popup) {
+      console.warn("got a popup event, but the popup is not ready yet, skip");
+      return;
+    }
+    const url = await popup.url();
+    console.log(`Popup opened: ${url}`);
+    if (!popup.isClosed()) {
+      try {
+        await popup.close();
+      } catch (error) {
+        debug(`failed to close popup ${url}, error: ${error}`);
+      }
+    } else {
+      debug(`popup is already closed, skip close ${url}`);
+    }
+    if (!page.isClosed()) {
+      try {
+        await page.goto(url);
+      } catch (error) {
+        debug(`failed to goto ${url}, error: ${error}`);
+      }
+    } else {
+      debug(`page is already closed, skip goto ${url}`);
+    }
+  });
+}
+function matchElementFromPlan(planLocateParam, tree) {
+  if (!planLocateParam) {
+    return void 0;
+  }
+  if (planLocateParam.id) {
+    return getNodeFromCacheList(planLocateParam.id);
+  }
+  if (planLocateParam.bbox) {
+    const centerPosition = {
+      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),
+      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2)
+    };
+    let element = elementByPositionWithElementInfo(tree, centerPosition);
+    if (!element) {
+      element = generateElementByPosition(centerPosition);
+    }
+    return element;
+  }
+  return void 0;
+}
+export {
+  ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED,
+  forceClosePopup,
+  generateCacheId,
+  getCurrentExecutionFile,
+  matchElementFromPlan,
+  parseContextFromWebPage,
+  printReportMsg,
+  replaceIllegalPathCharsAndSpace,
+  reportFileName
+};
+
+//# sourceMappingURL=utils.js.map

package/dist/es/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"mappings":";AAQA,SAAS,wCAAwC;AACjD,SAAS,8BAA8B;AACvC,SAAS,0BAA0B,mBAAmB;AAEtD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uBAAuB;AAEhC,SAAS,QAAQ,QAAQ,YAAY;AACrC,OAAO,WAAW;;;ACVX,IAAM,iBAAN,MAA4C;AAAA,EAsBjD,YAAY;AAAA,IACV;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAYG;AACD,SAAK,UAAU;AACf,SAAK,OAAO;AACZ,SAAK,SAAS;AAAA,MACZ,KAAK,MAAM,KAAK,OAAO,KAAK,QAAQ,CAAC;AAAA,MACrC,KAAK,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC;AAAA,IACvC;AAEA,SAAK,UAAU;AACf,SAAK,KAAK;AACV,SAAK,aAAa;AAClB,SAAK,UAAU;AACf,SAAK,SAAS;AAAA,EAChB;AACF;;;ADrCA,eAAsB,wBACpB,MACA,MACuB;AACvB,SAAO,MAAM,kBAAkB;AAC/B,MAAK,KAAoB,sBAAsB;AAC7C,WAAO,MAAO,KAAa,qBAAqB;AAAA,EAClD;AACA,QAAM,MAAM,MAAM,KAAK,IAAI;AAC3B,yBAAuB,EAAE,SAAS,IAAI,CAAC;AAEvC,MAAI;AACJ,MAAI;AAEJ,QAAM,QAAQ,IAAI;AAAA,IAChB,KAAK,iBAAiB,EAAE,KAAK,CAAC,WAAW;AACvC,yBAAmB;AAAA,IACrB,CAAC;AAAA,IACD,KAAK,oBAAoB,EAAE,KAAK,OAAO,aAAa;AAClD,aAAO;AAAA,IACT,CAAC;AAAA,EACH,CAAC;AAED,QAAM,UAAU,aAAa,MAAO,CAAC,gBAAgB;AACnD,UAAM,EAAE,MAAM,IAAI,SAAS,YAAY,SAAS,QAAQ,IAAI;AAC5D,WAAO,IAAI,eAAe;AAAA,MACxB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,SAAO,kBAAmB,8BAA8B;AAExD,QAAM,eAAe,WAAW,OAAO;AACvC,QAAM,OAAO,MAAM,KAAK,KAAK;AAE7B,MAAI,KAAK,OAAO,KAAK,MAAM,GAAG;AAE5B,uBAAmB,MAAM,gBAAgB,kBAAkB;AAAA,MACzD,OAAO,KAAK;AAAA,MACZ,QAAQ,KAAK;AAAA,IACf,CAAC;AAAA,EAEH;AAEA,SAAO;AAAA,IACL,SAAS;AAAA,IACT,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,eAAe,MAAM,OAAO;AAC1C,QAAM,gBAAgB,YAAY,wBAAwB;AAC1D,QAAM,qBAAqB,MAAM,EAAE,OAAO,qBAAqB;AAE/D,QAAM,WAAW,KAAK,EAAE,UAAU,GAAG,CAAC;AACtC,SAAO,GAAG,iBAAiB,GAAG,IAAI,kBAAkB,IAAI,QAAQ;AAClE;AAEO,SAAS,eAAe,UAAkB;AAC/C,SAAO,mCAAmC,QAAQ,EAAE;AACtD;AAMO,SAAS,wBAAwB,OAAgC;AACtE,QAAM,QAAQ,IAAI,MAAM;AACxB,QAAM,aAAa,SAAS,MAAM;AAClC,QAAM,SAAS,QAAQ,IAAI,KAAK;AAChC,MAAI,YAAY;AACd,UAAM,aAAa,WAAW,MAAM,IAAI;AACxC,eAAW,QAAQ,YAAY;AAC7B,UACE,KAAK,SAAS,QAAQ,KACtB,KAAK,SAAS,QAAQ,KACtB,KAAK,SAAS,KAAK,KACnB,KAAK,SAAS,KAAK,GACnB;AACA,cAAM,QAAQ,KAAK,MAAM,uCAAuC;AAChE,YAAI,QAAQ,CAAC,GAAG;AACd,gBAAM,iBAAiB,MAAM,CAAC,EAC3B,QAAQ,QAAQ,EAAE,EAClB,KAAK,EACL,QAAQ,OAAO,EAAE;AACpB,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,gBAAgB,oBAAI,IAAoB;AAEvC,SAAS,gBAAgB,UAA2B;AACzD,MAAI,WAAW,YAAY,wBAAwB;AACnD,MAAI,CAAC,UAAU;AACb,eAAW,KAAK;AAChB,YAAQ;AAAA,MACN;AAAA,IACF;AAAA,EACF;AAEA,MAAI,cAAc,IAAI,QAAQ,GAAG;AAC/B,UAAM,eAAe,cAAc,IAAI,QAAQ;AAC/C,QAAI,iBAAiB,QAAW;AAC9B,oBAAc,IAAI,UAAU,eAAe,CAAC;AAAA,IAC9C;AAAA,EACF,OAAO;AACL,kBAAc,IAAI,UAAU,CAAC;AAAA,EAC/B;AACA,SAAO,GAAG,QAAQ,IAAI,cAAc,IAAI,QAAQ,CAAC;AACnD;AAEO,IAAM,yCACX;AAEK,SAAS,gCAAgC,KAAa;AAC3D,SAAO,IAAI,QAAQ,kBAAkB,GAAG;AAC1C;AAEO,SAAS,gBACd,MACA,OACA;AACA,OAAK,GAAG,SAAS,OAAO,UAAU;AAChC,QAAI,CAAC,OAAO;AACV,cAAQ,KAAK,yDAAyD;AACtE;AAAA,IACF;AACA,UAAM,MAAM,MAAO,MAAwB,IAAI;AAC/C,YAAQ,IAAI,iBAAiB,GAAG,EAAE;AAClC,QAAI,CAAE,MAAwB,SAAS,GAAG;AACxC,UAAI;AACF,cAAO,MAAwB,MAAM;AAAA,MACvC,SAAS,OAAO;AACd,cAAM,yBAAyB,GAAG,YAAY,KAAK,EAAE;AAAA,MACvD;AAAA,IACF,OAAO;AACL,YAAM,uCAAuC,GAAG,EAAE;AAAA,IACpD;AAEA,QAAI,CAAC,KAAK,SAAS,GAAG;AACpB,UAAI;AACF,cAAM,KAAK,KAAK,GAAG;AAAA,MACrB,SAAS,OAAO;AACd,cAAM,kBAAkB,GAAG,YAAY,KAAK,EAAE;AAAA,MAChD;AAAA,IACF,OAAO;AACL,YAAM,qCAAqC,GAAG,EAAE;AAAA,IAClD;AAAA,EACF,CAAC;AACH;AAEO,SAAS,qBACd,iBACA,MACA;AACA,MAAI,CAAC,iBAAiB;AACpB,WAAO;AAAA,EACT;AACA,MAAI,gBAAgB,IAAI;AACtB,WAAO,qBAAqB,gBAAgB,EAAE;AAAA,EAChD;AAEA,MAAI,gBAAgB,MAAM;AACxB,UAAM,iBAAiB;AAAA,MACrB,GAAG,KAAK,OAAO,gBAAgB,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,KAAK,CAAC;AAAA,MACrE,GAAG,KAAK,OAAO,gBAAgB,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,KAAK,CAAC;AAAA,IACvE;AACA,QAAI,UAAU,iCAAiC,MAAM,cAAc;AAEnE,QAAI,CAAC,SAAS;AACZ,gBAAU,0BAA0B,cAAc;AAAA,IACpD;AAEA,WAAO;AAAA,EACT;AAEA,SAAO;AACT","names":[],"ignoreList":[],"sources":["../../src/common/utils.ts","../../src/web-element.ts"],"sourcesContent":["import type { StaticPage } from '@/playground';\r\nimport type {\r\n  BaseElement,\r\n  ElementTreeNode,\r\n  PlanningLocateParam,\r\n  PlaywrightParserOpt,\r\n  UIContext,\r\n} from 'misoai-core';\r\nimport { elementByPositionWithElementInfo } from 'misoai-core/ai-model';\r\nimport { uploadTestInfoToServer } from 'misoai-core/utils';\r\nimport { MIDSCENE_REPORT_TAG_NAME, getAIConfig } from 'misoai-shared/env';\r\nimport type { ElementInfo } from 'misoai-shared/extractor';\r\nimport {\r\n  generateElementByPosition,\r\n  getNodeFromCacheList,\r\n  traverseTree,\r\n  treeToList,\r\n} from 'misoai-shared/extractor';\r\nimport { resizeImgBase64 } from 'misoai-shared/img';\r\nimport type { DebugFunction } from 'misoai-shared/logger';\r\nimport { assert, logMsg, uuid } from 'misoai-shared/utils';\r\nimport dayjs from 'dayjs';\r\nimport type { Page as PlaywrightPage } from 'playwright';\r\nimport type { Page as PuppeteerPage } from 'puppeteer';\r\nimport { WebElementInfo } from '../web-element';\r\nimport type { WebPage } from './page';\r\n\r\nexport type WebUIContext = UIContext<WebElementInfo> & {\r\n  url: string;\r\n};\r\n\r\nexport async function parseContextFromWebPage(\r\n  page: WebPage,\r\n  _opt?: PlaywrightParserOpt,\r\n): Promise<WebUIContext> {\r\n  assert(page, 'page is required');\r\n  if ((page as StaticPage)._forceUsePageContext) {\r\n    return await (page as any)._forceUsePageContext();\r\n  }\r\n  const url = await page.url();\r\n  uploadTestInfoToServer({ testUrl: url });\r\n\r\n  let screenshotBase64: string;\r\n  let tree: ElementTreeNode<ElementInfo>;\r\n\r\n  await Promise.all([\r\n    page.screenshotBase64().then((base64) => {\r\n      screenshotBase64 = base64;\r\n    }),\r\n    page.getElementsNodeTree().then(async (treeRoot) => {\r\n      tree = treeRoot;\r\n    }),\r\n  ]);\r\n\r\n  const webTree = traverseTree(tree!, (elementInfo) => {\r\n    const { rect, id, content, attributes, locator, indexId } = elementInfo;\r\n    return new WebElementInfo({\r\n      rect,\r\n      locator,\r\n      id,\r\n      content,\r\n      attributes,\r\n      indexId,\r\n    });\r\n  });\r\n\r\n  assert(screenshotBase64!, 'screenshotBase64 is required');\r\n\r\n  const elementsInfo = treeToList(webTree);\r\n  const size = await page.size();\r\n\r\n  if (size.dpr && size.dpr > 1) {\r\n    // console.time('resizeImgBase64');\r\n    screenshotBase64 = await resizeImgBase64(screenshotBase64, {\r\n      width: size.width,\r\n      height: size.height,\r\n    });\r\n    // console.timeEnd('resizeImgBase64');\r\n  }\r\n\r\n  return {\r\n    content: elementsInfo!,\r\n    tree: webTree,\r\n    size,\r\n    screenshotBase64: screenshotBase64!,\r\n    url,\r\n  };\r\n}\r\n\r\nexport function reportFileName(tag = 'web') {\r\n  const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);\r\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\r\n  // ensure uniqueness at the same time\r\n  const uniqueId = uuid().substring(0, 8);\r\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\r\n}\r\n\r\nexport function printReportMsg(filepath: string) {\r\n  logMsg(`Midscene - report file updated: ${filepath}`);\r\n}\r\n\r\n/**\r\n * Get the current execution file name\r\n * @returns The name of the current execution file\r\n */\r\nexport function getCurrentExecutionFile(trace?: string): string | false {\r\n  const error = new Error();\r\n  const stackTrace = trace || error.stack;\r\n  const pkgDir = process.cwd() || '';\r\n  if (stackTrace) {\r\n    const stackLines = stackTrace.split('\\n');\r\n    for (const line of stackLines) {\r\n      if (\r\n        line.includes('.spec.') ||\r\n        line.includes('.test.') ||\r\n        line.includes('.ts') ||\r\n        line.includes('.js')\r\n      ) {\r\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\r\n        if (match?.[1]) {\r\n          const targetFileName = match[1]\r\n            .replace(pkgDir, '')\r\n            .trim()\r\n            .replace('at ', '');\r\n          return targetFileName;\r\n        }\r\n      }\r\n    }\r\n  }\r\n  return false;\r\n}\r\n\r\nconst testFileIndex = new Map<string, number>();\r\n\r\nexport function generateCacheId(fileName?: string): string {\r\n  let taskFile = fileName || getCurrentExecutionFile();\r\n  if (!taskFile) {\r\n    taskFile = uuid();\r\n    console.warn(\r\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\r\n    );\r\n  }\r\n\r\n  if (testFileIndex.has(taskFile)) {\r\n    const currentIndex = testFileIndex.get(taskFile);\r\n    if (currentIndex !== undefined) {\r\n      testFileIndex.set(taskFile, currentIndex + 1);\r\n    }\r\n  } else {\r\n    testFileIndex.set(taskFile, 1);\r\n  }\r\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\r\n}\r\n\r\nexport const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =\r\n  'NOT_IMPLEMENTED_AS_DESIGNED';\r\n\r\nexport function replaceIllegalPathCharsAndSpace(str: string) {\r\n  return str.replace(/[/\\\\:*?\"<>| ]/g, '-');\r\n}\r\n\r\nexport function forceClosePopup(\r\n  page: PuppeteerPage | PlaywrightPage,\r\n  debug: DebugFunction,\r\n) {\r\n  page.on('popup', async (popup) => {\r\n    if (!popup) {\r\n      console.warn('got a popup event, but the popup is not ready yet, skip');\r\n      return;\r\n    }\r\n    const url = await (popup as PuppeteerPage).url();\r\n    console.log(`Popup opened: ${url}`);\r\n    if (!(popup as PuppeteerPage).isClosed()) {\r\n      try {\r\n        await (popup as PuppeteerPage).close(); // Close the newly opened TAB\r\n      } catch (error) {\r\n        debug(`failed to close popup ${url}, error: ${error}`);\r\n      }\r\n    } else {\r\n      debug(`popup is already closed, skip close ${url}`);\r\n    }\r\n\r\n    if (!page.isClosed()) {\r\n      try {\r\n        await page.goto(url);\r\n      } catch (error) {\r\n        debug(`failed to goto ${url}, error: ${error}`);\r\n      }\r\n    } else {\r\n      debug(`page is already closed, skip goto ${url}`);\r\n    }\r\n  });\r\n}\r\n\r\nexport function matchElementFromPlan(\r\n  planLocateParam: PlanningLocateParam,\r\n  tree: ElementTreeNode<BaseElement>,\r\n) {\r\n  if (!planLocateParam) {\r\n    return undefined;\r\n  }\r\n  if (planLocateParam.id) {\r\n    return getNodeFromCacheList(planLocateParam.id);\r\n  }\r\n\r\n  if (planLocateParam.bbox) {\r\n    const centerPosition = {\r\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\r\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\r\n    };\r\n    let element = elementByPositionWithElementInfo(tree, centerPosition);\r\n\r\n    if (!element) {\r\n      element = generateElementByPosition(centerPosition) as BaseElement;\r\n    }\r\n\r\n    return element;\r\n  }\r\n\r\n  return undefined;\r\n}\r\n","import type { BaseElement, Rect } from 'misoai-core';\r\nimport type { NodeType } from 'misoai-shared/constants';\r\nexport interface WebElementInfoType extends BaseElement {\r\n  id: string;\r\n  locator: string;\r\n  attributes: {\r\n    nodeType: NodeType;\r\n    [key: string]: string;\r\n  };\r\n}\r\n\r\nexport class WebElementInfo implements BaseElement {\r\n  content: string;\r\n\r\n  locator?: string;\r\n\r\n  rect: Rect;\r\n\r\n  center: [number, number];\r\n\r\n  // page: WebPage;\r\n\r\n  id: string;\r\n\r\n  indexId: number;\r\n\r\n  attributes: {\r\n    nodeType: NodeType;\r\n    [key: string]: string;\r\n  };\r\n\r\n  xpaths?: string[];\r\n\r\n  constructor({\r\n    content,\r\n    rect,\r\n    // page,\r\n    locator,\r\n    id,\r\n    attributes,\r\n    indexId,\r\n    xpaths,\r\n  }: {\r\n    content: string;\r\n    rect: Rect;\r\n    // page: WebPage;\r\n    locator?: string;\r\n    id: string;\r\n    attributes: {\r\n      nodeType: NodeType;\r\n      [key: string]: string;\r\n    };\r\n    indexId: number;\r\n    xpaths?: string[];\r\n  }) {\r\n    this.content = content;\r\n    this.rect = rect;\r\n    this.center = [\r\n      Math.floor(rect.left + rect.width / 2),\r\n      Math.floor(rect.top + rect.height / 2),\r\n    ];\r\n    // this.page = page;\r\n    this.locator = locator;\r\n    this.id = id;\r\n    this.attributes = attributes;\r\n    this.indexId = indexId;\r\n    this.xpaths = xpaths;\r\n  }\r\n}\r\n"]}