@midscene/web 0.3.2 → 0.3.3

package/dist/lib/playwright.js

@@ -0,0 +1,1245 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __defProps = Object.defineProperties;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+var __commonJS = (cb, mod) => function __require() {
+  return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// ../../node_modules/.pnpm/dayjs@1.11.11/node_modules/dayjs/dayjs.min.js
+var require_dayjs_min = __commonJS({
+  "../../node_modules/.pnpm/dayjs@1.11.11/node_modules/dayjs/dayjs.min.js"(exports, module2) {
+    "use strict";
+    !function(t, e) {
+      "object" == typeof exports && "undefined" != typeof module2 ? module2.exports = e() : "function" == typeof define && define.amd ? define(e) : (t = "undefined" != typeof globalThis ? globalThis : t || self).dayjs = e();
+    }(exports, function() {
+      "use strict";
+      var t = 1e3, e = 6e4, n = 36e5, r = "millisecond", i = "second", s = "minute", u = "hour", a = "day", o = "week", c = "month", f = "quarter", h = "year", d = "date", l = "Invalid Date", $ = /^(\d{4})[-/]?(\d{1,2})?[-/]?(\d{0,2})[Tt\s]*(\d{1,2})?:?(\d{1,2})?:?(\d{1,2})?[.:]?(\d+)?$/, y = /\[([^\]]+)]|Y{1,4}|M{1,4}|D{1,2}|d{1,4}|H{1,2}|h{1,2}|a|A|m{1,2}|s{1,2}|Z{1,2}|SSS/g, M = { name: "en", weekdays: "Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"), months: "January_February_March_April_May_June_July_August_September_October_November_December".split("_"), ordinal: function(t2) {
+        var e2 = ["th", "st", "nd", "rd"], n2 = t2 % 100;
+        return "[" + t2 + (e2[(n2 - 20) % 10] || e2[n2] || e2[0]) + "]";
+      } }, m = function(t2, e2, n2) {
+        var r2 = String(t2);
+        return !r2 || r2.length >= e2 ? t2 : "" + Array(e2 + 1 - r2.length).join(n2) + t2;
+      }, v = { s: m, z: function(t2) {
+        var e2 = -t2.utcOffset(), n2 = Math.abs(e2), r2 = Math.floor(n2 / 60), i2 = n2 % 60;
+        return (e2 <= 0 ? "+" : "-") + m(r2, 2, "0") + ":" + m(i2, 2, "0");
+      }, m: function t2(e2, n2) {
+        if (e2.date() < n2.date())
+          return -t2(n2, e2);
+        var r2 = 12 * (n2.year() - e2.year()) + (n2.month() - e2.month()), i2 = e2.clone().add(r2, c), s2 = n2 - i2 < 0, u2 = e2.clone().add(r2 + (s2 ? -1 : 1), c);
+        return +(-(r2 + (n2 - i2) / (s2 ? i2 - u2 : u2 - i2)) || 0);
+      }, a: function(t2) {
+        return t2 < 0 ? Math.ceil(t2) || 0 : Math.floor(t2);
+      }, p: function(t2) {
+        return { M: c, y: h, w: o, d: a, D: d, h: u, m: s, s: i, ms: r, Q: f }[t2] || String(t2 || "").toLowerCase().replace(/s$/, "");
+      }, u: function(t2) {
+        return void 0 === t2;
+      } }, g = "en", D = {};
+      D[g] = M;
+      var p = "$isDayjsObject", S = function(t2) {
+        return t2 instanceof _ || !(!t2 || !t2[p]);
+      }, w = function t2(e2, n2, r2) {
+        var i2;
+        if (!e2)
+          return g;
+        if ("string" == typeof e2) {
+          var s2 = e2.toLowerCase();
+          D[s2] && (i2 = s2), n2 && (D[s2] = n2, i2 = s2);
+          var u2 = e2.split("-");
+          if (!i2 && u2.length > 1)
+            return t2(u2[0]);
+        } else {
+          var a2 = e2.name;
+          D[a2] = e2, i2 = a2;
+        }
+        return !r2 && i2 && (g = i2), i2 || !r2 && g;
+      }, O = function(t2, e2) {
+        if (S(t2))
+          return t2.clone();
+        var n2 = "object" == typeof e2 ? e2 : {};
+        return n2.date = t2, n2.args = arguments, new _(n2);
+      }, b = v;
+      b.l = w, b.i = S, b.w = function(t2, e2) {
+        return O(t2, { locale: e2.$L, utc: e2.$u, x: e2.$x, $offset: e2.$offset });
+      };
+      var _ = function() {
+        function M2(t2) {
+          this.$L = w(t2.locale, null, true), this.parse(t2), this.$x = this.$x || t2.x || {}, this[p] = true;
+        }
+        var m2 = M2.prototype;
+        return m2.parse = function(t2) {
+          this.$d = function(t3) {
+            var e2 = t3.date, n2 = t3.utc;
+            if (null === e2)
+              return /* @__PURE__ */ new Date(NaN);
+            if (b.u(e2))
+              return /* @__PURE__ */ new Date();
+            if (e2 instanceof Date)
+              return new Date(e2);
+            if ("string" == typeof e2 && !/Z$/i.test(e2)) {
+              var r2 = e2.match($);
+              if (r2) {
+                var i2 = r2[2] - 1 || 0, s2 = (r2[7] || "0").substring(0, 3);
+                return n2 ? new Date(Date.UTC(r2[1], i2, r2[3] || 1, r2[4] || 0, r2[5] || 0, r2[6] || 0, s2)) : new Date(r2[1], i2, r2[3] || 1, r2[4] || 0, r2[5] || 0, r2[6] || 0, s2);
+              }
+            }
+            return new Date(e2);
+          }(t2), this.init();
+        }, m2.init = function() {
+          var t2 = this.$d;
+          this.$y = t2.getFullYear(), this.$M = t2.getMonth(), this.$D = t2.getDate(), this.$W = t2.getDay(), this.$H = t2.getHours(), this.$m = t2.getMinutes(), this.$s = t2.getSeconds(), this.$ms = t2.getMilliseconds();
+        }, m2.$utils = function() {
+          return b;
+        }, m2.isValid = function() {
+          return !(this.$d.toString() === l);
+        }, m2.isSame = function(t2, e2) {
+          var n2 = O(t2);
+          return this.startOf(e2) <= n2 && n2 <= this.endOf(e2);
+        }, m2.isAfter = function(t2, e2) {
+          return O(t2) < this.startOf(e2);
+        }, m2.isBefore = function(t2, e2) {
+          return this.endOf(e2) < O(t2);
+        }, m2.$g = function(t2, e2, n2) {
+          return b.u(t2) ? this[e2] : this.set(n2, t2);
+        }, m2.unix = function() {
+          return Math.floor(this.valueOf() / 1e3);
+        }, m2.valueOf = function() {
+          return this.$d.getTime();
+        }, m2.startOf = function(t2, e2) {
+          var n2 = this, r2 = !!b.u(e2) || e2, f2 = b.p(t2), l2 = function(t3, e3) {
+            var i2 = b.w(n2.$u ? Date.UTC(n2.$y, e3, t3) : new Date(n2.$y, e3, t3), n2);
+            return r2 ? i2 : i2.endOf(a);
+          }, $2 = function(t3, e3) {
+            return b.w(n2.toDate()[t3].apply(n2.toDate("s"), (r2 ? [0, 0, 0, 0] : [23, 59, 59, 999]).slice(e3)), n2);
+          }, y2 = this.$W, M3 = this.$M, m3 = this.$D, v2 = "set" + (this.$u ? "UTC" : "");
+          switch (f2) {
+            case h:
+              return r2 ? l2(1, 0) : l2(31, 11);
+            case c:
+              return r2 ? l2(1, M3) : l2(0, M3 + 1);
+            case o:
+              var g2 = this.$locale().weekStart || 0, D2 = (y2 < g2 ? y2 + 7 : y2) - g2;
+              return l2(r2 ? m3 - D2 : m3 + (6 - D2), M3);
+            case a:
+            case d:
+              return $2(v2 + "Hours", 0);
+            case u:
+              return $2(v2 + "Minutes", 1);
+            case s:
+              return $2(v2 + "Seconds", 2);
+            case i:
+              return $2(v2 + "Milliseconds", 3);
+            default:
+              return this.clone();
+          }
+        }, m2.endOf = function(t2) {
+          return this.startOf(t2, false);
+        }, m2.$set = function(t2, e2) {
+          var n2, o2 = b.p(t2), f2 = "set" + (this.$u ? "UTC" : ""), l2 = (n2 = {}, n2[a] = f2 + "Date", n2[d] = f2 + "Date", n2[c] = f2 + "Month", n2[h] = f2 + "FullYear", n2[u] = f2 + "Hours", n2[s] = f2 + "Minutes", n2[i] = f2 + "Seconds", n2[r] = f2 + "Milliseconds", n2)[o2], $2 = o2 === a ? this.$D + (e2 - this.$W) : e2;
+          if (o2 === c || o2 === h) {
+            var y2 = this.clone().set(d, 1);
+            y2.$d[l2]($2), y2.init(), this.$d = y2.set(d, Math.min(this.$D, y2.daysInMonth())).$d;
+          } else
+            l2 && this.$d[l2]($2);
+          return this.init(), this;
+        }, m2.set = function(t2, e2) {
+          return this.clone().$set(t2, e2);
+        }, m2.get = function(t2) {
+          return this[b.p(t2)]();
+        }, m2.add = function(r2, f2) {
+          var d2, l2 = this;
+          r2 = Number(r2);
+          var $2 = b.p(f2), y2 = function(t2) {
+            var e2 = O(l2);
+            return b.w(e2.date(e2.date() + Math.round(t2 * r2)), l2);
+          };
+          if ($2 === c)
+            return this.set(c, this.$M + r2);
+          if ($2 === h)
+            return this.set(h, this.$y + r2);
+          if ($2 === a)
+            return y2(1);
+          if ($2 === o)
+            return y2(7);
+          var M3 = (d2 = {}, d2[s] = e, d2[u] = n, d2[i] = t, d2)[$2] || 1, m3 = this.$d.getTime() + r2 * M3;
+          return b.w(m3, this);
+        }, m2.subtract = function(t2, e2) {
+          return this.add(-1 * t2, e2);
+        }, m2.format = function(t2) {
+          var e2 = this, n2 = this.$locale();
+          if (!this.isValid())
+            return n2.invalidDate || l;
+          var r2 = t2 || "YYYY-MM-DDTHH:mm:ssZ", i2 = b.z(this), s2 = this.$H, u2 = this.$m, a2 = this.$M, o2 = n2.weekdays, c2 = n2.months, f2 = n2.meridiem, h2 = function(t3, n3, i3, s3) {
+            return t3 && (t3[n3] || t3(e2, r2)) || i3[n3].slice(0, s3);
+          }, d2 = function(t3) {
+            return b.s(s2 % 12 || 12, t3, "0");
+          }, $2 = f2 || function(t3, e3, n3) {
+            var r3 = t3 < 12 ? "AM" : "PM";
+            return n3 ? r3.toLowerCase() : r3;
+          };
+          return r2.replace(y, function(t3, r3) {
+            return r3 || function(t4) {
+              switch (t4) {
+                case "YY":
+                  return String(e2.$y).slice(-2);
+                case "YYYY":
+                  return b.s(e2.$y, 4, "0");
+                case "M":
+                  return a2 + 1;
+                case "MM":
+                  return b.s(a2 + 1, 2, "0");
+                case "MMM":
+                  return h2(n2.monthsShort, a2, c2, 3);
+                case "MMMM":
+                  return h2(c2, a2);
+                case "D":
+                  return e2.$D;
+                case "DD":
+                  return b.s(e2.$D, 2, "0");
+                case "d":
+                  return String(e2.$W);
+                case "dd":
+                  return h2(n2.weekdaysMin, e2.$W, o2, 2);
+                case "ddd":
+                  return h2(n2.weekdaysShort, e2.$W, o2, 3);
+                case "dddd":
+                  return o2[e2.$W];
+                case "H":
+                  return String(s2);
+                case "HH":
+                  return b.s(s2, 2, "0");
+                case "h":
+                  return d2(1);
+                case "hh":
+                  return d2(2);
+                case "a":
+                  return $2(s2, u2, true);
+                case "A":
+                  return $2(s2, u2, false);
+                case "m":
+                  return String(u2);
+                case "mm":
+                  return b.s(u2, 2, "0");
+                case "s":
+                  return String(e2.$s);
+                case "ss":
+                  return b.s(e2.$s, 2, "0");
+                case "SSS":
+                  return b.s(e2.$ms, 3, "0");
+                case "Z":
+                  return i2;
+              }
+              return null;
+            }(t3) || i2.replace(":", "");
+          });
+        }, m2.utcOffset = function() {
+          return 15 * -Math.round(this.$d.getTimezoneOffset() / 15);
+        }, m2.diff = function(r2, d2, l2) {
+          var $2, y2 = this, M3 = b.p(d2), m3 = O(r2), v2 = (m3.utcOffset() - this.utcOffset()) * e, g2 = this - m3, D2 = function() {
+            return b.m(y2, m3);
+          };
+          switch (M3) {
+            case h:
+              $2 = D2() / 12;
+              break;
+            case c:
+              $2 = D2();
+              break;
+            case f:
+              $2 = D2() / 3;
+              break;
+            case o:
+              $2 = (g2 - v2) / 6048e5;
+              break;
+            case a:
+              $2 = (g2 - v2) / 864e5;
+              break;
+            case u:
+              $2 = g2 / n;
+              break;
+            case s:
+              $2 = g2 / e;
+              break;
+            case i:
+              $2 = g2 / t;
+              break;
+            default:
+              $2 = g2;
+          }
+          return l2 ? $2 : b.a($2);
+        }, m2.daysInMonth = function() {
+          return this.endOf(c).$D;
+        }, m2.$locale = function() {
+          return D[this.$L];
+        }, m2.locale = function(t2, e2) {
+          if (!t2)
+            return this.$L;
+          var n2 = this.clone(), r2 = w(t2, e2, true);
+          return r2 && (n2.$L = r2), n2;
+        }, m2.clone = function() {
+          return b.w(this.$d, this);
+        }, m2.toDate = function() {
+          return new Date(this.valueOf());
+        }, m2.toJSON = function() {
+          return this.isValid() ? this.toISOString() : null;
+        }, m2.toISOString = function() {
+          return this.$d.toISOString();
+        }, m2.toString = function() {
+          return this.$d.toUTCString();
+        }, M2;
+      }(), k = _.prototype;
+      return O.prototype = k, [["$ms", r], ["$s", i], ["$m", s], ["$H", u], ["$W", a], ["$M", c], ["$y", h], ["$D", d]].forEach(function(t2) {
+        k[t2[1]] = function(e2) {
+          return this.$g(e2, t2[0], t2[1]);
+        };
+      }), O.extend = function(t2, e2) {
+        return t2.$i || (t2(e2, _, O), t2.$i = true), O;
+      }, O.locale = w, O.isDayjs = S, O.unix = function(t2) {
+        return O(1e3 * t2);
+      }, O.en = D[g], O.Ls = D, O.p = {}, O;
+    });
+  }
+});
+
+// src/playwright/index.ts
+var playwright_exports = {};
+__export(playwright_exports, {
+  PlaywrightAiFixture: () => PlaywrightAiFixture
+});
+module.exports = __toCommonJS(playwright_exports);
+
+// src/playwright/ai-fixture.ts
+var import_node_crypto = require("crypto");
+
+// src/common/agent.ts
+var import_utils4 = require("@midscene/core/utils");
+
+// src/common/tasks.ts
+var import_node_assert2 = __toESM(require("assert"));
+var import_core = __toESM(require("@midscene/core"));
+var import_image2 = require("@midscene/core/image");
+var import_utils2 = require("@midscene/core/utils");
+
+// src/common/task-cache.ts
+var TaskCache = class {
+  constructor(opts) {
+    this.cache = opts == null ? void 0 : opts.cache;
+    this.newCache = {
+      aiTasks: []
+    };
+  }
+  readCache(pageContext, type, userPrompt) {
+    var _a;
+    if (this.cache) {
+      const { aiTasks } = this.cache;
+      const index = aiTasks.findIndex((item) => item.prompt === userPrompt);
+      if (index === -1) {
+        return false;
+      }
+      const taskRes = aiTasks.splice(index, 1)[0];
+      if ((taskRes == null ? void 0 : taskRes.type) === "locate" && !((_a = taskRes.response) == null ? void 0 : _a.elements.every((element) => {
+        const findIndex = pageContext.content.findIndex(
+          (contentElement) => contentElement.id === element.id
+        );
+        if (findIndex === -1) {
+          return false;
+        }
+        return true;
+      }))) {
+        return false;
+      }
+      if (taskRes && taskRes.type === type && taskRes.prompt === userPrompt && this.pageContextEqual(taskRes.pageContext, pageContext)) {
+        return taskRes.response;
+      }
+    }
+    return false;
+  }
+  saveCache(cache) {
+    var _a;
+    if (cache) {
+      (_a = this.newCache) == null ? void 0 : _a.aiTasks.push(cache);
+    }
+  }
+  pageContextEqual(taskPageContext, pageContext) {
+    return taskPageContext.size.width === pageContext.size.width && taskPageContext.size.height === pageContext.size.height;
+  }
+  /**
+   * Generate task cache data.
+   * This method is mainly used to create or obtain some cached data for tasks, and it returns a new cache object.
+   * In the cache object, it may contain task-related information, states, or other necessary data.
+   * It is assumed that the `newCache` property already exists in the current class or object and is a data structure used to store task cache.
+   * @returns {Object} Returns a new cache object, which may include task cache data.
+   */
+  generateTaskCache() {
+    return this.newCache;
+  }
+};
+
+// src/common/utils.ts
+var import_node_assert = __toESM(require("assert"));
+var import_node_fs = __toESM(require("fs"));
+var import_node_path = __toESM(require("path"));
+var import_image = require("@midscene/core/image");
+var import_utils = require("@midscene/core/utils");
+var import_dayjs = __toESM(require_dayjs_min());
+
+// src/web-element.ts
+var WebElementInfo = class {
+  constructor({
+    content,
+    rect,
+    page,
+    locator,
+    id,
+    attributes
+  }) {
+    this.content = content;
+    this.rect = rect;
+    this.center = [
+      Math.floor(rect.left + rect.width / 2),
+      Math.floor(rect.top + rect.height / 2)
+    ];
+    this.page = page;
+    this.locator = locator;
+    this.id = id;
+    this.attributes = attributes;
+  }
+};
+
+// src/common/utils.ts
+async function parseContextFromWebPage(page, _opt) {
+  (0, import_node_assert.default)(page, "page is required");
+  const url = page.url();
+  const file = (0, import_utils.getTmpFile)("jpeg");
+  await page.screenshot({ path: file, type: "jpeg", quality: 75 });
+  const screenshotBuffer = (0, import_node_fs.readFileSync)(file);
+  const screenshotBase64 = (0, import_image.base64Encoded)(file);
+  const captureElementSnapshot = await getElementInfosFromPage(page);
+  const elementsInfo = await alignElements(
+    screenshotBuffer,
+    captureElementSnapshot,
+    page
+  );
+  const size = await (0, import_image.imageInfoOfBase64)(screenshotBase64);
+  return {
+    content: elementsInfo,
+    size,
+    screenshotBase64,
+    url
+  };
+}
+async function getElementInfosFromPage(page) {
+  const pathDir = findNearestPackageJson(__dirname);
+  (0, import_node_assert.default)(pathDir, `can't find pathDir, with ${__dirname}`);
+  const scriptPath = import_node_path.default.join(pathDir, "./dist/script/htmlElement.js");
+  const elementInfosScriptContent = (0, import_node_fs.readFileSync)(scriptPath, "utf-8");
+  const extraReturnLogic = `${elementInfosScriptContent}midscene_element_inspector.extractTextWithPosition()`;
+  const captureElementSnapshot = await page.evaluate(extraReturnLogic);
+  return captureElementSnapshot;
+}
+var sizeThreshold = 3;
+async function alignElements(screenshotBuffer, elements, page) {
+  const textsAligned = [];
+  const validElements = elements.filter((item) => {
+    return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
+  });
+  for (const item of validElements) {
+    const { rect, id, content, attributes, locator } = item;
+    textsAligned.push(
+      new WebElementInfo({
+        rect,
+        locator,
+        id,
+        content,
+        attributes,
+        page
+      })
+    );
+  }
+  return textsAligned;
+}
+function findNearestPackageJson(dir) {
+  const packageJsonPath = import_node_path.default.join(dir, "package.json");
+  if (import_node_fs.default.existsSync(packageJsonPath)) {
+    return dir;
+  }
+  const parentDir = import_node_path.default.dirname(dir);
+  if (parentDir === dir) {
+    return null;
+  }
+  return findNearestPackageJson(parentDir);
+}
+function reportFileName(tag = "web") {
+  const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
+  return `${tag}-${dateTimeInFileName}`;
+}
+function printReportMsg(filepath) {
+  console.log("Midscene - report file updated:", filepath);
+}
+
+// src/common/tasks.ts
+var PageTaskExecutor = class {
+  constructor(page, opts) {
+    this.page = page;
+    this.insight = new import_core.default(async () => {
+      return await parseContextFromWebPage(page);
+    });
+    this.taskCache = new TaskCache(opts);
+  }
+  async recordScreenshot(timing) {
+    const file = (0, import_utils2.getTmpFile)("jpeg");
+    await this.page.screenshot(__spreadProps(__spreadValues({}, import_utils2.commonScreenshotParam), {
+      path: file
+    }));
+    const item = {
+      type: "screenshot",
+      ts: Date.now(),
+      screenshot: (0, import_image2.base64Encoded)(file),
+      timing
+    };
+    return item;
+  }
+  wrapExecutorWithScreenshot(taskApply) {
+    const taskWithScreenshot = __spreadProps(__spreadValues({}, taskApply), {
+      executor: async (param, context, ...args) => {
+        const recorder = [];
+        const { task } = context;
+        task.recorder = recorder;
+        const shot = await this.recordScreenshot(`before ${task.type}`);
+        recorder.push(shot);
+        const result = await taskApply.executor(param, context, ...args);
+        if (taskApply.type === "Action") {
+          await (0, import_utils2.sleep)(1e3);
+          const shot2 = await this.recordScreenshot("after Action");
+          recorder.push(shot2);
+        }
+        return result;
+      }
+    });
+    return taskWithScreenshot;
+  }
+  async convertPlanToExecutable(plans) {
+    const tasks = plans.map((plan2) => {
+      if (plan2.type === "Locate") {
+        const taskFind = {
+          type: "Insight",
+          subType: "Locate",
+          param: plan2.param,
+          executor: async (param) => {
+            let insightDump;
+            const dumpCollector = (dump) => {
+              insightDump = dump;
+            };
+            this.insight.onceDumpUpdatedFn = dumpCollector;
+            const pageContext = await this.insight.contextRetrieverFn();
+            const locateCache = this.taskCache.readCache(
+              pageContext,
+              "locate",
+              param.prompt
+            );
+            let locateResult;
+            const callAI = this.insight.aiVendorFn;
+            const element = await this.insight.locate(param.prompt, {
+              callAI: async (...message) => {
+                if (locateCache) {
+                  locateResult = locateCache;
+                  return Promise.resolve(locateCache);
+                }
+                locateResult = await callAI(...message);
+                (0, import_node_assert2.default)(locateResult);
+                return locateResult;
+              }
+            });
+            (0, import_node_assert2.default)(element, `Element not found: ${param.prompt}`);
+            if (locateResult) {
+              this.taskCache.saveCache({
+                type: "locate",
+                pageContext: {
+                  url: pageContext.url,
+                  size: pageContext.size
+                },
+                prompt: param.prompt,
+                response: locateResult
+              });
+            }
+            return {
+              output: {
+                element
+              },
+              log: {
+                dump: insightDump
+              },
+              cache: {
+                hit: Boolean(locateResult)
+              }
+            };
+          }
+        };
+        return taskFind;
+      }
+      if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
+        const assertPlan = plan2;
+        const taskAssert = {
+          type: "Insight",
+          subType: "Assert",
+          param: assertPlan.param,
+          executor: async (param, taskContext) => {
+            const { task } = taskContext;
+            let insightDump;
+            const dumpCollector = (dump) => {
+              insightDump = dump;
+            };
+            this.insight.onceDumpUpdatedFn = dumpCollector;
+            const assertion = await this.insight.assert(
+              assertPlan.param.assertion
+            );
+            if (!assertion.pass) {
+              if (plan2.type === "Assert") {
+                task.output = assertion;
+                task.log = {
+                  dump: insightDump
+                };
+                throw new Error(
+                  assertion.thought || "Assertion failed without reason"
+                );
+              }
+              task.error = assertion.thought;
+            }
+            return {
+              output: assertion,
+              log: {
+                dump: insightDump
+              }
+            };
+          }
+        };
+        return taskAssert;
+      }
+      if (plan2.type === "Input") {
+        const taskActionInput = {
+          type: "Action",
+          subType: "Input",
+          param: plan2.param,
+          executor: async (taskParam, { element }) => {
+            if (element) {
+              await this.page.mouse.click(
+                element.center[0],
+                element.center[1]
+              );
+            }
+            (0, import_node_assert2.default)(taskParam.value, "No value to input");
+            await this.page.keyboard.type(taskParam.value);
+          }
+        };
+        return taskActionInput;
+      }
+      if (plan2.type === "KeyboardPress") {
+        const taskActionKeyboardPress = {
+          type: "Action",
+          subType: "KeyboardPress",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            (0, import_node_assert2.default)(taskParam.value, "No key to press");
+            await this.page.keyboard.press(taskParam.value);
+          }
+        };
+        return taskActionKeyboardPress;
+      }
+      if (plan2.type === "Tap") {
+        const taskActionTap = {
+          type: "Action",
+          subType: "Tap",
+          executor: async (param, { element }) => {
+            (0, import_node_assert2.default)(element, "Element not found, cannot tap");
+            await this.page.mouse.click(
+              element.center[0],
+              element.center[1]
+            );
+          }
+        };
+        return taskActionTap;
+      }
+      if (plan2.type === "Hover") {
+        const taskActionHover = {
+          type: "Action",
+          subType: "Hover",
+          executor: async (param, { element }) => {
+            (0, import_node_assert2.default)(element, "Element not found, cannot hover");
+            await this.page.mouse.move(
+              element.center[0],
+              element.center[1]
+            );
+          }
+        };
+        return taskActionHover;
+      }
+      if (plan2.type === "Scroll") {
+        const taskActionScroll = {
+          type: "Action",
+          subType: "Scroll",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            const scrollToEventName = taskParam.scrollType;
+            const innerHeight = await this.page.evaluate(
+              () => window.innerHeight
+            );
+            switch (scrollToEventName) {
+              case "ScrollUntilTop":
+                await this.page.mouse.wheel(0, -9999999);
+                break;
+              case "ScrollUntilBottom":
+                await this.page.mouse.wheel(0, 9999999);
+                break;
+              case "ScrollUp":
+                await this.page.mouse.wheel(0, -innerHeight);
+                break;
+              case "ScrollDown":
+                await this.page.mouse.wheel(0, innerHeight);
+                break;
+              default:
+                console.error(
+                  "Unknown scroll event type:",
+                  scrollToEventName
+                );
+            }
+          }
+        };
+        return taskActionScroll;
+      }
+      if (plan2.type === "Sleep") {
+        const taskActionSleep = {
+          type: "Action",
+          subType: "Sleep",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            (0, import_node_assert2.default)(taskParam.timeMs, "No time to sleep");
+            await (0, import_utils2.sleep)(taskParam.timeMs);
+          }
+        };
+        return taskActionSleep;
+      }
+      if (plan2.type === "Error") {
+        const taskActionError = {
+          type: "Action",
+          subType: "Error",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            (0, import_node_assert2.default)(
+              taskParam.thought,
+              "An error occurred, but no thought provided"
+            );
+            throw new Error(taskParam.thought);
+          }
+        };
+        return taskActionError;
+      }
+      throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
+    }).map((task) => {
+      return this.wrapExecutorWithScreenshot(task);
+    });
+    return tasks;
+  }
+  async action(userPrompt) {
+    const taskExecutor = new import_core.Executor(userPrompt);
+    let plans = [];
+    const planningTask = {
+      type: "Planning",
+      param: {
+        userPrompt
+      },
+      executor: async (param) => {
+        const pageContext = await this.insight.contextRetrieverFn();
+        let planResult;
+        const planCache = this.taskCache.readCache(
+          pageContext,
+          "plan",
+          userPrompt
+        );
+        if (planCache) {
+          planResult = planCache;
+        } else {
+          planResult = await (0, import_core.plan)(param.userPrompt, {
+            context: pageContext
+          });
+        }
+        (0, import_node_assert2.default)(planResult.plans.length > 0, "No plans found");
+        plans = planResult.plans;
+        this.taskCache.saveCache({
+          type: "plan",
+          pageContext: {
+            url: pageContext.url,
+            size: pageContext.size
+          },
+          prompt: userPrompt,
+          response: planResult
+        });
+        return {
+          output: planResult,
+          cache: {
+            hit: Boolean(planCache)
+          }
+        };
+      }
+    };
+    await taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
+    let output = await taskExecutor.flush();
+    if (taskExecutor.isInErrorState()) {
+      return {
+        output,
+        executor: taskExecutor
+      };
+    }
+    const executables = await this.convertPlanToExecutable(plans);
+    await taskExecutor.append(executables);
+    output = await taskExecutor.flush();
+    return {
+      output,
+      executor: taskExecutor
+    };
+  }
+  async query(demand) {
+    const description = typeof demand === "string" ? demand : JSON.stringify(demand);
+    const taskExecutor = new import_core.Executor(description);
+    const queryTask = {
+      type: "Insight",
+      subType: "Query",
+      param: {
+        dataDemand: demand
+      },
+      executor: async (param) => {
+        let insightDump;
+        const dumpCollector = (dump) => {
+          insightDump = dump;
+        };
+        this.insight.onceDumpUpdatedFn = dumpCollector;
+        const data = await this.insight.extract(param.dataDemand);
+        return {
+          output: data,
+          log: { dump: insightDump }
+        };
+      }
+    };
+    await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
+    const output = await taskExecutor.flush();
+    return {
+      output,
+      executor: taskExecutor
+    };
+  }
+  async assert(assertion) {
+    const description = `assert: ${assertion}`;
+    const taskExecutor = new import_core.Executor(description);
+    const assertionPlan = {
+      type: "Assert",
+      param: {
+        assertion
+      }
+    };
+    const assertTask = await this.convertPlanToExecutable([assertionPlan]);
+    await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
+    const output = await taskExecutor.flush();
+    return {
+      output,
+      executor: taskExecutor
+    };
+  }
+  async waitFor(assertion, opt) {
+    const description = `waitFor: ${assertion}`;
+    const taskExecutor = new import_core.Executor(description);
+    const { timeoutMs, checkIntervalMs } = opt;
+    (0, import_node_assert2.default)(assertion, "No assertion for waitFor");
+    (0, import_node_assert2.default)(timeoutMs, "No timeoutMs for waitFor");
+    (0, import_node_assert2.default)(checkIntervalMs, "No checkIntervalMs for waitFor");
+    const overallStartTime = Date.now();
+    let startTime = Date.now();
+    let errorThought = "";
+    while (Date.now() - overallStartTime < timeoutMs) {
+      startTime = Date.now();
+      const assertPlan = {
+        type: "AssertWithoutThrow",
+        param: {
+          assertion
+        }
+      };
+      const assertTask = await this.convertPlanToExecutable([assertPlan]);
+      await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
+      const output = await taskExecutor.flush();
+      if (output.pass) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      errorThought = output.thought;
+      const now = Date.now();
+      if (now - startTime < checkIntervalMs) {
+        const timeRemaining = checkIntervalMs - (now - startTime);
+        const sleepPlan = {
+          type: "Sleep",
+          param: {
+            timeMs: timeRemaining
+          }
+        };
+        const sleepTask = await this.convertPlanToExecutable([sleepPlan]);
+        await taskExecutor.append(
+          this.wrapExecutorWithScreenshot(sleepTask[0])
+        );
+        await taskExecutor.flush();
+      }
+    }
+    const errorPlan = {
+      type: "Error",
+      param: {
+        thought: `waitFor timeout: ${errorThought}`
+      }
+    };
+    const errorTask = await this.convertPlanToExecutable([errorPlan]);
+    await taskExecutor.append(errorTask[0]);
+    await taskExecutor.flush();
+    return {
+      output: void 0,
+      executor: taskExecutor
+    };
+  }
+};
+
+// src/common/agent.ts
+var PageAgent = class {
+  constructor(page, opts) {
+    this.page = page;
+    this.opts = Object.assign(
+      {
+        generateReport: true,
+        autoPrintReportMsg: true,
+        groupName: "Midscene Report",
+        groupDescription: ""
+      },
+      opts || {}
+    );
+    this.dump = {
+      groupName: this.opts.groupName,
+      groupDescription: this.opts.groupDescription,
+      executions: []
+    };
+    this.taskExecutor = new PageTaskExecutor(this.page, {
+      cache: (opts == null ? void 0 : opts.cache) || { aiTasks: [] }
+    });
+    this.reportFileName = reportFileName((opts == null ? void 0 : opts.testId) || "web");
+  }
+  appendExecutionDump(execution) {
+    const currentDump = this.dump;
+    currentDump.executions.push(execution);
+  }
+  dumpDataString() {
+    this.dump.groupName = this.opts.groupName;
+    this.dump.groupDescription = this.opts.groupDescription;
+    return (0, import_utils4.stringifyDumpData)(this.dump);
+  }
+  writeOutActionDumps() {
+    const { generateReport, autoPrintReportMsg } = this.opts;
+    this.reportFile = (0, import_utils4.writeLogFile)({
+      fileName: this.reportFileName,
+      fileExt: import_utils4.groupedActionDumpFileExt,
+      fileContent: this.dumpDataString(),
+      type: "dump",
+      generateReport
+    });
+    if (generateReport && autoPrintReportMsg) {
+      printReportMsg(this.reportFile);
+    }
+  }
+  async aiAction(taskPrompt) {
+    const { executor } = await this.taskExecutor.action(taskPrompt);
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState()) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask == null ? void 0 : errorTask.error}
+${errorTask == null ? void 0 : errorTask.errorStack}`);
+    }
+  }
+  async aiQuery(demand) {
+    const { output, executor } = await this.taskExecutor.query(demand);
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState()) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask == null ? void 0 : errorTask.error}
+${errorTask == null ? void 0 : errorTask.errorStack}`);
+    }
+    return output;
+  }
+  async aiAssert(assertion, msg) {
+    const { output, executor } = await this.taskExecutor.assert(assertion);
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (!(output == null ? void 0 : output.pass)) {
+      const errMsg = msg || `Assertion failed: ${assertion}`;
+      const reasonMsg = `Reason: ${output == null ? void 0 : output.thought} || (no_reason)`;
+      throw new Error(`${errMsg}
+${reasonMsg}`);
+    }
+  }
+  async aiWaitFor(assertion, opt) {
+    const { executor } = await this.taskExecutor.waitFor(assertion, {
+      timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
+      checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
+      assertion
+    });
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState()) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask == null ? void 0 : errorTask.error}
+${errorTask == null ? void 0 : errorTask.errorStack}`);
+    }
+  }
+  async ai(taskPrompt, type = "action") {
+    if (type === "action") {
+      return this.aiAction(taskPrompt);
+    }
+    if (type === "query") {
+      return this.aiQuery(taskPrompt);
+    }
+    if (type === "assert") {
+      return this.aiAssert(taskPrompt);
+    }
+    throw new Error(
+      `Unknown type: ${type}, only support 'action', 'query', 'assert'`
+    );
+  }
+};
+
+// src/playwright/ai-fixture.ts
+var import_test = require("@playwright/test");
+
+// src/playwright/cache.ts
+var import_node_fs2 = __toESM(require("fs"));
+var import_node_path2 = __toESM(require("path"));
+var import_utils7 = require("@midscene/core/utils");
+function writeTestCache(taskFile, taskTitle, taskCacheJson) {
+  const packageJson = getPkgInfo();
+  (0, import_utils7.writeLogFile)({
+    fileName: `${taskFile}(${taskTitle})`,
+    fileExt: "json",
+    fileContent: (0, import_utils7.stringifyDumpData)(
+      __spreadValues({
+        pkgName: packageJson.name,
+        pkgVersion: packageJson.version,
+        taskFile,
+        taskTitle
+      }, taskCacheJson),
+      2
+    ),
+    type: "cache"
+  });
+}
+function readTestCache(taskFile, taskTitle) {
+  const cacheFile = (0, import_node_path2.join)(
+    (0, import_utils7.getLogDirByType)("cache"),
+    `${taskFile}(${taskTitle}).json`
+  );
+  const pkgInfo = getPkgInfo();
+  if (process.env.MIDSCENE_CACHE === "true" && import_node_fs2.default.existsSync(cacheFile)) {
+    try {
+      const data = import_node_fs2.default.readFileSync(cacheFile, "utf8");
+      const jsonData = JSON.parse(data);
+      if (jsonData.pkgName !== pkgInfo.name || jsonData.pkgVersion !== pkgInfo.version) {
+        return void 0;
+      }
+      return jsonData;
+    } catch (err) {
+      return void 0;
+    }
+  }
+  return void 0;
+}
+function getPkgInfo() {
+  const packageJsonDir = findNearestPackageJson(__dirname);
+  if (!packageJsonDir) {
+    console.error("Cannot find package.json directory: ", __dirname);
+    return {
+      name: "@midscene/web",
+      version: "0.0.0"
+    };
+  }
+  const packageJsonPath = import_node_path2.default.join(packageJsonDir, "package.json");
+  const data = import_node_fs2.default.readFileSync(packageJsonPath, "utf8");
+  const packageJson = JSON.parse(data);
+  return {
+    name: packageJson.name,
+    version: packageJson.version
+  };
+}
+
+// src/playwright/ai-fixture.ts
+var groupAndCaseForTest = (testInfo) => {
+  let taskFile;
+  let taskTitle;
+  const titlePath = [...testInfo.titlePath];
+  if (titlePath.length > 1) {
+    taskTitle = titlePath.pop() || "unnamed";
+    taskFile = `${titlePath.join(" > ")}`;
+  } else if (titlePath.length === 1) {
+    taskTitle = titlePath[0];
+    taskFile = `${taskTitle}`;
+  } else {
+    taskTitle = "unnamed";
+    taskFile = "unnamed";
+  }
+  return { taskFile, taskTitle };
+};
+var midsceneAgentKeyId = "_midsceneAgentId";
+var midsceneDumpAnnotationId = "MIDSCENE_DUMP_ANNOTATION";
+var PlaywrightAiFixture = () => {
+  const pageAgentMap = {};
+  const agentForPage = (page, testInfo) => {
+    let idForPage = page[midsceneAgentKeyId];
+    if (!idForPage) {
+      idForPage = (0, import_node_crypto.randomUUID)();
+      page[midsceneAgentKeyId] = idForPage;
+      const { testId } = testInfo;
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      const testCase = readTestCache(taskFile, taskTitle) || {
+        aiTasks: []
+      };
+      pageAgentMap[idForPage] = new PageAgent(page, {
+        testId: `playwright-${testId}-${idForPage}`,
+        groupName: taskTitle,
+        groupDescription: taskFile,
+        cache: testCase,
+        generateReport: false
+        // we will generate it in the reporter
+      });
+    }
+    return pageAgentMap[idForPage];
+  };
+  const updateDumpAnnotation = (test2, dump) => {
+    const currentAnnotation = test2.annotations.find((item) => {
+      return item.type === midsceneDumpAnnotationId;
+    });
+    if (currentAnnotation) {
+      currentAnnotation.description = dump;
+    } else {
+      test2.annotations.push({
+        type: midsceneDumpAnnotationId,
+        description: dump
+      });
+    }
+  };
+  return {
+    ai: async ({ page }, use, testInfo) => {
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      const agent = agentForPage(page, testInfo);
+      await use(
+        async (taskPrompt, opts) => {
+          return new Promise((resolve, reject) => {
+            import_test.test.step(`ai - ${taskPrompt}`, async () => {
+              await page.waitForLoadState("networkidle");
+              const actionType = (opts == null ? void 0 : opts.type) || "action";
+              const result = await agent.ai(taskPrompt, actionType);
+              resolve(result);
+            });
+          });
+        }
+      );
+      const taskCacheJson = agent.taskExecutor.taskCache.generateTaskCache();
+      writeTestCache(taskFile, taskTitle, taskCacheJson);
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiAction: async ({ page }, use, testInfo) => {
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      const agent = agentForPage(page, testInfo);
+      await use(async (taskPrompt) => {
+        import_test.test.step(`aiAction - ${taskPrompt}`, async () => {
+          await page.waitForLoadState("networkidle");
+          await agent.aiAction(taskPrompt);
+        });
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiQuery: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(async (demand) => {
+        return new Promise((resolve, reject) => {
+          import_test.test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
+            await page.waitForLoadState("networkidle");
+            const result = await agent.aiQuery(demand);
+            resolve(result);
+          });
+        });
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiAssert: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(async (assertion, errorMsg) => {
+        return new Promise((resolve, reject) => {
+          import_test.test.step(`aiAssert - ${assertion}`, async () => {
+            await page.waitForLoadState("networkidle");
+            await agent.aiAssert(assertion, errorMsg);
+            resolve(null);
+          });
+        });
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiWaitFor: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(async (assertion, opt) => {
+        return new Promise((resolve, reject) => {
+          import_test.test.step(`aiWaitFor - ${assertion}`, async () => {
+            await agent.aiWaitFor(assertion, opt);
+            resolve(null);
+          });
+        });
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    }
+  };
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  PlaywrightAiFixture
+});