@midscene/core 0.0.1

package/dist/lib/index.js

@@ -0,0 +1,1124 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __defProps = Object.defineProperties;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+var __objRest = (source, exclude) => {
+  var target = {};
+  for (var prop in source)
+    if (__hasOwnProp.call(source, prop) && exclude.indexOf(prop) < 0)
+      target[prop] = source[prop];
+  if (source != null && __getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(source)) {
+      if (exclude.indexOf(prop) < 0 && __propIsEnum.call(source, prop))
+        target[prop] = source[prop];
+    }
+  return target;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var __publicField = (obj, key, value) => {
+  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+  return value;
+};
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  AIResponseFormat: () => AIResponseFormat,
+  BaseElement: () => BaseElement,
+  Executor: () => Executor,
+  UIContext: () => UIContext,
+  default: () => src_default,
+  getElement: () => getElement,
+  getSection: () => getSection,
+  plan: () => plan,
+  setDumpDir: () => setDumpDir
+});
+module.exports = __toCommonJS(src_exports);
+
+// src/insight/index.ts
+var import_assert5 = __toESM(require("assert"));
+
+// src/ai-model/prompt/util.ts
+var import_assert2 = __toESM(require("assert"));
+
+// src/image/info.ts
+var import_node_assert = __toESM(require("assert"));
+var import_buffer = require("buffer");
+var import_node_fs = require("fs");
+var import_sharp = __toESM(require("sharp"));
+async function imageInfo(image) {
+  const { width, height } = await (0, import_sharp.default)(image).metadata();
+  (0, import_node_assert.default)(width && height, `invalid image: ${image}`);
+  return { width, height };
+}
+async function imageInfoOfBase64(imageBase64) {
+  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
+  return imageInfo(import_buffer.Buffer.from(base64Data, "base64"));
+}
+
+// src/image/transform.ts
+var import_node_buffer = require("buffer");
+var import_sharp2 = __toESM(require("sharp"));
+
+// src/image/visualization.ts
+var import_buffer2 = require("buffer");
+var import_sharp3 = __toESM(require("sharp"));
+
+// src/utils.ts
+var import_os = require("os");
+var import_path = require("path");
+var import_fs = require("fs");
+var import_crypto = require("crypto");
+var import_assert = __toESM(require("assert"));
+var pkg;
+function getPkgInfo() {
+  if (pkg) {
+    return pkg;
+  }
+  let pkgJsonFile = "";
+  if ((0, import_fs.existsSync)((0, import_path.join)(__dirname, "../package.json"))) {
+    pkgJsonFile = (0, import_path.join)(__dirname, "../package.json");
+  } else if ((0, import_fs.existsSync)((0, import_path.join)(__dirname, "../../../package.json"))) {
+    pkgJsonFile = (0, import_path.join)(__dirname, "../../../package.json");
+  }
+  if (pkgJsonFile) {
+    const { name, version } = JSON.parse((0, import_fs.readFileSync)(pkgJsonFile, "utf-8"));
+    pkg = { name, version };
+    return pkg;
+  } else {
+    return {
+      name: "midscene-unknown-page-name",
+      version: "0.0.0"
+    };
+  }
+}
+var logDir = (0, import_path.join)(process.cwd(), "./midscene_run/");
+var logEnvReady = false;
+var insightDumpFileExt = "insight-dump.json";
+function getDumpDir() {
+  return logDir;
+}
+function setDumpDir(dir) {
+  logDir = dir;
+}
+function writeDumpFile(fileName, fileExt, fileContent) {
+  if (!logEnvReady) {
+    (0, import_assert.default)(logDir, "logDir should be set before writing dump file");
+    if (!(0, import_fs.existsSync)(logDir)) {
+      (0, import_fs.mkdirSync)(logDir, { recursive: true });
+    }
+    const gitIgnorePath = (0, import_path.join)(logDir, "../.gitignore");
+    let gitIgnoreContent = "";
+    if ((0, import_fs.existsSync)(gitIgnorePath)) {
+      gitIgnoreContent = (0, import_fs.readFileSync)(gitIgnorePath, "utf-8");
+    }
+    const logDirName = (0, import_path.basename)(logDir);
+    if (!gitIgnoreContent.includes(`${logDirName}/`)) {
+      (0, import_fs.writeFileSync)(
+        gitIgnorePath,
+        `${gitIgnoreContent}
+# MidScene.js dump files
+${logDirName}/
+`,
+        "utf-8"
+      );
+    }
+    logEnvReady = true;
+  }
+  const filePath = (0, import_path.join)(getDumpDir(), `${fileName}.${fileExt}`);
+  (0, import_fs.writeFileSync)(filePath, fileContent);
+  (0, import_fs.copyFileSync)(filePath, (0, import_path.join)(getDumpDir(), `latest.${fileExt}`));
+  return filePath;
+}
+
+// src/ai-model/prompt/util.ts
+var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
+var contextFormatIntro = `
+The user will give you a screenshot and the texts on it. There may be some none-English characters (like Chinese) on it, indicating it's an non-English app.`;
+var ONE_ELEMENT_LOCATOR_PREFIX = "LOCATE_ONE_ELEMENT";
+var ELEMENTS_LOCATOR_PREFIX = "LOCATE_ONE_OR_MORE_ELEMENTS";
+var SECTION_MATCHER_FLAG = "SECTION_MATCHER_FLAG/";
+var skillSegment = `skill name: segment_a_web_page 
+skill content:
+Based on the functions and content of various elements on the page, segment the screenshot into different sections like navigation bar, product list, news area, etc. 
+Some general rules for segmentation:
+* Each section should NOT overlap with each other.
+* Each text should only belong to one section.
+* [IMPORTANT] Whether the content visually appears to belong to different sections is a significant factor in segmenting the page.
+* Analyze the page in a top-to-bottom and left-to-right order.
+* The evidence indicates a separate section, for example 
+  - The background color of certain parts of the page changes.
+  - A section of a page includes a title.
+* Provide the following data for each of the UI section you found.
+  {
+    "name": "name of the section",
+    "description": "briefly summarize the key content or usage of this section.",
+    "sectionCharacteristics": "In view of the need to distinguish this section from the surrounding sections, explain the characteristics and how to define boundaries and what precautions to take.",
+    "textIds": ["5", "6", "7"], // ids of all text elements in this section
+  }
+`;
+var skillExtractData = `skill name: extract_data_from_UI
+related input: DATA_DEMAND
+skill content: 
+* User will give you some data requirements in DATA_DEMAND. Consider the UI context, follow the user's instructions, and provide comprehensive data accordingly.
+* There may be some special commands in DATA_DEMAND, please pay extra attention
+  - ${ONE_ELEMENT_LOCATOR_PREFIX} and ${ELEMENTS_LOCATOR_PREFIX}: if you see a description that mentions the keyword ${ONE_ELEMENT_LOCATOR_PREFIX} or ${ELEMENTS_LOCATOR_PREFIX}(e.g. follow ${ONE_ELEMENT_LOCATOR_PREFIX} : i want to find ...), it means user wants to locate a specific element meets the description. Return in this way: prefix + the id / comma-separated ids, for example: ${ONE_ELEMENT_LOCATOR_PREFIX}/1 , ${ELEMENTS_LOCATOR_PREFIX}/1,2,3 . If not found, keep the prefix and leave the suffix empty, like ${ONE_ELEMENT_LOCATOR_PREFIX}/ .`;
+function promptsOfSectionQuery(constraints) {
+  if (!constraints.length) {
+    return "";
+  }
+  const instruction = "Use your segment_a_web_page skill to find the following section(s)";
+  const singleSection = (c) => {
+    (0, import_assert2.default)(
+      c.name || c.description,
+      "either `name` or `description` is required to define a section constraint"
+    );
+    const number = "One section";
+    const name = c.name ? `named \`${c.name}\`` : "";
+    const description = c.description ? `, usage or criteria : ${c.description}` : "";
+    const basic = `* ${number} ${name}${description}`;
+    return basic;
+  };
+  return `${instruction}
+${constraints.map(singleSection).join("\n")}`;
+}
+function systemPromptToExtract(dataQuery, sections) {
+  const allSectionNames = (sections == null ? void 0 : sections.filter((c) => c.name).map((c) => c.name || "")) || [];
+  const sectionFindingPrompt = promptsOfSectionQuery(sections || []);
+  const sectionReturnFormat = allSectionNames.length ? "  sections: [], // detailed information of each section from segment_a_web_page skill" : "";
+  return `
+${characteristic}
+${contextFormatIntro}
+
+You have the following skills:
+${allSectionNames.length ? skillSegment : ""}
+${skillExtractData}
+
+Now, do the following jobs:
+${sectionFindingPrompt}
+Use your extract_data_from_UI skill to find the following data, placing it in the \`data\` field
+DATA_DEMAND start:
+${typeof dataQuery === "object" ? `return in key-value style object, keys are ${Object.keys(dataQuery).join(",")}` : ""};
+${typeof dataQuery === "string" ? dataQuery : JSON.stringify(dataQuery, null, 2)}
+DATA_DEMAND ends.
+
+Return in the following JSON format:
+{
+  language: "en", // "en" or "zh", the language of the page. Use the same language to describe section name, description, and similar fields.
+  ${sectionReturnFormat}
+  data: any, // the extracted data from extract_data_from_UI skill. Make sure both the value and scheme meet the DATA_DEMAND.
+  errors?: [], // string[], error message if any
+}
+`;
+}
+function describeSize(size) {
+  return `${size.width} x ${size.height}`;
+}
+function truncateText(text) {
+  const maxLength = 50;
+  if (text && text.length > maxLength) {
+    return `${text.slice(0, maxLength)}...`;
+  }
+  return text;
+}
+async function describeUserPage(context) {
+  const { screenshotBase64 } = context;
+  const { width, height } = await imageInfoOfBase64(screenshotBase64);
+  const elementsInfo = context.content;
+  const idElementMap = {};
+  elementsInfo.forEach((item) => {
+    idElementMap[item.id] = item;
+    return __spreadValues({}, item);
+  });
+  const elementInfosDescription = cropfieldInformation(elementsInfo);
+  return {
+    description: `
+    {
+      // The size of the page
+      "pageSize": ${describeSize({ width, height })},
+
+      
+      // json description of the element
+      "elementInfos": ${JSON.stringify(elementInfosDescription)}
+    }`,
+    elementById(id) {
+      (0, import_assert2.default)(typeof id !== "undefined", "id is required for query");
+      const item = idElementMap[`${id}`];
+      return item;
+    }
+  };
+}
+function cropfieldInformation(elementsInfo) {
+  const elementInfosDescription = elementsInfo.map((item) => {
+    const { id, attributes = {}, rect, content } = item;
+    const tailorContent = truncateText(content);
+    const tailorAttributes = Object.keys(attributes).reduce((res, currentKey) => {
+      const attributeVal = attributes[currentKey];
+      res[currentKey] = truncateText(attributeVal);
+      return res;
+    }, {});
+    return {
+      id,
+      attributes: tailorAttributes,
+      rect,
+      content: tailorContent
+    };
+  });
+  return JSON.stringify(elementInfosDescription);
+}
+function retrieveElement(prompt, opt) {
+  if (opt == null ? void 0 : opt.multi) {
+    return `follow ${ELEMENTS_LOCATOR_PREFIX}: ${prompt}`;
+  }
+  return `follow ${ONE_ELEMENT_LOCATOR_PREFIX}: ${prompt}`;
+}
+function ifElementTypeResponse(response) {
+  if (typeof response !== "string") {
+    return false;
+  }
+  return response.startsWith(ONE_ELEMENT_LOCATOR_PREFIX) || response.startsWith(ELEMENTS_LOCATOR_PREFIX);
+}
+function splitElementResponse(response) {
+  const oneElementSplitter = `${ONE_ELEMENT_LOCATOR_PREFIX}/`;
+  if (response.startsWith(oneElementSplitter)) {
+    const id = response.slice(oneElementSplitter.length);
+    if (id.indexOf(",") >= 0) {
+      console.warn(`unexpected comma in one element response: ${id}`);
+    }
+    return id ? id : null;
+  }
+  const elementsSplitter = `${ELEMENTS_LOCATOR_PREFIX}/`;
+  if (response.startsWith(elementsSplitter)) {
+    const idsString = response.slice(elementsSplitter.length);
+    if (!idsString) {
+      return [];
+    }
+    return idsString.split(",");
+  }
+  return null;
+}
+function retrieveSection(prompt) {
+  return `${SECTION_MATCHER_FLAG}${prompt}`;
+}
+function extractSectionQuery(input) {
+  if (typeof input === "string" && input.startsWith(SECTION_MATCHER_FLAG)) {
+    return input.slice(SECTION_MATCHER_FLAG.length);
+  }
+  return false;
+}
+
+// src/insight/utils.ts
+var import_fs2 = require("fs");
+var import_path2 = require("path");
+var import_crypto2 = require("crypto");
+var import_assert3 = __toESM(require("assert"));
+var logFileName = "";
+var logContent = [];
+var logIdIndexMap = {};
+var { pid } = process;
+var logFileExt = insightDumpFileExt;
+function writeInsightDump(data, logId, dumpSubscriber) {
+  const logDir2 = getDumpDir();
+  (0, import_assert3.default)(logDir2, "logDir should be set before writing dump file");
+  const id = logId || (0, import_crypto2.randomUUID)();
+  const baseData = {
+    sdkVersion: getPkgInfo().version,
+    logTime: Date.now()
+  };
+  const finalData = __spreadValues(__spreadValues({
+    logId: id
+  }, baseData), data);
+  dumpSubscriber == null ? void 0 : dumpSubscriber(finalData);
+  if (!logFileName) {
+    logFileName = `pid_${pid}_${baseData.logTime}`;
+    while ((0, import_fs2.existsSync)((0, import_path2.join)(logDir2, `${logFileName}.${logFileExt}`))) {
+      logFileName = `${pid}_${baseData.logTime}-${Math.random()}`;
+    }
+  }
+  const dataString = JSON.stringify(finalData, null, 2);
+  if (typeof logIdIndexMap[id] === "number") {
+    logContent[logIdIndexMap[id]] = dataString;
+  } else {
+    const length = logContent.push(dataString);
+    logIdIndexMap[id] = length - 1;
+  }
+  writeDumpFile(logFileName, logFileExt, `[
+${logContent.join(",\n")}
+]`);
+  return id;
+}
+function idsIntoElements(ids, elementById) {
+  return ids.reduce((acc, id) => {
+    const element = elementById(id);
+    if (element) {
+      acc.push(element);
+    } else {
+      console.warn(`element not found by id: ${id}`);
+    }
+    return acc;
+  }, []);
+}
+function shallowExpandIds(data, ifMeet, elementsById) {
+  const keys = Object.keys(data);
+  keys.forEach((key) => {
+    const value = data[key];
+    if (typeof value === "string" && ifMeet(value)) {
+      data[key] = elementsById(value);
+    } else if (Array.isArray(value)) {
+      const newValue = value.map((id) => ifMeet(id) ? elementsById(id) : id);
+      data[key] = newValue;
+    }
+  });
+  return data;
+}
+function expandLiteSection(liteSection, elementById) {
+  const _a = liteSection, { textIds } = _a, remainingFields = __objRest(_a, ["textIds"]);
+  const texts = idsIntoElements(textIds, elementById);
+  let leftMost = -1;
+  let topMost = -1;
+  let rightMost = -1;
+  let bottomMost = -1;
+  texts.forEach((text) => {
+    leftMost = leftMost === -1 ? text.rect.left : Math.min(leftMost, text.rect.left);
+    topMost = topMost === -1 ? text.rect.top : Math.min(topMost, text.rect.top);
+    rightMost = Math.max(rightMost, text.rect.left + text.rect.width);
+    bottomMost = Math.max(bottomMost, text.rect.top + text.rect.height);
+  });
+  const sectionRect = {
+    left: leftMost,
+    top: topMost,
+    width: rightMost - leftMost,
+    height: bottomMost - topMost
+  };
+  const section = __spreadProps(__spreadValues({}, remainingFields), {
+    content: texts,
+    rect: sectionRect
+  });
+  return section;
+}
+
+// src/ai-model/openai.ts
+var import_assert4 = __toESM(require("assert"));
+var import_openai = __toESM(require("openai"));
+var import_wrappers = __toESM(require("langsmith/wrappers"));
+
+// src/types.ts
+var BaseElement = class {
+};
+var AIResponseFormat = /* @__PURE__ */ ((AIResponseFormat2) => {
+  AIResponseFormat2["JSON"] = "json_object";
+  AIResponseFormat2["TEXT"] = "text";
+  return AIResponseFormat2;
+})(AIResponseFormat || {});
+var UIContext = class {
+};
+
+// src/ai-model/openai.ts
+var envConfigKey = "MIDSCENE_OPENAI_INIT_CONFIG_JSON";
+var envModelKey = "MIDSCENE_MODEL_NAME";
+var envSmithDebug = "MIDSCENE_LANGSMITH_DEBUG";
+var extraConfig = {};
+if (typeof process.env[envConfigKey] === "string") {
+  console.log("will use env config for openai");
+  extraConfig = JSON.parse(process.env[envConfigKey]);
+}
+var model = "gpt-4o";
+if (typeof process.env[envModelKey] === "string") {
+  console.log(`will use model: ${process.env[envModelKey]}`);
+  model = process.env[envModelKey];
+}
+async function createOpenAI() {
+  const openai = new import_openai.default(extraConfig);
+  if (process.env[envSmithDebug]) {
+    console.log("DEBUGGING MODE: using langsmith wrapper");
+    const openai2 = import_wrappers.default.wrapOpenAI(new import_openai.default());
+    return openai2;
+  }
+  return openai;
+}
+async function call(messages, responseFormat) {
+  const openai = await createOpenAI();
+  const completion = await openai.chat.completions.create({
+    model,
+    messages,
+    response_format: { type: responseFormat }
+  });
+  const { content } = completion.choices[0].message;
+  (0, import_assert4.default)(content, "empty content");
+  return content;
+}
+async function callToGetJSONObject(messages) {
+  const response = await call(messages, "json_object" /* JSON */);
+  (0, import_assert4.default)(response, "empty response");
+  return JSON.parse(response);
+}
+
+// src/ai-model/prompt/element_inspector.ts
+function systemPromptToFindElement(description, multi) {
+  return `
+## Role:
+You are an expert in software page image (2D) and page element text analysis.
+
+## Objective:
+- Identify elements in screenshots and text that match the user's description.
+- Return JSON data containing the selection reason and element ID.
+
+## Skills:
+- Image analysis and recognition
+- Multilingual text understanding
+- Software UI design and testing
+
+## Workflow:
+1. Receive the user's element description, screenshot, and element description information. Note that the text may contain non-English characters (e.g., Chinese), indicating that the application may be non-English.
+2. Based on the description (${description}), locate the target element ID in the list of element descriptions and the screenshot.
+3. Return the number of elements: ${multi ? "multiple elements matching the description (two or more)" : "The element closest to the description (only one)"}.
+4. Return JSON data containing the selection reason and element ID.
+
+## Constraints:
+- Strictly adhere to the specified location when describing the required element; do not select elements from other locations.
+- Elements in the image with NodeType other than "TEXT Node" have been highlighted to identify the element among multiple non-text elements.
+- Accurately identify element information based on the user's description and return the corresponding element ID from the element description information, not extracted from the image.
+- If no elements are found, the "elements" array should be empty.
+- The returned data must conform to the specified JSON format.
+
+## Output Format:
+\`\`\`json
+{
+  "elements": [
+    // If no matching elements are found, return an empty array []
+    {
+      "reason": "xxx", // The thought process for finding the element, replace xxx with your thought process
+      "text": "xxx", // Replace xxx with the text of elementInfo, if none, leave empty
+      "id": "xxx" // Replace xxx with the ID of elementInfo
+    }
+    // More elements...
+  ],
+  "errors": [] // Array of strings containing any error messages
+}
+\`\`\`
+
+## Example:
+Example 1:
+Input Example:
+\`\`\`json
+// Description: "Shopping cart icon in the upper right corner"
+{
+  "screenshot": "path/screenshot.png",
+  "text": '{
+      "pageSize": {
+        "width": 400, // Width of the page
+        "height": 905 // Height of the page
+      },
+      "elementInfos": [
+        {
+          "id": "3", // ID of the element
+          "attributes": { // Attributes of the element
+            "nodeType": "IMG Node", // Type of element, types include: TEXT Node, IMG Node, BUTTON Node, INPUT Node
+            "src": "https://ap-southeast-3.m",
+            "class": ".img"
+          },
+          "content": "", // Text content of the element
+          "rect": {
+            "left": 280, // Distance from the left side of the page
+            "top": 8, // Distance from the top of the page
+            "width": 44, // Width of the element
+            "height": 44 // Height of the element
+          }
+        },
+        {
+          "id": "4", // ID of the element
+          "attributes": { // Attributes of the element
+            "nodeType": "IMG Node", // Type of element, types include: TEXT Node, IMG Node, BUTTON Node, INPUT Node
+            "src": "data:image/png;base64,iVBORw0KGgoAAAANSU...",
+            "class": ".icon"
+          },
+          "content": "", // Text content of the element
+          "rect": {
+            "left": 350, // Distance from the left side of the page
+            "top": 16, // Distance from the top of the page
+            "width": 25, // Width of the element
+            "height": 25 // Height of the element
+          }
+        },
+        ...
+        {
+          "id": "27",
+          "attributes": {
+            "nodeType": "TEXT Node",
+            "class": ".product-name"
+          },
+          "center": [
+            288,
+            834
+          ],
+          "content": "Mango Drink",
+          "rect": {
+            "left": 188,
+            "top": 827,
+            "width": 199,
+            "height": 13
+          }
+        },
+        ...
+      ]
+    }
+  '
+}
+\`\`\`
+Output Example:
+\`\`\`json
+{
+  "elements": [
+    {
+      // Describe the reason for finding this element, replace with actual value in practice
+      "reason": "Reason for finding element 4: It is located in the upper right corner, is an image type, and according to the screenshot, it is a shopping cart icon button",
+      "text": "",
+      // ID of this element, replace with actual value in practice
+      "id": "4"
+    }
+  ],
+  "errors": []
+}
+\`\`\`
+  
+  `;
+}
+
+// src/ai-model/inspect.ts
+async function AiInspectElement(options) {
+  const { context, multi, findElementDescription, callAI = callToGetJSONObject } = options;
+  const { screenshotBase64 } = context;
+  const { description, elementById } = await describeUserPage(context);
+  const systemPrompt = systemPromptToFindElement(findElementDescription, multi);
+  const msgs = [
+    { role: "system", content: systemPrompt },
+    {
+      role: "user",
+      content: [
+        {
+          type: "image_url",
+          image_url: {
+            url: screenshotBase64,
+            detail: "high"
+          }
+        },
+        {
+          type: "text",
+          text: description
+        }
+      ]
+    }
+  ];
+  const parseResult = await callAI(msgs);
+  return {
+    parseResult,
+    elementById,
+    systemPrompt
+  };
+}
+async function AiExtractElementInfo(options) {
+  const { dataQuery, sectionConstraints, context, callAI = callToGetJSONObject } = options;
+  const systemPrompt = systemPromptToExtract(dataQuery, sectionConstraints);
+  const { screenshotBase64 } = context;
+  const { description, elementById } = await describeUserPage(context);
+  const msgs = [
+    { role: "system", content: systemPrompt },
+    {
+      role: "user",
+      content: [
+        {
+          type: "image_url",
+          image_url: {
+            url: screenshotBase64
+          }
+        },
+        {
+          type: "text",
+          text: description
+        }
+      ]
+    }
+  ];
+  const parseResult = await callAI(msgs);
+  return {
+    parseResult,
+    elementById,
+    systemPrompt
+  };
+}
+
+// src/insight/index.ts
+var sortByOrder = (a, b) => {
+  if (a.rect.top - b.rect.top !== 0) {
+    return a.rect.top - b.rect.top;
+  } else {
+    return a.rect.left - b.rect.left;
+  }
+};
+var Insight = class {
+  constructor(context, opt) {
+    __publicField(this, "contextRetrieverFn");
+    __publicField(this, "aiVendorFn", callToGetJSONObject);
+    __publicField(this, "onceDumpUpdatedFn");
+    __publicField(this, "taskInfo");
+    (0, import_assert5.default)(context, "context is required for Insight");
+    if (typeof context === "function") {
+      this.contextRetrieverFn = context;
+    } else {
+      this.contextRetrieverFn = () => Promise.resolve(context);
+    }
+    if (typeof (opt == null ? void 0 : opt.aiVendorFn) !== "undefined") {
+      this.aiVendorFn = opt.aiVendorFn;
+    }
+    if (typeof (opt == null ? void 0 : opt.taskInfo) !== "undefined") {
+      this.taskInfo = opt.taskInfo;
+    }
+  }
+  async locate(queryPrompt, opt) {
+    var _a;
+    (0, import_assert5.default)(queryPrompt, "query is required for located");
+    const dumpSubscriber = this.onceDumpUpdatedFn;
+    this.onceDumpUpdatedFn = void 0;
+    const context = await this.contextRetrieverFn();
+    const startTime = Date.now();
+    const { parseResult, systemPrompt, elementById } = await AiInspectElement({
+      callAI: this.aiVendorFn,
+      context,
+      multi: Boolean(opt == null ? void 0 : opt.multi),
+      findElementDescription: queryPrompt
+    });
+    const timeCost = Date.now() - startTime;
+    const taskInfo = __spreadProps(__spreadValues({}, this.taskInfo ? this.taskInfo : {}), {
+      durationMs: timeCost,
+      rawResponse: JSON.stringify(parseResult),
+      systemPrompt
+    });
+    let errorLog;
+    if ((_a = parseResult.errors) == null ? void 0 : _a.length) {
+      errorLog = `locate - AI response error: 
+${parseResult.errors.join("\n")}`;
+    }
+    const dumpData = {
+      type: "locate",
+      context,
+      userQuery: {
+        element: queryPrompt
+      },
+      matchedSection: [],
+      matchedElement: [],
+      data: null,
+      taskInfo,
+      error: errorLog
+    };
+    const logId = writeInsightDump(dumpData, void 0, dumpSubscriber);
+    if (errorLog) {
+      console.error(errorLog);
+      throw new Error(errorLog);
+    }
+    const elements = [];
+    parseResult.elements.forEach((item) => {
+      const element = elementById(item.id);
+      if (!element) {
+        console.warn(`locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`);
+        return;
+      }
+      elements.push(element);
+    });
+    writeInsightDump(
+      __spreadProps(__spreadValues({}, dumpData), {
+        matchedElement: elements
+      }),
+      logId,
+      dumpSubscriber
+    );
+    if (opt == null ? void 0 : opt.multi) {
+      return elements;
+    } else if (elements.length >= 2) {
+      console.warn(`locate: multiple elements found, return the first one. (query: ${queryPrompt})`);
+      return elements[0];
+    } else if (elements.length === 1) {
+      return elements[0];
+    } else {
+      return null;
+    }
+  }
+  async extract(dataDemand) {
+    var _a;
+    let dataQuery = {};
+    const sectionQueryMap = {};
+    (0, import_assert5.default)(
+      typeof dataDemand === "object" || typeof dataDemand === "string",
+      `dataDemand should be object or string, but get ${typeof dataDemand}`
+    );
+    const dumpSubscriber = this.onceDumpUpdatedFn;
+    this.onceDumpUpdatedFn = void 0;
+    if (typeof dataDemand === "string") {
+      dataQuery = dataDemand;
+    } else {
+      for (const key in dataDemand) {
+        const query = dataDemand[key];
+        const sectionQuery = extractSectionQuery(query);
+        if (sectionQuery) {
+          sectionQueryMap[key] = sectionQuery;
+        } else {
+          dataQuery[key] = query;
+        }
+      }
+      dataQuery = dataDemand;
+    }
+    const sectionConstraints = Object.keys(sectionQueryMap).map((name) => {
+      const sectionQueryPrompt = sectionQueryMap[name];
+      return {
+        name,
+        description: sectionQueryPrompt || ""
+      };
+    });
+    const context = await this.contextRetrieverFn();
+    const startTime = Date.now();
+    const { parseResult, systemPrompt, elementById } = await AiExtractElementInfo({
+      context,
+      dataQuery,
+      sectionConstraints,
+      callAI: this.aiVendorFn
+    });
+    const timeCost = Date.now() - startTime;
+    const taskInfo = __spreadProps(__spreadValues({}, this.taskInfo ? this.taskInfo : {}), {
+      durationMs: timeCost,
+      rawResponse: JSON.stringify(parseResult),
+      systemPrompt
+    });
+    let errorLog;
+    if ((_a = parseResult.errors) == null ? void 0 : _a.length) {
+      errorLog = `segment - AI response error: 
+${parseResult.errors.join("\n")}`;
+    }
+    const dumpData = {
+      type: "extract",
+      context,
+      userQuery: {
+        dataDemand
+      },
+      matchedSection: [],
+      matchedElement: [],
+      data: null,
+      taskInfo,
+      error: errorLog
+    };
+    const logId = writeInsightDump(dumpData, void 0, dumpSubscriber);
+    if (errorLog) {
+      console.error(errorLog);
+      throw new Error(errorLog);
+    }
+    const sectionsArr = (parseResult.sections || []).map((liteSection) => {
+      const section = expandLiteSection(liteSection, (id) => elementById(id));
+      return section;
+    }).sort(sortByOrder);
+    const sectionMap = sectionsArr.reduce((acc, section) => {
+      const { name } = section;
+      if (acc[name]) {
+        let i = 1;
+        while (acc[`${name}_${i}`]) {
+          i++;
+        }
+        console.warn(`section name conflict: ${name}, rename to ${name}_${i}`);
+        acc[`${name}_${i}`] = section;
+      } else {
+        acc[name] = section;
+      }
+      return acc;
+    }, {});
+    const { data } = parseResult;
+    let mergedData = data;
+    if (data && typeof data === "object" && !Array.isArray(data)) {
+      shallowExpandIds(data, ifElementTypeResponse, (id) => {
+        const idList = splitElementResponse(id);
+        if (typeof idList === "string") {
+          return elementById(idList);
+        } else if (Array.isArray(idList)) {
+          return idsIntoElements(idList, elementById);
+        }
+        return idList;
+      });
+      mergedData = __spreadValues(__spreadValues({}, data), sectionMap);
+    }
+    writeInsightDump(
+      __spreadProps(__spreadValues({}, dumpData), {
+        matchedSection: Object.values(sectionMap),
+        data: mergedData
+      }),
+      logId,
+      dumpSubscriber
+    );
+    return mergedData;
+  }
+};
+
+// src/action/executor.ts
+var import_assert6 = __toESM(require("assert"));
+var Executor = class {
+  constructor(name, description, tasks) {
+    __publicField(this, "name");
+    __publicField(this, "description");
+    __publicField(this, "tasks");
+    // status of executor
+    __publicField(this, "status");
+    __publicField(this, "errorMsg");
+    __publicField(this, "dumpFileName");
+    this.status = tasks && tasks.length > 0 ? "pending" : "init";
+    this.name = name;
+    this.description = description;
+    this.tasks = (tasks || []).map((item) => this.markTaskAsPending(item));
+  }
+  markTaskAsPending(task) {
+    return __spreadValues({
+      status: "pending"
+    }, task);
+  }
+  async append(task) {
+    (0, import_assert6.default)(this.status !== "error", "executor is in error state, cannot append task");
+    if (Array.isArray(task)) {
+      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));
+    } else {
+      this.tasks.push(this.markTaskAsPending(task));
+    }
+    if (this.status !== "running") {
+      this.status = "pending";
+    }
+  }
+  async flush() {
+    if (this.status === "init" && this.tasks.length > 0) {
+      console.warn("illegal state for executor, status is init but tasks are not empty");
+    }
+    (0, import_assert6.default)(this.status !== "running", "executor is already running");
+    (0, import_assert6.default)(this.status !== "completed", "executor is already completed");
+    (0, import_assert6.default)(this.status !== "error", "executor is in error state");
+    const nextPendingIndex = this.tasks.findIndex((task) => task.status === "pending");
+    if (nextPendingIndex < 0) {
+      return;
+    }
+    this.status = "running";
+    let taskIndex = nextPendingIndex;
+    let successfullyCompleted = true;
+    let errorMsg = "";
+    let previousFindOutput;
+    while (taskIndex < this.tasks.length) {
+      const task = this.tasks[taskIndex];
+      (0, import_assert6.default)(task.status === "pending", `task status should be pending, but got: ${task.status}`);
+      task.timing = {
+        start: Date.now()
+      };
+      try {
+        task.status = "running";
+        (0, import_assert6.default)(
+          ["Insight", "Action", "Planning"].indexOf(task.type) >= 0,
+          `unsupported task type: ${task.type}`
+        );
+        const { executor, param } = task;
+        (0, import_assert6.default)(executor, `executor is required for task type: ${task.type}`);
+        let returnValue;
+        const executorContext = {
+          task,
+          element: previousFindOutput == null ? void 0 : previousFindOutput.element
+        };
+        if (task.type === "Insight") {
+          (0, import_assert6.default)(
+            task.subType === "Locate" || task.subType === "Query",
+            `unsupported insight subType: ${task.subType}`
+          );
+          returnValue = await task.executor(param, executorContext);
+          if (task.subType === "Locate") {
+            previousFindOutput = returnValue == null ? void 0 : returnValue.output;
+          }
+        } else if (task.type === "Action" || task.type === "Planning") {
+          returnValue = await task.executor(param, executorContext);
+        } else {
+          console.warn(`unsupported task type: ${task.type}, will try to execute it directly`);
+          returnValue = await task.executor(param, executorContext);
+        }
+        Object.assign(task, returnValue);
+        task.status = "success";
+        task.timing.end = Date.now();
+        task.timing.cost = task.timing.end - task.timing.start;
+        taskIndex++;
+      } catch (e) {
+        successfullyCompleted = false;
+        task.status = "fail";
+        errorMsg = `${e == null ? void 0 : e.message}
+${e == null ? void 0 : e.stack}`;
+        task.error = errorMsg;
+        task.timing.end = Date.now();
+        task.timing.cost = task.timing.end - task.timing.start;
+        this.errorMsg = errorMsg;
+        break;
+      }
+    }
+    for (let i = taskIndex + 1; i < this.tasks.length; i++) {
+      this.tasks[i].status = "cancelled";
+    }
+    if (successfullyCompleted) {
+      this.status = "completed";
+    } else {
+      this.status = "error";
+      throw new Error(`executor failed: ${errorMsg}`);
+    }
+  }
+  dump() {
+    const dumpData = {
+      sdkVersion: getPkgInfo().version,
+      logTime: Date.now(),
+      name: this.name,
+      description: this.description,
+      tasks: this.tasks
+    };
+    return dumpData;
+  }
+};
+
+// src/query/index.ts
+var getElement = retrieveElement;
+var getSection = retrieveSection;
+
+// src/automation/planning.ts
+var characteristic2 = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
+function systemPromptToTaskPlanning(query) {
+  return `
+  ${characteristic2}
+  
+  Based on the page context information (screenshot and description) you get, decompose the task user asked into a series of actions.
+  Actions are executed in the order listed in the list. After executing the actions, the task should be completed.
+
+  Each action has a type and corresponding param. To be detailed:
+  * type: 'Locate', it means to locate one element
+    * param: { prompt: string }, the prompt describes 'which element to focus on page'. Our AI engine will use this prompt to locate the element, so it should clearly describe the obvious features of the element, such as its content, color, size, shape, and position. For example, 'The biggest Download Button on the left side of the page.'
+  * type: 'Tap', tap the previous element found 
+    * param: null
+  * type: 'Hover', hover the previous element found
+    * param: null
+  * type: 'Input', 'KeyboardPress', input something or press a key
+    * param: { value: string }, the value to input or the key to press. Use （Enter, Shift, Control, Alt, Meta, ShiftLeft, ControlOrMeta, ControlOrMeta） to represent the key.
+  * type: 'Scroll'
+    * param: { scrollType: 'ScrollUntilBottom', 'ScrollUntilTop', 'ScrollDown', 'ScrollUp' }
+  * type: 'Error'
+    * param: { message: string }, the error message
+  
+  Here is an example of how to decompose a task.
+  When a user says 'Input "Weather in Shanghai" into the search bar, hit enter', by viewing the page screenshot and description, you my decompose this task into something like this:
+  * Find: 'The search bar'
+  * Input: 'Weather in Shanghai'
+  * KeyboardPress: 'Enter'
+  
+  Remember: 
+  1. The actions you composed MUST be based on the page context information you get. Instead of making up actions that are not related to the page context.
+  2. In most cases, you should Locate one element first, then do other actions on it. For example, alway Find one element, then hover on it. But if you think it's necessary to do other actions first (like global scroll, global key press), you can do that.
+
+  If any error occurs during the task planning (like the page content and task are irrelevant, or the element mentioned does not exist at all), please return the error message with explanation in the errors field. The thoughts、prompts、error messages should all in the same language as the user query.
+  
+  Return in the following JSON format:
+  {
+    queryLanguage: '', // language of the description of the task
+    actions: [ // always return in Array
+      {
+        "thought": "Reasons for generating this task, and why this task is feasible on this page",
+        "type": "Tap", // Type of action, like 'Tap' 'Hover' ...
+        "param": any, // Parameter towards the task type
+      },
+      // ... more actions
+    ],
+    error?: string, // Overall error messages. If there is any error occurs during the task planning (i.e. error in previous 'actions' array), conclude the errors again, put error messages here
+  }
+
+  Here is the description of the task. Just go ahead:
+  =====================================
+  ${query}
+  =====================================
+  `;
+}
+async function plan(context, userPrompt) {
+  const { screenshotBase64 } = context;
+  const { description } = await describeUserPage(context);
+  const systemPrompt = systemPromptToTaskPlanning(userPrompt);
+  const msgs = [
+    { role: "system", content: systemPrompt },
+    {
+      role: "user",
+      content: [
+        {
+          type: "image_url",
+          image_url: {
+            url: screenshotBase64,
+            detail: "high"
+          }
+        },
+        {
+          type: "text",
+          text: description
+        }
+      ]
+    }
+  ];
+  const planFromAI = await callToGetJSONObject(msgs);
+  if (planFromAI.error) {
+    throw new Error(planFromAI.error);
+  }
+  const { actions } = planFromAI;
+  actions.forEach((task) => {
+    if (task.type === "Error") {
+      throw new Error(task.thought);
+    }
+  });
+  return { plans: actions };
+}
+
+// src/index.ts
+var src_default = Insight;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AIResponseFormat,
+  BaseElement,
+  Executor,
+  UIContext,
+  getElement,
+  getSection,
+  plan,
+  setDumpDir
+});