@midscene/core 0.1.4 → 0.2.0

package/dist/es/index.js

@@ -1008,41 +1008,21 @@ var require_dist = __commonJS({
   }
 });
 
-// src/insight/index.ts
-import assert6 from "assert";
-
-// src/ai-model/prompt/util.ts
-import assert3 from "assert";
-
-// src/image/info.ts
-import assert from "assert";
-import { Buffer as Buffer2 } from "buffer";
-import { readFileSync } from "fs";
-import Sharp from "sharp";
-async function imageInfo(image) {
-  const { width, height } = await Sharp(image).metadata();
-  assert(width && height, `invalid image: ${image}`);
-  return { width, height };
-}
-async function imageInfoOfBase64(imageBase64) {
-  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
-  return imageInfo(Buffer2.from(base64Data, "base64"));
-}
-
-// src/image/transform.ts
-import { Buffer as Buffer3 } from "buffer";
-import Sharp2 from "sharp";
-
-// src/image/visualization.ts
-import { Buffer as Buffer4 } from "buffer";
-import Sharp3 from "sharp";
+// src/action/executor.ts
+import assert2 from "assert";
 
 // src/utils.ts
+import assert from "assert";
+import { randomUUID } from "crypto";
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync
+} from "fs";
 import { tmpdir } from "os";
 import { basename, join } from "path";
-import { copyFileSync, existsSync, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
-import { randomUUID } from "crypto";
-import assert2 from "assert";
 var pkg;
 function getPkgInfo() {
   if (pkg) {
@@ -1055,15 +1035,14 @@ function getPkgInfo() {
     pkgJsonFile = join(__dirname, "../../../package.json");
   }
   if (pkgJsonFile) {
-    const { name, version } = JSON.parse(readFileSync2(pkgJsonFile, "utf-8"));
+    const { name, version } = JSON.parse(readFileSync(pkgJsonFile, "utf-8"));
     pkg = { name, version };
     return pkg;
-  } else {
-    return {
-      name: "midscene-unknown-page-name",
-      version: "0.0.0"
-    };
   }
+  return {
+    name: "midscene-unknown-page-name",
+    version: "0.0.0"
+  };
 }
 var logDir = join(process.cwd(), "./midscene_run/");
 var logEnvReady = false;
@@ -1084,18 +1063,18 @@ function writeDumpFile(opts) {
     mkdirSync(targetDir, { recursive: true });
   }
   if (!logEnvReady) {
-    assert2(targetDir, "logDir should be set before writing dump file");
+    assert(targetDir, "logDir should be set before writing dump file");
     const gitIgnorePath = join(targetDir, "../../.gitignore");
     let gitIgnoreContent = "";
     if (existsSync(gitIgnorePath)) {
-      gitIgnoreContent = readFileSync2(gitIgnorePath, "utf-8");
+      gitIgnoreContent = readFileSync(gitIgnorePath, "utf-8");
     }
     const logDirName = basename(logDir);
     if (!gitIgnoreContent.includes(`${logDirName}/`)) {
       writeFileSync(
         gitIgnorePath,
         `${gitIgnoreContent}
-# MidScene.js dump files
+# Midscene.js dump files
 ${logDirName}/report
 ${logDirName}/dump-logger
 `,
@@ -1112,286 +1091,157 @@ ${logDirName}/dump-logger
   return filePath;
 }
 
-// src/ai-model/prompt/util.ts
-var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
-var contextFormatIntro = `
-The user will give you a screenshot and the texts on it. There may be some none-English characters (like Chinese) on it, indicating it's an non-English app.`;
-var ONE_ELEMENT_LOCATOR_PREFIX = "LOCATE_ONE_ELEMENT";
-var ELEMENTS_LOCATOR_PREFIX = "LOCATE_ONE_OR_MORE_ELEMENTS";
-var SECTION_MATCHER_FLAG = "SECTION_MATCHER_FLAG/";
-var skillSegment = `skill name: segment_a_web_page 
-skill content:
-Based on the functions and content of various elements on the page, segment the screenshot into different sections like navigation bar, product list, news area, etc. 
-Some general rules for segmentation:
-* Each section should NOT overlap with each other.
-* Each text should only belong to one section.
-* [IMPORTANT] Whether the content visually appears to belong to different sections is a significant factor in segmenting the page.
-* Analyze the page in a top-to-bottom and left-to-right order.
-* The evidence indicates a separate section, for example 
-  - The background color of certain parts of the page changes.
-  - A section of a page includes a title.
-* Provide the following data for each of the UI section you found.
-  {
-    "name": "name of the section",
-    "description": "briefly summarize the key content or usage of this section.",
-    "sectionCharacteristics": "In view of the need to distinguish this section from the surrounding sections, explain the characteristics and how to define boundaries and what precautions to take.",
-    "textIds": ["5", "6", "7"], // ids of all text elements in this section
+// src/action/executor.ts
+var Executor = class {
+  constructor(name, description, tasks) {
+    __publicField(this, "name");
+    __publicField(this, "description");
+    __publicField(this, "tasks");
+    // status of executor
+    __publicField(this, "status");
+    __publicField(this, "errorMsg");
+    __publicField(this, "dumpFileName");
+    this.status = tasks && tasks.length > 0 ? "pending" : "init";
+    this.name = name;
+    this.description = description;
+    this.tasks = (tasks || []).map((item) => this.markTaskAsPending(item));
   }
-`;
-var skillExtractData = `skill name: extract_data_from_UI
-related input: DATA_DEMAND
-skill content: 
-* User will give you some data requirements in DATA_DEMAND. Consider the UI context, follow the user's instructions, and provide comprehensive data accordingly.
-* There may be some special commands in DATA_DEMAND, please pay extra attention
-  - ${ONE_ELEMENT_LOCATOR_PREFIX} and ${ELEMENTS_LOCATOR_PREFIX}: if you see a description that mentions the keyword ${ONE_ELEMENT_LOCATOR_PREFIX} or ${ELEMENTS_LOCATOR_PREFIX}(e.g. follow ${ONE_ELEMENT_LOCATOR_PREFIX} : i want to find ...), it means user wants to locate a specific element meets the description. Return in this way: prefix + the id / comma-separated ids, for example: ${ONE_ELEMENT_LOCATOR_PREFIX}/1 , ${ELEMENTS_LOCATOR_PREFIX}/1,2,3 . If not found, keep the prefix and leave the suffix empty, like ${ONE_ELEMENT_LOCATOR_PREFIX}/ .`;
-function promptsOfSectionQuery(constraints) {
-  if (!constraints.length) {
-    return "";
+  markTaskAsPending(task) {
+    return __spreadValues({
+      status: "pending"
+    }, task);
   }
-  const instruction = "Use your segment_a_web_page skill to find the following section(s)";
-  const singleSection = (c) => {
-    assert3(
-      c.name || c.description,
-      "either `name` or `description` is required to define a section constraint"
+  async append(task) {
+    assert2(
+      this.status !== "error",
+      "executor is in error state, cannot append task"
     );
-    const number = "One section";
-    const name = c.name ? `named \`${c.name}\`` : "";
-    const description = c.description ? `, usage or criteria : ${c.description}` : "";
-    const basic = `* ${number} ${name}${description}`;
-    return basic;
-  };
-  return `${instruction}
-${constraints.map(singleSection).join("\n")}`;
-}
-function systemPromptToExtract(dataQuery, sections) {
-  const allSectionNames = (sections == null ? void 0 : sections.filter((c) => c.name).map((c) => c.name || "")) || [];
-  const sectionFindingPrompt = promptsOfSectionQuery(sections || []);
-  const sectionReturnFormat = allSectionNames.length ? "  sections: [], // detailed information of each section from segment_a_web_page skill" : "";
-  return `
-${characteristic}
-${contextFormatIntro}
-
-You have the following skills:
-${allSectionNames.length ? skillSegment : ""}
-${skillExtractData}
-
-Now, do the following jobs:
-${sectionFindingPrompt}
-Use your extract_data_from_UI skill to find the following data, placing it in the \`data\` field
-DATA_DEMAND start:
-${typeof dataQuery === "object" ? `return in key-value style object, keys are ${Object.keys(dataQuery).join(",")}` : ""};
-${typeof dataQuery === "string" ? dataQuery : JSON.stringify(dataQuery, null, 2)}
-DATA_DEMAND ends.
-
-Return in the following JSON format:
-{
-  language: "en", // "en" or "zh", the language of the page. Use the same language to describe section name, description, and similar fields.
-  ${sectionReturnFormat}
-  data: any, // the extracted data from extract_data_from_UI skill. Make sure both the value and scheme meet the DATA_DEMAND.
-  errors?: [], // string[], error message if any
-}
-`;
-}
-function describeSize(size) {
-  return `${size.width} x ${size.height}`;
-}
-function truncateText(text) {
-  const maxLength = 50;
-  if (text && text.length > maxLength) {
-    return `${text.slice(0, maxLength)}...`;
-  }
-  return text;
-}
-async function describeUserPage(context) {
-  const { screenshotBase64 } = context;
-  let width;
-  let height;
-  if (context.size) {
-    ({ width, height } = context.size);
-  } else {
-    const imgSize = await imageInfoOfBase64(screenshotBase64);
-    ({ width, height } = imgSize);
-  }
-  const elementsInfo = context.content;
-  const idElementMap = {};
-  elementsInfo.forEach((item) => {
-    idElementMap[item.id] = item;
-    return __spreadValues({}, item);
-  });
-  const elementInfosDescription = cropfieldInformation(elementsInfo);
-  return {
-    description: `
-    {
-      // The size of the page
-      "pageSize": ${describeSize({ width, height })},
-
-      
-      // json description of the element
-      "elementInfos": ${JSON.stringify(elementInfosDescription)}
-    }`,
-    elementById(id) {
-      assert3(typeof id !== "undefined", "id is required for query");
-      const item = idElementMap[`${id}`];
-      return item;
-    }
-  };
-}
-function cropfieldInformation(elementsInfo) {
-  const elementInfosDescription = elementsInfo.map((item) => {
-    const { id, attributes = {}, rect, content } = item;
-    const tailorContent = truncateText(content);
-    const tailorAttributes = Object.keys(attributes).reduce((res, currentKey) => {
-      const attributeVal = attributes[currentKey];
-      res[currentKey] = truncateText(attributeVal);
-      return res;
-    }, {});
-    return {
-      id,
-      attributes: tailorAttributes,
-      rect,
-      content: tailorContent
-    };
-  });
-  return JSON.stringify(elementInfosDescription);
-}
-function retrieveElement(prompt, opt) {
-  if (opt == null ? void 0 : opt.multi) {
-    return `follow ${ELEMENTS_LOCATOR_PREFIX}: ${prompt}`;
-  }
-  return `follow ${ONE_ELEMENT_LOCATOR_PREFIX}: ${prompt}`;
-}
-function ifElementTypeResponse(response) {
-  if (typeof response !== "string") {
-    return false;
-  }
-  return response.startsWith(ONE_ELEMENT_LOCATOR_PREFIX) || response.startsWith(ELEMENTS_LOCATOR_PREFIX);
-}
-function splitElementResponse(response) {
-  const oneElementSplitter = `${ONE_ELEMENT_LOCATOR_PREFIX}/`;
-  if (response.startsWith(oneElementSplitter)) {
-    const id = response.slice(oneElementSplitter.length);
-    if (id.indexOf(",") >= 0) {
-      console.warn(`unexpected comma in one element response: ${id}`);
-    }
-    return id ? id : null;
-  }
-  const elementsSplitter = `${ELEMENTS_LOCATOR_PREFIX}/`;
-  if (response.startsWith(elementsSplitter)) {
-    const idsString = response.slice(elementsSplitter.length);
-    if (!idsString) {
-      return [];
+    if (Array.isArray(task)) {
+      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));
+    } else {
+      this.tasks.push(this.markTaskAsPending(task));
     }
-    return idsString.split(",");
-  }
-  return null;
-}
-function retrieveSection(prompt) {
-  return `${SECTION_MATCHER_FLAG}${prompt}`;
-}
-function extractSectionQuery(input) {
-  if (typeof input === "string" && input.startsWith(SECTION_MATCHER_FLAG)) {
-    return input.slice(SECTION_MATCHER_FLAG.length);
-  }
-  return false;
-}
-
-// src/insight/utils.ts
-import { existsSync as existsSync2 } from "fs";
-import { join as join2 } from "path";
-import { randomUUID as randomUUID2 } from "crypto";
-import assert4 from "assert";
-var logFileName = "";
-var logContent = [];
-var logIdIndexMap = {};
-var { pid } = process;
-var logFileExt = insightDumpFileExt;
-function writeInsightDump(data, logId, dumpSubscriber) {
-  const logDir2 = getDumpDir();
-  assert4(logDir2, "logDir should be set before writing dump file");
-  const id = logId || randomUUID2();
-  const baseData = {
-    sdkVersion: getPkgInfo().version,
-    logTime: Date.now()
-  };
-  const finalData = __spreadValues(__spreadValues({
-    logId: id
-  }, baseData), data);
-  dumpSubscriber == null ? void 0 : dumpSubscriber(finalData);
-  if (!logFileName) {
-    logFileName = `pid_${pid}_${baseData.logTime}`;
-    while (existsSync2(join2(logDir2, `${logFileName}.${logFileExt}`))) {
-      logFileName = `${pid}_${baseData.logTime}-${Math.random()}`;
+    if (this.status !== "running") {
+      this.status = "pending";
     }
   }
-  const dataString = JSON.stringify(finalData, null, 2);
-  if (typeof logIdIndexMap[id] === "number") {
-    logContent[logIdIndexMap[id]] = dataString;
-  } else {
-    const length = logContent.push(dataString);
-    logIdIndexMap[id] = length - 1;
-  }
-  writeDumpFile({
-    fileName: logFileName,
-    fileExt: logFileExt,
-    fileContent: `[
-${logContent.join(",\n")}
-]`
-  });
-  return id;
-}
-function idsIntoElements(ids, elementById) {
-  return ids.reduce((acc, id) => {
-    const element = elementById(id);
-    if (element) {
-      acc.push(element);
-    } else {
-      console.warn(`element not found by id: ${id}`);
+  async flush() {
+    if (this.status === "init" && this.tasks.length > 0) {
+      console.warn(
+        "illegal state for executor, status is init but tasks are not empty"
+      );
     }
-    return acc;
-  }, []);
-}
-function shallowExpandIds(data, ifMeet, elementsById) {
-  const keys = Object.keys(data);
-  keys.forEach((key) => {
-    const value = data[key];
-    if (typeof value === "string" && ifMeet(value)) {
-      data[key] = elementsById(value);
-    } else if (Array.isArray(value)) {
-      const newValue = value.map((id) => ifMeet(id) ? elementsById(id) : id);
-      data[key] = newValue;
+    assert2(this.status !== "running", "executor is already running");
+    assert2(this.status !== "completed", "executor is already completed");
+    assert2(this.status !== "error", "executor is in error state");
+    const nextPendingIndex = this.tasks.findIndex(
+      (task) => task.status === "pending"
+    );
+    if (nextPendingIndex < 0) {
+      return;
     }
-  });
-  return data;
-}
-function expandLiteSection(liteSection, elementById) {
-  const _a = liteSection, { textIds } = _a, remainingFields = __objRest(_a, ["textIds"]);
-  const texts = idsIntoElements(textIds, elementById);
-  let leftMost = -1;
-  let topMost = -1;
-  let rightMost = -1;
-  let bottomMost = -1;
-  texts.forEach((text) => {
-    leftMost = leftMost === -1 ? text.rect.left : Math.min(leftMost, text.rect.left);
-    topMost = topMost === -1 ? text.rect.top : Math.min(topMost, text.rect.top);
-    rightMost = Math.max(rightMost, text.rect.left + text.rect.width);
-    bottomMost = Math.max(bottomMost, text.rect.top + text.rect.height);
-  });
-  const sectionRect = {
-    left: leftMost,
-    top: topMost,
-    width: rightMost - leftMost,
-    height: bottomMost - topMost
-  };
-  const section = __spreadProps(__spreadValues({}, remainingFields), {
-    content: texts,
-    rect: sectionRect
-  });
-  return section;
-}
+    this.status = "running";
+    let taskIndex = nextPendingIndex;
+    let successfullyCompleted = true;
+    let errorMsg = "";
+    let previousFindOutput;
+    while (taskIndex < this.tasks.length) {
+      const task = this.tasks[taskIndex];
+      assert2(
+        task.status === "pending",
+        `task status should be pending, but got: ${task.status}`
+      );
+      task.timing = {
+        start: Date.now()
+      };
+      try {
+        task.status = "running";
+        assert2(
+          ["Insight", "Action", "Planning"].indexOf(task.type) >= 0,
+          `unsupported task type: ${task.type}`
+        );
+        const { executor, param } = task;
+        assert2(executor, `executor is required for task type: ${task.type}`);
+        let returnValue;
+        const executorContext = {
+          task,
+          element: previousFindOutput == null ? void 0 : previousFindOutput.element
+        };
+        if (task.type === "Insight") {
+          assert2(
+            task.subType === "Locate" || task.subType === "Query" || task.subType === "Assert",
+            `unsupported insight subType: ${task.subType}`
+          );
+          returnValue = await task.executor(param, executorContext);
+          if (task.subType === "Locate") {
+            previousFindOutput = returnValue == null ? void 0 : returnValue.output;
+          }
+        } else if (task.type === "Action" || task.type === "Planning") {
+          returnValue = await task.executor(param, executorContext);
+        } else {
+          console.warn(
+            `unsupported task type: ${task.type}, will try to execute it directly`
+          );
+          returnValue = await task.executor(param, executorContext);
+        }
+        Object.assign(task, returnValue);
+        task.status = "success";
+        task.timing.end = Date.now();
+        task.timing.cost = task.timing.end - task.timing.start;
+        taskIndex++;
+      } catch (e) {
+        successfullyCompleted = false;
+        task.status = "fail";
+        errorMsg = `${e == null ? void 0 : e.message}
+${e == null ? void 0 : e.stack}`;
+        task.error = errorMsg;
+        task.timing.end = Date.now();
+        task.timing.cost = task.timing.end - task.timing.start;
+        this.errorMsg = errorMsg;
+        break;
+      }
+    }
+    for (let i = taskIndex + 1; i < this.tasks.length; i++) {
+      this.tasks[i].status = "cancelled";
+    }
+    if (successfullyCompleted) {
+      this.status = "completed";
+      if (this.tasks.length) {
+        return this.tasks[this.tasks.length - 1].output;
+      }
+    } else {
+      this.status = "error";
+      throw new Error(`executor failed: ${errorMsg}`);
+    }
+  }
+  dump() {
+    const dumpData = {
+      sdkVersion: getPkgInfo().version,
+      logTime: Date.now(),
+      name: this.name,
+      description: this.description,
+      tasks: this.tasks
+    };
+    return dumpData;
+  }
+};
+
+// src/insight/index.ts
+import assert8 from "assert";
 
 // src/ai-model/openai.ts
-import assert5 from "assert";
-import OpenAI from "openai";
+import assert3 from "assert";
+
+// src/types.ts
+var BaseElement = class {
+};
+var AIResponseFormat = /* @__PURE__ */ ((AIResponseFormat2) => {
+  AIResponseFormat2["JSON"] = "json_object";
+  AIResponseFormat2["TEXT"] = "text";
+  return AIResponseFormat2;
+})(AIResponseFormat || {});
+var UIContext = class {
+};
 
 // ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.47.1/node_modules/langsmith/dist/traceable.js
 import { AsyncLocalStorage } from "async_hooks";
@@ -4605,23 +4455,13 @@ var wrapOpenAI = (openai, options) => {
   return openai;
 };
 
-// src/types.ts
-var BaseElement = class {
-};
-var AIResponseFormat = /* @__PURE__ */ ((AIResponseFormat2) => {
-  AIResponseFormat2["JSON"] = "json_object";
-  AIResponseFormat2["TEXT"] = "text";
-  return AIResponseFormat2;
-})(AIResponseFormat || {});
-var UIContext = class {
-};
-
 // src/ai-model/openai.ts
+import OpenAI from "openai";
 var envConfigKey = "MIDSCENE_OPENAI_INIT_CONFIG_JSON";
 var envModelKey = "MIDSCENE_MODEL_NAME";
 var envSmithDebug = "MIDSCENE_LANGSMITH_DEBUG";
 var extraConfig = {};
-if (typeof process.env[envConfigKey] === "string") {
+if (typeof process.env[envConfigKey] === "string" && process.env[envConfigKey]) {
   console.log("config for openai loaded");
   extraConfig = JSON.parse(process.env[envConfigKey]);
 }
@@ -4647,12 +4487,12 @@ async function call(messages, responseFormat) {
     response_format: { type: responseFormat }
   });
   const { content } = completion.choices[0].message;
-  assert5(content, "empty content");
+  assert3(content, "empty content");
   return content;
 }
 async function callToGetJSONObject(messages) {
   const response = await call(messages, "json_object" /* JSON */);
-  assert5(response, "empty response");
+  assert3(response, "empty response");
   return JSON.parse(response);
 }
 
@@ -4690,9 +4530,9 @@ You are an expert in software page image (2D) and page element text analysis.
   "elements": [
     // If no matching elements are found, return an empty array []
     {
-      "reason": "xxx", // The thought process for finding the element, replace xxx with your thought process
-      "text": "xxx", // Replace xxx with the text of elementInfo, if none, leave empty
-      "id": "xxx" // Replace xxx with the ID of elementInfo
+      "reason": "PLACEHOLDER", // The thought process for finding the element, replace PLACEHOLDER with your thought process
+      "text": "PLACEHOLDER", // Replace PLACEHOLDER with the text of elementInfo, if none, leave empty
+      "id": "PLACEHOLDER" // Replace PLACEHOLDER with the ID of elementInfo
     }
     // More elements...
   ],
@@ -4787,9 +4627,244 @@ Output Example:
   `;
 }
 
+// src/ai-model/prompt/util.ts
+import assert5 from "assert";
+
+// src/image/info.ts
+import assert4 from "assert";
+import { Buffer as Buffer2 } from "buffer";
+import { readFileSync as readFileSync2 } from "fs";
+import Sharp from "sharp";
+async function imageInfo(image) {
+  const { width, height } = await Sharp(image).metadata();
+  assert4(width && height, `invalid image: ${image}`);
+  return { width, height };
+}
+async function imageInfoOfBase64(imageBase64) {
+  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
+  return imageInfo(Buffer2.from(base64Data, "base64"));
+}
+
+// src/image/transform.ts
+import { Buffer as Buffer3 } from "buffer";
+import Sharp2 from "sharp";
+
+// src/image/visualization.ts
+import { Buffer as Buffer4 } from "buffer";
+import Sharp3 from "sharp";
+
+// src/ai-model/prompt/util.ts
+var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
+var contextFormatIntro = `
+The user will give you a screenshot and the texts on it. There may be some none-English characters (like Chinese) on it, indicating it's an non-English app.`;
+var ONE_ELEMENT_LOCATOR_PREFIX = "LOCATE_ONE_ELEMENT";
+var ELEMENTS_LOCATOR_PREFIX = "LOCATE_ONE_OR_MORE_ELEMENTS";
+var SECTION_MATCHER_FLAG = "SECTION_MATCHER_FLAG/";
+var skillSegment = `skill name: segment_a_web_page 
+skill content:
+Based on the functions and content of various elements on the page, segment the screenshot into different sections like navigation bar, product list, news area, etc. 
+Some general rules for segmentation:
+* Each section should NOT overlap with each other.
+* Each text should only belong to one section.
+* [IMPORTANT] Whether the content visually appears to belong to different sections is a significant factor in segmenting the page.
+* Analyze the page in a top-to-bottom and left-to-right order.
+* The evidence indicates a separate section, for example 
+  - The background color of certain parts of the page changes.
+  - A section of a page includes a title.
+* Provide the following data for each of the UI section you found.
+  {
+    "name": "name of the section",
+    "description": "briefly summarize the key content or usage of this section.",
+    "sectionCharacteristics": "In view of the need to distinguish this section from the surrounding sections, explain the characteristics and how to define boundaries and what precautions to take.",
+    "textIds": ["5", "6", "7"], // ids of all text elements in this section
+  }
+`;
+var skillExtractData = `skill name: extract_data_from_UI
+related input: DATA_DEMAND
+skill content: 
+* User will give you some data requirements in DATA_DEMAND. Consider the UI context, follow the user's instructions, and provide comprehensive data accordingly.
+* There may be some special commands in DATA_DEMAND, please pay extra attention
+  - ${ONE_ELEMENT_LOCATOR_PREFIX} and ${ELEMENTS_LOCATOR_PREFIX}: if you see a description that mentions the keyword ${ONE_ELEMENT_LOCATOR_PREFIX} or ${ELEMENTS_LOCATOR_PREFIX}(e.g. follow ${ONE_ELEMENT_LOCATOR_PREFIX} : i want to find ...), it means user wants to locate a specific element meets the description. Return in this way: prefix + the id / comma-separated ids, for example: ${ONE_ELEMENT_LOCATOR_PREFIX}/1 , ${ELEMENTS_LOCATOR_PREFIX}/1,2,3 . If not found, keep the prefix and leave the suffix empty, like ${ONE_ELEMENT_LOCATOR_PREFIX}/ .`;
+function promptsOfSectionQuery(constraints) {
+  if (!constraints.length) {
+    return "";
+  }
+  const instruction = "Use your segment_a_web_page skill to find the following section(s)";
+  const singleSection = (c) => {
+    assert5(
+      c.name || c.description,
+      "either `name` or `description` is required to define a section constraint"
+    );
+    const number = "One section";
+    const name = c.name ? `named \`${c.name}\`` : "";
+    const description = c.description ? `, usage or criteria : ${c.description}` : "";
+    const basic = `* ${number} ${name}${description}`;
+    return basic;
+  };
+  return `${instruction}
+${constraints.map(singleSection).join("\n")}`;
+}
+function systemPromptToExtract(dataQuery, sections) {
+  const allSectionNames = (sections == null ? void 0 : sections.filter((c) => c.name).map((c) => c.name || "")) || [];
+  const sectionFindingPrompt = promptsOfSectionQuery(sections || []);
+  const sectionReturnFormat = allSectionNames.length ? "  sections: [], // detailed information of each section from segment_a_web_page skill" : "";
+  return `
+${characteristic}
+${contextFormatIntro}
+
+You have the following skills:
+${allSectionNames.length ? skillSegment : ""}
+${skillExtractData}
+
+Now, do the following jobs:
+${sectionFindingPrompt}
+Use your extract_data_from_UI skill to find the following data, placing it in the \`data\` field
+DATA_DEMAND start:
+${typeof dataQuery === "object" ? `return in key-value style object, keys are ${Object.keys(dataQuery).join(",")}` : ""};
+${typeof dataQuery === "string" ? dataQuery : JSON.stringify(dataQuery, null, 2)}
+DATA_DEMAND ends.
+
+Return in the following JSON format:
+{
+  language: "en", // "en" or "zh", the language of the page. Use the same language to describe section name, description, and similar fields.
+  ${sectionReturnFormat}
+  data: any, // the extracted data from extract_data_from_UI skill. Make sure both the value and scheme meet the DATA_DEMAND.
+  errors?: [], // string[], error message if any
+}
+`;
+}
+function systemPromptToAssert(assertion) {
+  return `
+${characteristic}
+${contextFormatIntro}
+
+Based on the information you get, assert the following:
+${assertion}
+
+Return in the following JSON format:
+{
+  thought: string, // string, the thought of the assertion
+  pass: true, // true or false, whether the assertion is passed
+}
+`;
+}
+function describeSize(size) {
+  return `${size.width} x ${size.height}`;
+}
+function truncateText(text) {
+  const maxLength = 50;
+  if (text && text.length > maxLength) {
+    return `${text.slice(0, maxLength)}...`;
+  }
+  return text;
+}
+async function describeUserPage(context) {
+  const { screenshotBase64 } = context;
+  let width;
+  let height;
+  if (context.size) {
+    ({ width, height } = context.size);
+  } else {
+    const imgSize = await imageInfoOfBase64(screenshotBase64);
+    ({ width, height } = imgSize);
+  }
+  const elementsInfo = context.content;
+  const idElementMap = {};
+  elementsInfo.forEach((item) => {
+    idElementMap[item.id] = item;
+    return __spreadValues({}, item);
+  });
+  const elementInfosDescription = cropfieldInformation(elementsInfo);
+  return {
+    description: `
+    {
+      // The size of the page
+      "pageSize": ${describeSize({ width, height })},
+
+      
+      // json description of the element
+      "elementInfos": ${JSON.stringify(elementInfosDescription)}
+    }`,
+    elementById(id) {
+      assert5(typeof id !== "undefined", "id is required for query");
+      const item = idElementMap[`${id}`];
+      return item;
+    }
+  };
+}
+function cropfieldInformation(elementsInfo) {
+  const elementInfosDescription = elementsInfo.map(
+    (item) => {
+      const { id, attributes = {}, rect, content } = item;
+      const tailorContent = truncateText(content);
+      const tailorAttributes = Object.keys(attributes).reduce(
+        (res, currentKey) => {
+          const attributeVal = attributes[currentKey];
+          res[currentKey] = truncateText(attributeVal);
+          return res;
+        },
+        {}
+      );
+      return {
+        id,
+        attributes: tailorAttributes,
+        rect,
+        content: tailorContent
+      };
+    }
+  );
+  return JSON.stringify(elementInfosDescription);
+}
+function retrieveElement(prompt, opt) {
+  if (opt == null ? void 0 : opt.multi) {
+    return `follow ${ELEMENTS_LOCATOR_PREFIX}: ${prompt}`;
+  }
+  return `follow ${ONE_ELEMENT_LOCATOR_PREFIX}: ${prompt}`;
+}
+function ifElementTypeResponse(response) {
+  if (typeof response !== "string") {
+    return false;
+  }
+  return response.startsWith(ONE_ELEMENT_LOCATOR_PREFIX) || response.startsWith(ELEMENTS_LOCATOR_PREFIX);
+}
+function splitElementResponse(response) {
+  const oneElementSplitter = `${ONE_ELEMENT_LOCATOR_PREFIX}/`;
+  if (response.startsWith(oneElementSplitter)) {
+    const id = response.slice(oneElementSplitter.length);
+    if (id.indexOf(",") >= 0) {
+      console.warn(`unexpected comma in one element response: ${id}`);
+    }
+    return id ? id : null;
+  }
+  const elementsSplitter = `${ELEMENTS_LOCATOR_PREFIX}/`;
+  if (response.startsWith(elementsSplitter)) {
+    const idsString = response.slice(elementsSplitter.length);
+    if (!idsString) {
+      return [];
+    }
+    return idsString.split(",");
+  }
+  return null;
+}
+function retrieveSection(prompt) {
+  return `${SECTION_MATCHER_FLAG}${prompt}`;
+}
+function extractSectionQuery(input) {
+  if (typeof input === "string" && input.startsWith(SECTION_MATCHER_FLAG)) {
+    return input.slice(SECTION_MATCHER_FLAG.length);
+  }
+  return false;
+}
+
 // src/ai-model/inspect.ts
+import assert6 from "assert";
 async function AiInspectElement(options) {
-  const { context, multi, findElementDescription, callAI = callToGetJSONObject } = options;
+  const {
+    context,
+    multi,
+    findElementDescription,
+    callAI = callToGetJSONObject
+  } = options;
   const { screenshotBase64 } = context;
   const { description, elementById } = await describeUserPage(context);
   const systemPrompt = systemPromptToFindElement(findElementDescription, multi);
@@ -4815,12 +4890,16 @@ async function AiInspectElement(options) {
   const parseResult = await callAI(msgs);
   return {
     parseResult,
-    elementById,
-    systemPrompt
+    elementById
   };
 }
 async function AiExtractElementInfo(options) {
-  const { dataQuery, sectionConstraints, context, callAI = callToGetJSONObject } = options;
+  const {
+    dataQuery,
+    sectionConstraints,
+    context,
+    callAI = callToGetJSONObject
+  } = options;
   const systemPrompt = systemPromptToExtract(dataQuery, sectionConstraints);
   const { screenshotBase64 } = context;
   const { description, elementById } = await describeUserPage(context);
@@ -4845,18 +4924,137 @@ async function AiExtractElementInfo(options) {
   const parseResult = await callAI(msgs);
   return {
     parseResult,
-    elementById,
-    systemPrompt
+    elementById
+  };
+}
+async function AiAssert(options) {
+  const { assertion, context, callAI = callToGetJSONObject } = options;
+  assert6(assertion, "assertion should be a string");
+  const systemPrompt = systemPromptToAssert(assertion);
+  const { screenshotBase64 } = context;
+  const { description, elementById } = await describeUserPage(context);
+  const msgs = [
+    { role: "system", content: systemPrompt },
+    {
+      role: "user",
+      content: [
+        {
+          type: "image_url",
+          image_url: {
+            url: screenshotBase64
+          }
+        },
+        {
+          type: "text",
+          text: description
+        }
+      ]
+    }
+  ];
+  const assertResult = await callAI(msgs);
+  return assertResult;
+}
+
+// src/insight/utils.ts
+import assert7 from "assert";
+import { randomUUID as randomUUID2 } from "crypto";
+import { existsSync as existsSync2 } from "fs";
+import { join as join2 } from "path";
+var logFileName = "";
+var logContent = [];
+var logIdIndexMap = {};
+var { pid } = process;
+var logFileExt = insightDumpFileExt;
+function writeInsightDump(data, logId, dumpSubscriber) {
+  const logDir2 = getDumpDir();
+  assert7(logDir2, "logDir should be set before writing dump file");
+  const id = logId || randomUUID2();
+  const baseData = {
+    sdkVersion: getPkgInfo().version,
+    logTime: Date.now()
+  };
+  const finalData = __spreadValues(__spreadValues({
+    logId: id
+  }, baseData), data);
+  dumpSubscriber == null ? void 0 : dumpSubscriber(finalData);
+  if (!logFileName) {
+    logFileName = `pid_${pid}_${baseData.logTime}`;
+    while (existsSync2(join2(logDir2, `${logFileName}.${logFileExt}`))) {
+      logFileName = `${pid}_${baseData.logTime}-${Math.random()}`;
+    }
+  }
+  const dataString = JSON.stringify(finalData, null, 2);
+  if (typeof logIdIndexMap[id] === "number") {
+    logContent[logIdIndexMap[id]] = dataString;
+  } else {
+    const length = logContent.push(dataString);
+    logIdIndexMap[id] = length - 1;
+  }
+  writeDumpFile({
+    fileName: logFileName,
+    fileExt: logFileExt,
+    fileContent: `[
+${logContent.join(",\n")}
+]`
+  });
+  return id;
+}
+function idsIntoElements(ids, elementById) {
+  return ids.reduce((acc, id) => {
+    const element = elementById(id);
+    if (element) {
+      acc.push(element);
+    } else {
+      console.warn(`element not found by id: ${id}`);
+    }
+    return acc;
+  }, []);
+}
+function shallowExpandIds(data, ifMeet, elementsById) {
+  const keys = Object.keys(data);
+  keys.forEach((key) => {
+    const value = data[key];
+    if (typeof value === "string" && ifMeet(value)) {
+      data[key] = elementsById(value);
+    } else if (Array.isArray(value)) {
+      const newValue = value.map((id) => ifMeet(id) ? elementsById(id) : id);
+      data[key] = newValue;
+    }
+  });
+  return data;
+}
+function expandLiteSection(liteSection, elementById) {
+  const _a = liteSection, { textIds } = _a, remainingFields = __objRest(_a, ["textIds"]);
+  const texts = idsIntoElements(textIds, elementById);
+  let leftMost = -1;
+  let topMost = -1;
+  let rightMost = -1;
+  let bottomMost = -1;
+  texts.forEach((text) => {
+    leftMost = leftMost === -1 ? text.rect.left : Math.min(leftMost, text.rect.left);
+    topMost = topMost === -1 ? text.rect.top : Math.min(topMost, text.rect.top);
+    rightMost = Math.max(rightMost, text.rect.left + text.rect.width);
+    bottomMost = Math.max(bottomMost, text.rect.top + text.rect.height);
+  });
+  const sectionRect = {
+    left: leftMost,
+    top: topMost,
+    width: rightMost - leftMost,
+    height: bottomMost - topMost
   };
+  const section = __spreadProps(__spreadValues({}, remainingFields), {
+    content: texts,
+    rect: sectionRect
+  });
+  return section;
 }
 
 // src/insight/index.ts
 var sortByOrder = (a, b) => {
   if (a.rect.top - b.rect.top !== 0) {
     return a.rect.top - b.rect.top;
-  } else {
-    return a.rect.left - b.rect.left;
   }
+  return a.rect.left - b.rect.left;
 };
 var Insight = class {
   constructor(context, opt) {
@@ -4864,7 +5062,7 @@ var Insight = class {
     __publicField(this, "aiVendorFn", callToGetJSONObject);
     __publicField(this, "onceDumpUpdatedFn");
     __publicField(this, "taskInfo");
-    assert6(context, "context is required for Insight");
+    assert8(context, "context is required for Insight");
     if (typeof context === "function") {
       this.contextRetrieverFn = context;
     } else {
@@ -4880,12 +5078,12 @@ var Insight = class {
   async locate(queryPrompt, opt) {
     var _a;
     const { callAI = this.aiVendorFn, multi = false } = opt || {};
-    assert6(queryPrompt, "query is required for located");
+    assert8(queryPrompt, "query is required for located");
     const dumpSubscriber = this.onceDumpUpdatedFn;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn();
     const startTime = Date.now();
-    const { parseResult, systemPrompt, elementById } = await AiInspectElement({
+    const { parseResult, elementById } = await AiInspectElement({
       callAI,
       context,
       multi: Boolean(multi),
@@ -4894,8 +5092,7 @@ var Insight = class {
     const timeCost = Date.now() - startTime;
     const taskInfo = __spreadProps(__spreadValues({}, this.taskInfo ? this.taskInfo : {}), {
       durationMs: timeCost,
-      rawResponse: JSON.stringify(parseResult),
-      systemPrompt
+      rawResponse: JSON.stringify(parseResult)
     });
     let errorLog;
     if ((_a = parseResult.errors) == null ? void 0 : _a.length) {
@@ -4923,7 +5120,9 @@ ${parseResult.errors.join("\n")}`;
     parseResult.elements.forEach((item) => {
       const element = elementById(item.id);
       if (!element) {
-        console.warn(`locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`);
+        console.warn(
+          `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`
+        );
         return;
       }
       elements.push(element);
@@ -4937,20 +5136,23 @@ ${parseResult.errors.join("\n")}`;
     );
     if (opt == null ? void 0 : opt.multi) {
       return elements;
-    } else if (elements.length >= 2) {
-      console.warn(`locate: multiple elements found, return the first one. (query: ${queryPrompt})`);
+    }
+    if (elements.length >= 2) {
+      console.warn(
+        `locate: multiple elements found, return the first one. (query: ${queryPrompt})`
+      );
       return elements[0];
-    } else if (elements.length === 1) {
+    }
+    if (elements.length === 1) {
       return elements[0];
-    } else {
-      return null;
     }
+    return null;
   }
   async extract(dataDemand) {
     var _a;
     let dataQuery = {};
     const sectionQueryMap = {};
-    assert6(
+    assert8(
       typeof dataDemand === "object" || typeof dataDemand === "string",
       `dataDemand should be object or string, but get ${typeof dataDemand}`
     );
@@ -4979,7 +5181,7 @@ ${parseResult.errors.join("\n")}`;
     });
     const context = await this.contextRetrieverFn();
     const startTime = Date.now();
-    const { parseResult, systemPrompt, elementById } = await AiExtractElementInfo({
+    const { parseResult, elementById } = await AiExtractElementInfo({
       context,
       dataQuery,
       sectionConstraints,
@@ -4988,8 +5190,7 @@ ${parseResult.errors.join("\n")}`;
     const timeCost = Date.now() - startTime;
     const taskInfo = __spreadProps(__spreadValues({}, this.taskInfo ? this.taskInfo : {}), {
       durationMs: timeCost,
-      rawResponse: JSON.stringify(parseResult),
-      systemPrompt
+      rawResponse: JSON.stringify(parseResult)
     });
     let errorLog;
     if ((_a = parseResult.errors) == null ? void 0 : _a.length) {
@@ -5014,7 +5215,10 @@ ${parseResult.errors.join("\n")}`;
       throw new Error(errorLog);
     }
     const sectionsArr = (parseResult.sections || []).map((liteSection) => {
-      const section = expandLiteSection(liteSection, (id) => elementById(id));
+      const section = expandLiteSection(
+        liteSection,
+        (id) => elementById(id)
+      );
       return section;
     }).sort(sortByOrder);
     const sectionMap = sectionsArr.reduce((acc, section) => {
@@ -5038,7 +5242,8 @@ ${parseResult.errors.join("\n")}`;
         const idList = splitElementResponse(id);
         if (typeof idList === "string") {
           return elementById(idList);
-        } else if (Array.isArray(idList)) {
+        }
+        if (Array.isArray(idList)) {
           return idsIntoElements(idList, elementById);
         }
         return idList;
@@ -5055,133 +5260,46 @@ ${parseResult.errors.join("\n")}`;
     );
     return mergedData;
   }
-  setAiVendorFn(aiVendorFn) {
-    const origin = this.aiVendorFn;
-    this.aiVendorFn = aiVendorFn;
-    return () => {
-      this.aiVendorFn = origin;
-    };
-  }
-};
-
-// src/action/executor.ts
-import assert7 from "assert";
-var Executor = class {
-  constructor(name, description, tasks) {
-    __publicField(this, "name");
-    __publicField(this, "description");
-    __publicField(this, "tasks");
-    // status of executor
-    __publicField(this, "status");
-    __publicField(this, "errorMsg");
-    __publicField(this, "dumpFileName");
-    this.status = tasks && tasks.length > 0 ? "pending" : "init";
-    this.name = name;
-    this.description = description;
-    this.tasks = (tasks || []).map((item) => this.markTaskAsPending(item));
-  }
-  markTaskAsPending(task) {
-    return __spreadValues({
-      status: "pending"
-    }, task);
-  }
-  async append(task) {
-    assert7(this.status !== "error", "executor is in error state, cannot append task");
-    if (Array.isArray(task)) {
-      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));
-    } else {
-      this.tasks.push(this.markTaskAsPending(task));
-    }
-    if (this.status !== "running") {
-      this.status = "pending";
-    }
-  }
-  async flush() {
-    if (this.status === "init" && this.tasks.length > 0) {
-      console.warn("illegal state for executor, status is init but tasks are not empty");
-    }
-    assert7(this.status !== "running", "executor is already running");
-    assert7(this.status !== "completed", "executor is already completed");
-    assert7(this.status !== "error", "executor is in error state");
-    const nextPendingIndex = this.tasks.findIndex((task) => task.status === "pending");
-    if (nextPendingIndex < 0) {
-      return;
-    }
-    this.status = "running";
-    let taskIndex = nextPendingIndex;
-    let successfullyCompleted = true;
-    let errorMsg = "";
-    let previousFindOutput;
-    while (taskIndex < this.tasks.length) {
-      const task = this.tasks[taskIndex];
-      assert7(task.status === "pending", `task status should be pending, but got: ${task.status}`);
-      task.timing = {
-        start: Date.now()
-      };
-      try {
-        task.status = "running";
-        assert7(
-          ["Insight", "Action", "Planning"].indexOf(task.type) >= 0,
-          `unsupported task type: ${task.type}`
-        );
-        const { executor, param } = task;
-        assert7(executor, `executor is required for task type: ${task.type}`);
-        let returnValue;
-        const executorContext = {
-          task,
-          element: previousFindOutput == null ? void 0 : previousFindOutput.element
-        };
-        if (task.type === "Insight") {
-          assert7(
-            task.subType === "Locate" || task.subType === "Query",
-            `unsupported insight subType: ${task.subType}`
-          );
-          returnValue = await task.executor(param, executorContext);
-          if (task.subType === "Locate") {
-            previousFindOutput = returnValue == null ? void 0 : returnValue.output;
-          }
-        } else if (task.type === "Action" || task.type === "Planning") {
-          returnValue = await task.executor(param, executorContext);
-        } else {
-          console.warn(`unsupported task type: ${task.type}, will try to execute it directly`);
-          returnValue = await task.executor(param, executorContext);
-        }
-        Object.assign(task, returnValue);
-        task.status = "success";
-        task.timing.end = Date.now();
-        task.timing.cost = task.timing.end - task.timing.start;
-        taskIndex++;
-      } catch (e) {
-        successfullyCompleted = false;
-        task.status = "fail";
-        errorMsg = `${e == null ? void 0 : e.message}
-${e == null ? void 0 : e.stack}`;
-        task.error = errorMsg;
-        task.timing.end = Date.now();
-        task.timing.cost = task.timing.end - task.timing.start;
-        this.errorMsg = errorMsg;
-        break;
-      }
-    }
-    for (let i = taskIndex + 1; i < this.tasks.length; i++) {
-      this.tasks[i].status = "cancelled";
-    }
-    if (successfullyCompleted) {
-      this.status = "completed";
-    } else {
-      this.status = "error";
-      throw new Error(`executor failed: ${errorMsg}`);
+  async assert(assertion) {
+    if (typeof assertion !== "string") {
+      throw new Error(
+        "This is the assert method for Midscene, the first argument should be a string. If you want to use the assert method from Node.js, please import it from the Node.js assert module."
+      );
     }
-  }
-  dump() {
+    const dumpSubscriber = this.onceDumpUpdatedFn;
+    this.onceDumpUpdatedFn = void 0;
+    const context = await this.contextRetrieverFn();
+    const startTime = Date.now();
+    const assertResult = await AiAssert({
+      assertion,
+      callAI: this.aiVendorFn,
+      context
+    });
+    const timeCost = Date.now() - startTime;
+    const taskInfo = __spreadProps(__spreadValues({}, this.taskInfo ? this.taskInfo : {}), {
+      durationMs: timeCost,
+      rawResponse: JSON.stringify(assertResult)
+    });
+    const { thought, pass } = assertResult;
     const dumpData = {
-      sdkVersion: getPkgInfo().version,
-      logTime: Date.now(),
-      name: this.name,
-      description: this.description,
-      tasks: this.tasks
+      type: "assert",
+      context,
+      userQuery: {
+        assertion
+      },
+      matchedSection: [],
+      matchedElement: [],
+      data: null,
+      taskInfo,
+      assertionPass: pass,
+      assertionThought: thought,
+      error: pass ? void 0 : thought
+    };
+    writeInsightDump(dumpData, void 0, dumpSubscriber);
+    return {
+      pass,
+      thought
     };
-    return dumpData;
   }
 };