@midscene/core 0.0.1

package/dist/es/utils.js

@@ -0,0 +1,96 @@
+// src/utils.ts
+import { tmpdir } from "os";
+import { basename, join } from "path";
+import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { randomUUID } from "crypto";
+import assert from "assert";
+var pkg;
+function getPkgInfo() {
+  if (pkg) {
+    return pkg;
+  }
+  let pkgJsonFile = "";
+  if (existsSync(join(__dirname, "../package.json"))) {
+    pkgJsonFile = join(__dirname, "../package.json");
+  } else if (existsSync(join(__dirname, "../../../package.json"))) {
+    pkgJsonFile = join(__dirname, "../../../package.json");
+  }
+  if (pkgJsonFile) {
+    const { name, version } = JSON.parse(readFileSync(pkgJsonFile, "utf-8"));
+    pkg = { name, version };
+    return pkg;
+  } else {
+    return {
+      name: "midscene-unknown-page-name",
+      version: "0.0.0"
+    };
+  }
+}
+var logDir = join(process.cwd(), "./midscene_run/");
+var logEnvReady = false;
+var insightDumpFileExt = "insight-dump.json";
+var groupedActionDumpFileExt = "web-dump.json";
+function getDumpDir() {
+  return logDir;
+}
+function setDumpDir(dir) {
+  logDir = dir;
+}
+function writeDumpFile(fileName, fileExt, fileContent) {
+  if (!logEnvReady) {
+    assert(logDir, "logDir should be set before writing dump file");
+    if (!existsSync(logDir)) {
+      mkdirSync(logDir, { recursive: true });
+    }
+    const gitIgnorePath = join(logDir, "../.gitignore");
+    let gitIgnoreContent = "";
+    if (existsSync(gitIgnorePath)) {
+      gitIgnoreContent = readFileSync(gitIgnorePath, "utf-8");
+    }
+    const logDirName = basename(logDir);
+    if (!gitIgnoreContent.includes(`${logDirName}/`)) {
+      writeFileSync(
+        gitIgnorePath,
+        `${gitIgnoreContent}
+# MidScene.js dump files
+${logDirName}/
+`,
+        "utf-8"
+      );
+    }
+    logEnvReady = true;
+  }
+  const filePath = join(getDumpDir(), `${fileName}.${fileExt}`);
+  writeFileSync(filePath, fileContent);
+  copyFileSync(filePath, join(getDumpDir(), `latest.${fileExt}`));
+  return filePath;
+}
+function getTmpDir() {
+  const path = join(tmpdir(), getPkgInfo().name);
+  mkdirSync(path, { recursive: true });
+  return path;
+}
+function getTmpFile(fileExt) {
+  const filename = `${randomUUID()}.${fileExt}`;
+  return join(getTmpDir(), filename);
+}
+function overlapped(container, target) {
+  return container.left < target.left + target.width && container.left + container.width > target.left && container.top < target.top + target.height && container.top + container.height > target.top;
+}
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+var commonScreenshotParam = { type: "jpeg", quality: 75 };
+export {
+  commonScreenshotParam,
+  getDumpDir,
+  getPkgInfo,
+  getTmpDir,
+  getTmpFile,
+  groupedActionDumpFileExt,
+  insightDumpFileExt,
+  overlapped,
+  setDumpDir,
+  sleep,
+  writeDumpFile
+};

package/dist/lib/ai-model.js

@@ -0,0 +1,467 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/ai-model/index.ts
+var ai_model_exports = {};
+__export(ai_model_exports, {
+  AiExtractElementInfo: () => AiExtractElementInfo,
+  AiInspectElement: () => AiInspectElement,
+  callToGetJSONObject: () => callToGetJSONObject,
+  describeUserPage: () => describeUserPage,
+  systemPromptToFindElement: () => systemPromptToFindElement
+});
+module.exports = __toCommonJS(ai_model_exports);
+
+// src/ai-model/openai.ts
+var import_assert = __toESM(require("assert"));
+var import_openai = __toESM(require("openai"));
+var import_wrappers = __toESM(require("langsmith/wrappers"));
+var envConfigKey = "MIDSCENE_OPENAI_INIT_CONFIG_JSON";
+var envModelKey = "MIDSCENE_MODEL_NAME";
+var envSmithDebug = "MIDSCENE_LANGSMITH_DEBUG";
+var extraConfig = {};
+if (typeof process.env[envConfigKey] === "string") {
+  console.log("will use env config for openai");
+  extraConfig = JSON.parse(process.env[envConfigKey]);
+}
+var model = "gpt-4o";
+if (typeof process.env[envModelKey] === "string") {
+  console.log(`will use model: ${process.env[envModelKey]}`);
+  model = process.env[envModelKey];
+}
+async function createOpenAI() {
+  const openai = new import_openai.default(extraConfig);
+  if (process.env[envSmithDebug]) {
+    console.log("DEBUGGING MODE: using langsmith wrapper");
+    const openai2 = import_wrappers.default.wrapOpenAI(new import_openai.default());
+    return openai2;
+  }
+  return openai;
+}
+async function call(messages, responseFormat) {
+  const openai = await createOpenAI();
+  const completion = await openai.chat.completions.create({
+    model,
+    messages,
+    response_format: { type: responseFormat }
+  });
+  const { content } = completion.choices[0].message;
+  (0, import_assert.default)(content, "empty content");
+  return content;
+}
+async function callToGetJSONObject(messages) {
+  const response = await call(messages, "json_object" /* JSON */);
+  (0, import_assert.default)(response, "empty response");
+  return JSON.parse(response);
+}
+
+// src/ai-model/prompt/element_inspector.ts
+function systemPromptToFindElement(description, multi) {
+  return `
+## Role:
+You are an expert in software page image (2D) and page element text analysis.
+
+## Objective:
+- Identify elements in screenshots and text that match the user's description.
+- Return JSON data containing the selection reason and element ID.
+
+## Skills:
+- Image analysis and recognition
+- Multilingual text understanding
+- Software UI design and testing
+
+## Workflow:
+1. Receive the user's element description, screenshot, and element description information. Note that the text may contain non-English characters (e.g., Chinese), indicating that the application may be non-English.
+2. Based on the description (${description}), locate the target element ID in the list of element descriptions and the screenshot.
+3. Return the number of elements: ${multi ? "multiple elements matching the description (two or more)" : "The element closest to the description (only one)"}.
+4. Return JSON data containing the selection reason and element ID.
+
+## Constraints:
+- Strictly adhere to the specified location when describing the required element; do not select elements from other locations.
+- Elements in the image with NodeType other than "TEXT Node" have been highlighted to identify the element among multiple non-text elements.
+- Accurately identify element information based on the user's description and return the corresponding element ID from the element description information, not extracted from the image.
+- If no elements are found, the "elements" array should be empty.
+- The returned data must conform to the specified JSON format.
+
+## Output Format:
+\`\`\`json
+{
+  "elements": [
+    // If no matching elements are found, return an empty array []
+    {
+      "reason": "xxx", // The thought process for finding the element, replace xxx with your thought process
+      "text": "xxx", // Replace xxx with the text of elementInfo, if none, leave empty
+      "id": "xxx" // Replace xxx with the ID of elementInfo
+    }
+    // More elements...
+  ],
+  "errors": [] // Array of strings containing any error messages
+}
+\`\`\`
+
+## Example:
+Example 1:
+Input Example:
+\`\`\`json
+// Description: "Shopping cart icon in the upper right corner"
+{
+  "screenshot": "path/screenshot.png",
+  "text": '{
+      "pageSize": {
+        "width": 400, // Width of the page
+        "height": 905 // Height of the page
+      },
+      "elementInfos": [
+        {
+          "id": "3", // ID of the element
+          "attributes": { // Attributes of the element
+            "nodeType": "IMG Node", // Type of element, types include: TEXT Node, IMG Node, BUTTON Node, INPUT Node
+            "src": "https://ap-southeast-3.m",
+            "class": ".img"
+          },
+          "content": "", // Text content of the element
+          "rect": {
+            "left": 280, // Distance from the left side of the page
+            "top": 8, // Distance from the top of the page
+            "width": 44, // Width of the element
+            "height": 44 // Height of the element
+          }
+        },
+        {
+          "id": "4", // ID of the element
+          "attributes": { // Attributes of the element
+            "nodeType": "IMG Node", // Type of element, types include: TEXT Node, IMG Node, BUTTON Node, INPUT Node
+            "src": "data:image/png;base64,iVBORw0KGgoAAAANSU...",
+            "class": ".icon"
+          },
+          "content": "", // Text content of the element
+          "rect": {
+            "left": 350, // Distance from the left side of the page
+            "top": 16, // Distance from the top of the page
+            "width": 25, // Width of the element
+            "height": 25 // Height of the element
+          }
+        },
+        ...
+        {
+          "id": "27",
+          "attributes": {
+            "nodeType": "TEXT Node",
+            "class": ".product-name"
+          },
+          "center": [
+            288,
+            834
+          ],
+          "content": "Mango Drink",
+          "rect": {
+            "left": 188,
+            "top": 827,
+            "width": 199,
+            "height": 13
+          }
+        },
+        ...
+      ]
+    }
+  '
+}
+\`\`\`
+Output Example:
+\`\`\`json
+{
+  "elements": [
+    {
+      // Describe the reason for finding this element, replace with actual value in practice
+      "reason": "Reason for finding element 4: It is located in the upper right corner, is an image type, and according to the screenshot, it is a shopping cart icon button",
+      "text": "",
+      // ID of this element, replace with actual value in practice
+      "id": "4"
+    }
+  ],
+  "errors": []
+}
+\`\`\`
+  
+  `;
+}
+
+// src/ai-model/prompt/util.ts
+var import_assert3 = __toESM(require("assert"));
+
+// src/image/info.ts
+var import_node_assert = __toESM(require("assert"));
+var import_buffer = require("buffer");
+var import_node_fs = require("fs");
+var import_sharp = __toESM(require("sharp"));
+async function imageInfo(image) {
+  const { width, height } = await (0, import_sharp.default)(image).metadata();
+  (0, import_node_assert.default)(width && height, `invalid image: ${image}`);
+  return { width, height };
+}
+async function imageInfoOfBase64(imageBase64) {
+  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
+  return imageInfo(import_buffer.Buffer.from(base64Data, "base64"));
+}
+
+// src/image/transform.ts
+var import_node_buffer = require("buffer");
+var import_sharp2 = __toESM(require("sharp"));
+
+// src/image/visualization.ts
+var import_buffer2 = require("buffer");
+var import_sharp3 = __toESM(require("sharp"));
+
+// src/utils.ts
+var import_os = require("os");
+var import_path = require("path");
+var import_fs = require("fs");
+var import_crypto = require("crypto");
+var import_assert2 = __toESM(require("assert"));
+var logDir = (0, import_path.join)(process.cwd(), "./midscene_run/");
+
+// src/ai-model/prompt/util.ts
+var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
+var contextFormatIntro = `
+The user will give you a screenshot and the texts on it. There may be some none-English characters (like Chinese) on it, indicating it's an non-English app.`;
+var ONE_ELEMENT_LOCATOR_PREFIX = "LOCATE_ONE_ELEMENT";
+var ELEMENTS_LOCATOR_PREFIX = "LOCATE_ONE_OR_MORE_ELEMENTS";
+var skillSegment = `skill name: segment_a_web_page 
+skill content:
+Based on the functions and content of various elements on the page, segment the screenshot into different sections like navigation bar, product list, news area, etc. 
+Some general rules for segmentation:
+* Each section should NOT overlap with each other.
+* Each text should only belong to one section.
+* [IMPORTANT] Whether the content visually appears to belong to different sections is a significant factor in segmenting the page.
+* Analyze the page in a top-to-bottom and left-to-right order.
+* The evidence indicates a separate section, for example 
+  - The background color of certain parts of the page changes.
+  - A section of a page includes a title.
+* Provide the following data for each of the UI section you found.
+  {
+    "name": "name of the section",
+    "description": "briefly summarize the key content or usage of this section.",
+    "sectionCharacteristics": "In view of the need to distinguish this section from the surrounding sections, explain the characteristics and how to define boundaries and what precautions to take.",
+    "textIds": ["5", "6", "7"], // ids of all text elements in this section
+  }
+`;
+var skillExtractData = `skill name: extract_data_from_UI
+related input: DATA_DEMAND
+skill content: 
+* User will give you some data requirements in DATA_DEMAND. Consider the UI context, follow the user's instructions, and provide comprehensive data accordingly.
+* There may be some special commands in DATA_DEMAND, please pay extra attention
+  - ${ONE_ELEMENT_LOCATOR_PREFIX} and ${ELEMENTS_LOCATOR_PREFIX}: if you see a description that mentions the keyword ${ONE_ELEMENT_LOCATOR_PREFIX} or ${ELEMENTS_LOCATOR_PREFIX}(e.g. follow ${ONE_ELEMENT_LOCATOR_PREFIX} : i want to find ...), it means user wants to locate a specific element meets the description. Return in this way: prefix + the id / comma-separated ids, for example: ${ONE_ELEMENT_LOCATOR_PREFIX}/1 , ${ELEMENTS_LOCATOR_PREFIX}/1,2,3 . If not found, keep the prefix and leave the suffix empty, like ${ONE_ELEMENT_LOCATOR_PREFIX}/ .`;
+function promptsOfSectionQuery(constraints) {
+  if (!constraints.length) {
+    return "";
+  }
+  const instruction = "Use your segment_a_web_page skill to find the following section(s)";
+  const singleSection = (c) => {
+    (0, import_assert3.default)(
+      c.name || c.description,
+      "either `name` or `description` is required to define a section constraint"
+    );
+    const number = "One section";
+    const name = c.name ? `named \`${c.name}\`` : "";
+    const description = c.description ? `, usage or criteria : ${c.description}` : "";
+    const basic = `* ${number} ${name}${description}`;
+    return basic;
+  };
+  return `${instruction}
+${constraints.map(singleSection).join("\n")}`;
+}
+function systemPromptToExtract(dataQuery, sections) {
+  const allSectionNames = (sections == null ? void 0 : sections.filter((c) => c.name).map((c) => c.name || "")) || [];
+  const sectionFindingPrompt = promptsOfSectionQuery(sections || []);
+  const sectionReturnFormat = allSectionNames.length ? "  sections: [], // detailed information of each section from segment_a_web_page skill" : "";
+  return `
+${characteristic}
+${contextFormatIntro}
+
+You have the following skills:
+${allSectionNames.length ? skillSegment : ""}
+${skillExtractData}
+
+Now, do the following jobs:
+${sectionFindingPrompt}
+Use your extract_data_from_UI skill to find the following data, placing it in the \`data\` field
+DATA_DEMAND start:
+${typeof dataQuery === "object" ? `return in key-value style object, keys are ${Object.keys(dataQuery).join(",")}` : ""};
+${typeof dataQuery === "string" ? dataQuery : JSON.stringify(dataQuery, null, 2)}
+DATA_DEMAND ends.
+
+Return in the following JSON format:
+{
+  language: "en", // "en" or "zh", the language of the page. Use the same language to describe section name, description, and similar fields.
+  ${sectionReturnFormat}
+  data: any, // the extracted data from extract_data_from_UI skill. Make sure both the value and scheme meet the DATA_DEMAND.
+  errors?: [], // string[], error message if any
+}
+`;
+}
+function describeSize(size) {
+  return `${size.width} x ${size.height}`;
+}
+function truncateText(text) {
+  const maxLength = 50;
+  if (text && text.length > maxLength) {
+    return `${text.slice(0, maxLength)}...`;
+  }
+  return text;
+}
+async function describeUserPage(context) {
+  const { screenshotBase64 } = context;
+  const { width, height } = await imageInfoOfBase64(screenshotBase64);
+  const elementsInfo = context.content;
+  const idElementMap = {};
+  elementsInfo.forEach((item) => {
+    idElementMap[item.id] = item;
+    return __spreadValues({}, item);
+  });
+  const elementInfosDescription = cropfieldInformation(elementsInfo);
+  return {
+    description: `
+    {
+      // The size of the page
+      "pageSize": ${describeSize({ width, height })},
+
+      
+      // json description of the element
+      "elementInfos": ${JSON.stringify(elementInfosDescription)}
+    }`,
+    elementById(id) {
+      (0, import_assert3.default)(typeof id !== "undefined", "id is required for query");
+      const item = idElementMap[`${id}`];
+      return item;
+    }
+  };
+}
+function cropfieldInformation(elementsInfo) {
+  const elementInfosDescription = elementsInfo.map((item) => {
+    const { id, attributes = {}, rect, content } = item;
+    const tailorContent = truncateText(content);
+    const tailorAttributes = Object.keys(attributes).reduce((res, currentKey) => {
+      const attributeVal = attributes[currentKey];
+      res[currentKey] = truncateText(attributeVal);
+      return res;
+    }, {});
+    return {
+      id,
+      attributes: tailorAttributes,
+      rect,
+      content: tailorContent
+    };
+  });
+  return JSON.stringify(elementInfosDescription);
+}
+
+// src/ai-model/inspect.ts
+async function AiInspectElement(options) {
+  const { context, multi, findElementDescription, callAI = callToGetJSONObject } = options;
+  const { screenshotBase64 } = context;
+  const { description, elementById } = await describeUserPage(context);
+  const systemPrompt = systemPromptToFindElement(findElementDescription, multi);
+  const msgs = [
+    { role: "system", content: systemPrompt },
+    {
+      role: "user",
+      content: [
+        {
+          type: "image_url",
+          image_url: {
+            url: screenshotBase64,
+            detail: "high"
+          }
+        },
+        {
+          type: "text",
+          text: description
+        }
+      ]
+    }
+  ];
+  const parseResult = await callAI(msgs);
+  return {
+    parseResult,
+    elementById,
+    systemPrompt
+  };
+}
+async function AiExtractElementInfo(options) {
+  const { dataQuery, sectionConstraints, context, callAI = callToGetJSONObject } = options;
+  const systemPrompt = systemPromptToExtract(dataQuery, sectionConstraints);
+  const { screenshotBase64 } = context;
+  const { description, elementById } = await describeUserPage(context);
+  const msgs = [
+    { role: "system", content: systemPrompt },
+    {
+      role: "user",
+      content: [
+        {
+          type: "image_url",
+          image_url: {
+            url: screenshotBase64
+          }
+        },
+        {
+          type: "text",
+          text: description
+        }
+      ]
+    }
+  ];
+  const parseResult = await callAI(msgs);
+  return {
+    parseResult,
+    elementById,
+    systemPrompt
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AiExtractElementInfo,
+  AiInspectElement,
+  callToGetJSONObject,
+  describeUserPage,
+  systemPromptToFindElement
+});