@midscene/core 0.0.1

package/demo_data/index.d.ts

@@ -0,0 +1 @@
+declare module 'midscene/demo_data';

package/demo_data/index.js

@@ -0,0 +1,6 @@
+import * as insightDemo from './demo.insight.json';
+import * as actionDemo from './demo.actions.json';
+
+export const insight = insightDemo.default;
+export const action = actionDemo.default;
+

package/dist/es/ai-model.js

@@ -0,0 +1,429 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+
+// src/ai-model/openai.ts
+import assert from "assert";
+import OpenAI from "openai";
+import wrapper from "langsmith/wrappers";
+var envConfigKey = "MIDSCENE_OPENAI_INIT_CONFIG_JSON";
+var envModelKey = "MIDSCENE_MODEL_NAME";
+var envSmithDebug = "MIDSCENE_LANGSMITH_DEBUG";
+var extraConfig = {};
+if (typeof process.env[envConfigKey] === "string") {
+  console.log("will use env config for openai");
+  extraConfig = JSON.parse(process.env[envConfigKey]);
+}
+var model = "gpt-4o";
+if (typeof process.env[envModelKey] === "string") {
+  console.log(`will use model: ${process.env[envModelKey]}`);
+  model = process.env[envModelKey];
+}
+async function createOpenAI() {
+  const openai = new OpenAI(extraConfig);
+  if (process.env[envSmithDebug]) {
+    console.log("DEBUGGING MODE: using langsmith wrapper");
+    const openai2 = wrapper.wrapOpenAI(new OpenAI());
+    return openai2;
+  }
+  return openai;
+}
+async function call(messages, responseFormat) {
+  const openai = await createOpenAI();
+  const completion = await openai.chat.completions.create({
+    model,
+    messages,
+    response_format: { type: responseFormat }
+  });
+  const { content } = completion.choices[0].message;
+  assert(content, "empty content");
+  return content;
+}
+async function callToGetJSONObject(messages) {
+  const response = await call(messages, "json_object" /* JSON */);
+  assert(response, "empty response");
+  return JSON.parse(response);
+}
+
+// src/ai-model/prompt/element_inspector.ts
+function systemPromptToFindElement(description, multi) {
+  return `
+## Role:
+You are an expert in software page image (2D) and page element text analysis.
+
+## Objective:
+- Identify elements in screenshots and text that match the user's description.
+- Return JSON data containing the selection reason and element ID.
+
+## Skills:
+- Image analysis and recognition
+- Multilingual text understanding
+- Software UI design and testing
+
+## Workflow:
+1. Receive the user's element description, screenshot, and element description information. Note that the text may contain non-English characters (e.g., Chinese), indicating that the application may be non-English.
+2. Based on the description (${description}), locate the target element ID in the list of element descriptions and the screenshot.
+3. Return the number of elements: ${multi ? "multiple elements matching the description (two or more)" : "The element closest to the description (only one)"}.
+4. Return JSON data containing the selection reason and element ID.
+
+## Constraints:
+- Strictly adhere to the specified location when describing the required element; do not select elements from other locations.
+- Elements in the image with NodeType other than "TEXT Node" have been highlighted to identify the element among multiple non-text elements.
+- Accurately identify element information based on the user's description and return the corresponding element ID from the element description information, not extracted from the image.
+- If no elements are found, the "elements" array should be empty.
+- The returned data must conform to the specified JSON format.
+
+## Output Format:
+\`\`\`json
+{
+  "elements": [
+    // If no matching elements are found, return an empty array []
+    {
+      "reason": "xxx", // The thought process for finding the element, replace xxx with your thought process
+      "text": "xxx", // Replace xxx with the text of elementInfo, if none, leave empty
+      "id": "xxx" // Replace xxx with the ID of elementInfo
+    }
+    // More elements...
+  ],
+  "errors": [] // Array of strings containing any error messages
+}
+\`\`\`
+
+## Example:
+Example 1:
+Input Example:
+\`\`\`json
+// Description: "Shopping cart icon in the upper right corner"
+{
+  "screenshot": "path/screenshot.png",
+  "text": '{
+      "pageSize": {
+        "width": 400, // Width of the page
+        "height": 905 // Height of the page
+      },
+      "elementInfos": [
+        {
+          "id": "3", // ID of the element
+          "attributes": { // Attributes of the element
+            "nodeType": "IMG Node", // Type of element, types include: TEXT Node, IMG Node, BUTTON Node, INPUT Node
+            "src": "https://ap-southeast-3.m",
+            "class": ".img"
+          },
+          "content": "", // Text content of the element
+          "rect": {
+            "left": 280, // Distance from the left side of the page
+            "top": 8, // Distance from the top of the page
+            "width": 44, // Width of the element
+            "height": 44 // Height of the element
+          }
+        },
+        {
+          "id": "4", // ID of the element
+          "attributes": { // Attributes of the element
+            "nodeType": "IMG Node", // Type of element, types include: TEXT Node, IMG Node, BUTTON Node, INPUT Node
+            "src": "data:image/png;base64,iVBORw0KGgoAAAANSU...",
+            "class": ".icon"
+          },
+          "content": "", // Text content of the element
+          "rect": {
+            "left": 350, // Distance from the left side of the page
+            "top": 16, // Distance from the top of the page
+            "width": 25, // Width of the element
+            "height": 25 // Height of the element
+          }
+        },
+        ...
+        {
+          "id": "27",
+          "attributes": {
+            "nodeType": "TEXT Node",
+            "class": ".product-name"
+          },
+          "center": [
+            288,
+            834
+          ],
+          "content": "Mango Drink",
+          "rect": {
+            "left": 188,
+            "top": 827,
+            "width": 199,
+            "height": 13
+          }
+        },
+        ...
+      ]
+    }
+  '
+}
+\`\`\`
+Output Example:
+\`\`\`json
+{
+  "elements": [
+    {
+      // Describe the reason for finding this element, replace with actual value in practice
+      "reason": "Reason for finding element 4: It is located in the upper right corner, is an image type, and according to the screenshot, it is a shopping cart icon button",
+      "text": "",
+      // ID of this element, replace with actual value in practice
+      "id": "4"
+    }
+  ],
+  "errors": []
+}
+\`\`\`
+  
+  `;
+}
+
+// src/ai-model/prompt/util.ts
+import assert4 from "assert";
+
+// src/image/info.ts
+import assert2 from "assert";
+import { Buffer as Buffer2 } from "buffer";
+import { readFileSync } from "fs";
+import Sharp from "sharp";
+async function imageInfo(image) {
+  const { width, height } = await Sharp(image).metadata();
+  assert2(width && height, `invalid image: ${image}`);
+  return { width, height };
+}
+async function imageInfoOfBase64(imageBase64) {
+  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
+  return imageInfo(Buffer2.from(base64Data, "base64"));
+}
+
+// src/image/transform.ts
+import { Buffer as Buffer3 } from "buffer";
+import Sharp2 from "sharp";
+
+// src/image/visualization.ts
+import { Buffer as Buffer4 } from "buffer";
+import Sharp3 from "sharp";
+
+// src/utils.ts
+import { tmpdir } from "os";
+import { basename, join } from "path";
+import { copyFileSync, existsSync, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
+import { randomUUID } from "crypto";
+import assert3 from "assert";
+var logDir = join(process.cwd(), "./midscene_run/");
+
+// src/ai-model/prompt/util.ts
+var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
+var contextFormatIntro = `
+The user will give you a screenshot and the texts on it. There may be some none-English characters (like Chinese) on it, indicating it's an non-English app.`;
+var ONE_ELEMENT_LOCATOR_PREFIX = "LOCATE_ONE_ELEMENT";
+var ELEMENTS_LOCATOR_PREFIX = "LOCATE_ONE_OR_MORE_ELEMENTS";
+var skillSegment = `skill name: segment_a_web_page 
+skill content:
+Based on the functions and content of various elements on the page, segment the screenshot into different sections like navigation bar, product list, news area, etc. 
+Some general rules for segmentation:
+* Each section should NOT overlap with each other.
+* Each text should only belong to one section.
+* [IMPORTANT] Whether the content visually appears to belong to different sections is a significant factor in segmenting the page.
+* Analyze the page in a top-to-bottom and left-to-right order.
+* The evidence indicates a separate section, for example 
+  - The background color of certain parts of the page changes.
+  - A section of a page includes a title.
+* Provide the following data for each of the UI section you found.
+  {
+    "name": "name of the section",
+    "description": "briefly summarize the key content or usage of this section.",
+    "sectionCharacteristics": "In view of the need to distinguish this section from the surrounding sections, explain the characteristics and how to define boundaries and what precautions to take.",
+    "textIds": ["5", "6", "7"], // ids of all text elements in this section
+  }
+`;
+var skillExtractData = `skill name: extract_data_from_UI
+related input: DATA_DEMAND
+skill content: 
+* User will give you some data requirements in DATA_DEMAND. Consider the UI context, follow the user's instructions, and provide comprehensive data accordingly.
+* There may be some special commands in DATA_DEMAND, please pay extra attention
+  - ${ONE_ELEMENT_LOCATOR_PREFIX} and ${ELEMENTS_LOCATOR_PREFIX}: if you see a description that mentions the keyword ${ONE_ELEMENT_LOCATOR_PREFIX} or ${ELEMENTS_LOCATOR_PREFIX}(e.g. follow ${ONE_ELEMENT_LOCATOR_PREFIX} : i want to find ...), it means user wants to locate a specific element meets the description. Return in this way: prefix + the id / comma-separated ids, for example: ${ONE_ELEMENT_LOCATOR_PREFIX}/1 , ${ELEMENTS_LOCATOR_PREFIX}/1,2,3 . If not found, keep the prefix and leave the suffix empty, like ${ONE_ELEMENT_LOCATOR_PREFIX}/ .`;
+function promptsOfSectionQuery(constraints) {
+  if (!constraints.length) {
+    return "";
+  }
+  const instruction = "Use your segment_a_web_page skill to find the following section(s)";
+  const singleSection = (c) => {
+    assert4(
+      c.name || c.description,
+      "either `name` or `description` is required to define a section constraint"
+    );
+    const number = "One section";
+    const name = c.name ? `named \`${c.name}\`` : "";
+    const description = c.description ? `, usage or criteria : ${c.description}` : "";
+    const basic = `* ${number} ${name}${description}`;
+    return basic;
+  };
+  return `${instruction}
+${constraints.map(singleSection).join("\n")}`;
+}
+function systemPromptToExtract(dataQuery, sections) {
+  const allSectionNames = (sections == null ? void 0 : sections.filter((c) => c.name).map((c) => c.name || "")) || [];
+  const sectionFindingPrompt = promptsOfSectionQuery(sections || []);
+  const sectionReturnFormat = allSectionNames.length ? "  sections: [], // detailed information of each section from segment_a_web_page skill" : "";
+  return `
+${characteristic}
+${contextFormatIntro}
+
+You have the following skills:
+${allSectionNames.length ? skillSegment : ""}
+${skillExtractData}
+
+Now, do the following jobs:
+${sectionFindingPrompt}
+Use your extract_data_from_UI skill to find the following data, placing it in the \`data\` field
+DATA_DEMAND start:
+${typeof dataQuery === "object" ? `return in key-value style object, keys are ${Object.keys(dataQuery).join(",")}` : ""};
+${typeof dataQuery === "string" ? dataQuery : JSON.stringify(dataQuery, null, 2)}
+DATA_DEMAND ends.
+
+Return in the following JSON format:
+{
+  language: "en", // "en" or "zh", the language of the page. Use the same language to describe section name, description, and similar fields.
+  ${sectionReturnFormat}
+  data: any, // the extracted data from extract_data_from_UI skill. Make sure both the value and scheme meet the DATA_DEMAND.
+  errors?: [], // string[], error message if any
+}
+`;
+}
+function describeSize(size) {
+  return `${size.width} x ${size.height}`;
+}
+function truncateText(text) {
+  const maxLength = 50;
+  if (text && text.length > maxLength) {
+    return `${text.slice(0, maxLength)}...`;
+  }
+  return text;
+}
+async function describeUserPage(context) {
+  const { screenshotBase64 } = context;
+  const { width, height } = await imageInfoOfBase64(screenshotBase64);
+  const elementsInfo = context.content;
+  const idElementMap = {};
+  elementsInfo.forEach((item) => {
+    idElementMap[item.id] = item;
+    return __spreadValues({}, item);
+  });
+  const elementInfosDescription = cropfieldInformation(elementsInfo);
+  return {
+    description: `
+    {
+      // The size of the page
+      "pageSize": ${describeSize({ width, height })},
+
+      
+      // json description of the element
+      "elementInfos": ${JSON.stringify(elementInfosDescription)}
+    }`,
+    elementById(id) {
+      assert4(typeof id !== "undefined", "id is required for query");
+      const item = idElementMap[`${id}`];
+      return item;
+    }
+  };
+}
+function cropfieldInformation(elementsInfo) {
+  const elementInfosDescription = elementsInfo.map((item) => {
+    const { id, attributes = {}, rect, content } = item;
+    const tailorContent = truncateText(content);
+    const tailorAttributes = Object.keys(attributes).reduce((res, currentKey) => {
+      const attributeVal = attributes[currentKey];
+      res[currentKey] = truncateText(attributeVal);
+      return res;
+    }, {});
+    return {
+      id,
+      attributes: tailorAttributes,
+      rect,
+      content: tailorContent
+    };
+  });
+  return JSON.stringify(elementInfosDescription);
+}
+
+// src/ai-model/inspect.ts
+async function AiInspectElement(options) {
+  const { context, multi, findElementDescription, callAI = callToGetJSONObject } = options;
+  const { screenshotBase64 } = context;
+  const { description, elementById } = await describeUserPage(context);
+  const systemPrompt = systemPromptToFindElement(findElementDescription, multi);
+  const msgs = [
+    { role: "system", content: systemPrompt },
+    {
+      role: "user",
+      content: [
+        {
+          type: "image_url",
+          image_url: {
+            url: screenshotBase64,
+            detail: "high"
+          }
+        },
+        {
+          type: "text",
+          text: description
+        }
+      ]
+    }
+  ];
+  const parseResult = await callAI(msgs);
+  return {
+    parseResult,
+    elementById,
+    systemPrompt
+  };
+}
+async function AiExtractElementInfo(options) {
+  const { dataQuery, sectionConstraints, context, callAI = callToGetJSONObject } = options;
+  const systemPrompt = systemPromptToExtract(dataQuery, sectionConstraints);
+  const { screenshotBase64 } = context;
+  const { description, elementById } = await describeUserPage(context);
+  const msgs = [
+    { role: "system", content: systemPrompt },
+    {
+      role: "user",
+      content: [
+        {
+          type: "image_url",
+          image_url: {
+            url: screenshotBase64
+          }
+        },
+        {
+          type: "text",
+          text: description
+        }
+      ]
+    }
+  ];
+  const parseResult = await callAI(msgs);
+  return {
+    parseResult,
+    elementById,
+    systemPrompt
+  };
+}
+export {
+  AiExtractElementInfo,
+  AiInspectElement,
+  callToGetJSONObject,
+  describeUserPage,
+  systemPromptToFindElement
+};

package/dist/es/image.js

@@ -0,0 +1,261 @@
+// src/image/info.ts
+import assert from "assert";
+import { Buffer } from "buffer";
+import { readFileSync } from "fs";
+import Sharp from "sharp";
+async function imageInfo(image) {
+  const { width, height } = await Sharp(image).metadata();
+  assert(width && height, `invalid image: ${image}`);
+  return { width, height };
+}
+async function imageInfoOfBase64(imageBase64) {
+  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
+  return imageInfo(Buffer.from(base64Data, "base64"));
+}
+function base64Encoded(image, withHeader = true) {
+  const imageBuffer = readFileSync(image);
+  if (!withHeader) {
+    return imageBuffer.toString("base64");
+  }
+  if (image.endsWith("png")) {
+    return `data:image/png;base64,${imageBuffer.toString("base64")}`;
+  } else if (image.endsWith("jpg") || image.endsWith("jpeg")) {
+    return `data:image/jpeg;base64,${imageBuffer.toString("base64")}`;
+  }
+  throw new Error("unsupported image type");
+}
+
+// src/image/transform.ts
+import { Buffer as Buffer2 } from "buffer";
+import Sharp2 from "sharp";
+async function saveBase64Image(options) {
+  const { base64Data, outputPath } = options;
+  const base64Image = base64Data.split(";base64,").pop() || base64Data;
+  const imageBuffer = Buffer2.from(base64Image, "base64");
+  await Sharp2(imageBuffer).toFile(outputPath);
+  console.log("Image successfully written to file.");
+}
+async function transformImgPathToBase64(inputPath) {
+  return await Sharp2(inputPath).toBuffer().then((data) => {
+    const base64Data = data.toString("base64");
+    return base64Data;
+  });
+}
+async function resizeImg(base64Data) {
+  const base64Image = base64Data.split(";base64,").pop() || base64Data;
+  const imageBuffer = Buffer2.from(base64Image, "base64");
+  const metadata = await Sharp2(imageBuffer).metadata();
+  const { width, height } = metadata;
+  if (!width || !height) {
+    throw Error("undefined width or height with url");
+  }
+  const newSize = calculateNewDimensions(width, height);
+  return await Sharp2(imageBuffer).resize(newSize.width, newSize.height).toBuffer().then((data) => {
+    const base64Data2 = data.toString("base64");
+    return base64Data2;
+  });
+}
+function calculateNewDimensions(originalWidth, originalHeight) {
+  const maxWidth = 768;
+  const maxHeight = 2048;
+  let newWidth = originalWidth;
+  let newHeight = originalHeight;
+  const aspectRatio = originalWidth / originalHeight;
+  if (originalWidth > maxWidth) {
+    newWidth = maxWidth;
+    newHeight = newWidth / aspectRatio;
+  }
+  if (newHeight > maxHeight) {
+    newHeight = maxHeight;
+    newWidth = newHeight * aspectRatio;
+  }
+  return {
+    width: Math.round(newWidth),
+    height: Math.round(newHeight)
+  };
+}
+async function trimImage(image) {
+  const { info } = await Sharp2(image).trim().toBuffer({
+    resolveWithObject: true
+  });
+  if (typeof info.trimOffsetLeft === "undefined" || typeof info.trimOffsetTop === "undefined") {
+    return null;
+  }
+  return {
+    trimOffsetLeft: info.trimOffsetLeft,
+    trimOffsetTop: info.trimOffsetTop,
+    width: info.width,
+    height: info.height
+  };
+}
+async function alignCoordByTrim(image, center) {
+  const img = await Sharp2(image).extract(center).toBuffer();
+  const trimInfo = await trimImage(img);
+  if (!trimInfo) {
+    return center;
+  }
+  return {
+    left: center.left - trimInfo.trimOffsetLeft,
+    top: center.top - trimInfo.trimOffsetTop,
+    width: trimInfo.width,
+    height: trimInfo.height
+  };
+}
+
+// src/image/visualization.ts
+import { Buffer as Buffer3 } from "buffer";
+import Sharp3 from "sharp";
+
+// src/utils.ts
+import { tmpdir } from "os";
+import { basename, join } from "path";
+import { copyFileSync, existsSync, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
+import { randomUUID } from "crypto";
+import assert2 from "assert";
+var pkg;
+function getPkgInfo() {
+  if (pkg) {
+    return pkg;
+  }
+  let pkgJsonFile = "";
+  if (existsSync(join(__dirname, "../package.json"))) {
+    pkgJsonFile = join(__dirname, "../package.json");
+  } else if (existsSync(join(__dirname, "../../../package.json"))) {
+    pkgJsonFile = join(__dirname, "../../../package.json");
+  }
+  if (pkgJsonFile) {
+    const { name, version } = JSON.parse(readFileSync2(pkgJsonFile, "utf-8"));
+    pkg = { name, version };
+    return pkg;
+  } else {
+    return {
+      name: "midscene-unknown-page-name",
+      version: "0.0.0"
+    };
+  }
+}
+var logDir = join(process.cwd(), "./midscene_run/");
+function getTmpDir() {
+  const path = join(tmpdir(), getPkgInfo().name);
+  mkdirSync(path, { recursive: true });
+  return path;
+}
+function getTmpFile(fileExt) {
+  const filename = `${randomUUID()}.${fileExt}`;
+  return join(getTmpDir(), filename);
+}
+
+// src/image/visualization.ts
+var colors = [
+  {
+    name: "Red",
+    hex: "#FF0000"
+  },
+  {
+    name: "Green",
+    hex: "#00FF00"
+  },
+  {
+    name: "Blue",
+    hex: "#0000FF"
+  },
+  {
+    name: "Yellow",
+    hex: "#FFFF00"
+  },
+  {
+    name: "Cyan",
+    hex: "#00FFFF"
+  },
+  {
+    name: "Magenta",
+    hex: "#FF00FF"
+  },
+  {
+    name: "Orange",
+    hex: "#FFA500"
+  },
+  {
+    name: "Purple",
+    hex: "#800080"
+  },
+  {
+    name: "Brown",
+    hex: "#A52A2A"
+  },
+  {
+    name: "Pink",
+    hex: "#FFC0CB"
+  },
+  {
+    name: "Light Blue",
+    hex: "#ADD8E6"
+  },
+  {
+    name: "Lime",
+    hex: "#00FF00"
+  },
+  {
+    name: "Violet",
+    hex: "#EE82EE"
+  },
+  {
+    name: "Gold",
+    hex: "#FFD700"
+  },
+  {
+    name: "Teal",
+    hex: "#008080"
+  }
+];
+var sizeLimit = 512;
+var textFontSize = 12;
+async function composeSectionDiagram(sections, context) {
+  const { width, height } = await imageInfo(context.screenshotBase64);
+  const ratio = Math.min(sizeLimit / width, sizeLimit / height, 1);
+  const canvasWidth = width * ratio;
+  const canvasHeight = height * ratio;
+  const sectionNameColorMap = {};
+  const rects = sections.map((section, index) => {
+    const { left, top, width: width2, height: height2 } = section.rect;
+    const color = colors[index % colors.length];
+    sectionNameColorMap[section.name] = color;
+    return `
+            <rect x="${left * ratio}" y="${top * ratio}" width="${width2 * ratio}" height="${height2 * ratio}" fill="${color.hex}" />
+            <text x="${left * ratio}" y="${top * ratio + textFontSize}" font-family="Arial" font-size="${textFontSize}" fill="black">
+                ${section.name}
+            </text>
+        `;
+  });
+  const rectangles = `
+        <svg width="${canvasWidth}" height="${canvasHeight}">
+        ${rects.join("\n")}
+        </svg>
+    `;
+  const svgBuffer = Buffer3.from(rectangles);
+  const file = getTmpFile("png");
+  await Sharp3({
+    create: {
+      width: canvasWidth,
+      height: canvasHeight,
+      channels: 4,
+      background: { r: 255, g: 255, b: 255, alpha: 1 }
+    }
+  }).composite([{ input: svgBuffer }]).png().toFile(file);
+  return {
+    file,
+    sectionNameColorMap
+  };
+}
+export {
+  alignCoordByTrim,
+  base64Encoded,
+  calculateNewDimensions,
+  composeSectionDiagram,
+  imageInfo,
+  imageInfoOfBase64,
+  resizeImg,
+  saveBase64Image,
+  transformImgPathToBase64,
+  trimImage
+};