@midscene/core 0.1.4 → 0.2.0

package/dist/es/utils.js

@@ -1,9 +1,15 @@
 // src/utils.ts
+import assert from "assert";
+import { randomUUID } from "crypto";
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync
+} from "fs";
 import { tmpdir } from "os";
 import { basename, join } from "path";
-import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { randomUUID } from "crypto";
-import assert from "assert";
 var pkg;
 function getPkgInfo() {
   if (pkg) {
@@ -19,12 +25,11 @@ function getPkgInfo() {
     const { name, version } = JSON.parse(readFileSync(pkgJsonFile, "utf-8"));
     pkg = { name, version };
     return pkg;
-  } else {
-    return {
-      name: "midscene-unknown-page-name",
-      version: "0.0.0"
-    };
   }
+  return {
+    name: "midscene-unknown-page-name",
+    version: "0.0.0"
+  };
 }
 var logDir = join(process.cwd(), "./midscene_run/");
 var logEnvReady = false;
@@ -57,7 +62,7 @@ function writeDumpFile(opts) {
       writeFileSync(
         gitIgnorePath,
         `${gitIgnoreContent}
-# MidScene.js dump files
+# Midscene.js dump files
 ${logDirName}/report
 ${logDirName}/dump-logger
 `,

package/dist/lib/ai-model.js

@@ -1022,8 +1022,7 @@ __export(ai_model_exports, {
 module.exports = __toCommonJS(ai_model_exports);
 
 // src/ai-model/openai.ts
-var import_assert = __toESM(require("assert"));
-var import_openai = __toESM(require("openai"));
+var import_node_assert = __toESM(require("assert"));
 
 // ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.47.1/node_modules/langsmith/dist/traceable.js
 var import_node_async_hooks = require("async_hooks");
@@ -4238,11 +4237,12 @@ var wrapOpenAI = (openai, options) => {
 };
 
 // src/ai-model/openai.ts
+var import_openai = __toESM(require("openai"));
 var envConfigKey = "MIDSCENE_OPENAI_INIT_CONFIG_JSON";
 var envModelKey = "MIDSCENE_MODEL_NAME";
 var envSmithDebug = "MIDSCENE_LANGSMITH_DEBUG";
 var extraConfig = {};
-if (typeof process.env[envConfigKey] === "string") {
+if (typeof process.env[envConfigKey] === "string" && process.env[envConfigKey]) {
   console.log("config for openai loaded");
   extraConfig = JSON.parse(process.env[envConfigKey]);
 }
@@ -4268,12 +4268,12 @@ async function call(messages, responseFormat) {
     response_format: { type: responseFormat }
   });
   const { content } = completion.choices[0].message;
-  (0, import_assert.default)(content, "empty content");
+  (0, import_node_assert.default)(content, "empty content");
   return content;
 }
 async function callToGetJSONObject(messages) {
   const response = await call(messages, "json_object" /* JSON */);
-  (0, import_assert.default)(response, "empty response");
+  (0, import_node_assert.default)(response, "empty response");
   return JSON.parse(response);
 }
 
@@ -4311,9 +4311,9 @@ You are an expert in software page image (2D) and page element text analysis.
   "elements": [
     // If no matching elements are found, return an empty array []
     {
-      "reason": "xxx", // The thought process for finding the element, replace xxx with your thought process
-      "text": "xxx", // Replace xxx with the text of elementInfo, if none, leave empty
-      "id": "xxx" // Replace xxx with the ID of elementInfo
+      "reason": "PLACEHOLDER", // The thought process for finding the element, replace PLACEHOLDER with your thought process
+      "text": "PLACEHOLDER", // Replace PLACEHOLDER with the text of elementInfo, if none, leave empty
+      "id": "PLACEHOLDER" // Replace PLACEHOLDER with the ID of elementInfo
     }
     // More elements...
   ],
@@ -4409,16 +4409,16 @@ Output Example:
 }
 
 // src/ai-model/prompt/util.ts
-var import_assert3 = __toESM(require("assert"));
+var import_node_assert4 = __toESM(require("assert"));
 
 // src/image/info.ts
-var import_node_assert = __toESM(require("assert"));
+var import_node_assert2 = __toESM(require("assert"));
 var import_node_buffer = require("buffer");
 var import_node_fs = require("fs");
 var import_sharp = __toESM(require("sharp"));
 async function imageInfo(image) {
   const { width, height } = await (0, import_sharp.default)(image).metadata();
-  (0, import_node_assert.default)(width && height, `invalid image: ${image}`);
+  (0, import_node_assert2.default)(width && height, `invalid image: ${image}`);
   return { width, height };
 }
 async function imageInfoOfBase64(imageBase64) {
@@ -4431,16 +4431,18 @@ var import_node_buffer2 = require("buffer");
 var import_sharp2 = __toESM(require("sharp"));
 
 // src/image/visualization.ts
-var import_buffer = require("buffer");
-var import_sharp3 = __toESM(require("sharp"));
+var import_node_buffer3 = require("buffer");
 
 // src/utils.ts
-var import_os = require("os");
-var import_path = require("path");
-var import_fs = require("fs");
-var import_crypto3 = require("crypto");
-var import_assert2 = __toESM(require("assert"));
-var logDir = (0, import_path.join)(process.cwd(), "./midscene_run/");
+var import_node_assert3 = __toESM(require("assert"));
+var import_node_crypto = require("crypto");
+var import_node_fs2 = require("fs");
+var import_node_os = require("os");
+var import_node_path = require("path");
+var logDir = (0, import_node_path.join)(process.cwd(), "./midscene_run/");
+
+// src/image/visualization.ts
+var import_sharp3 = __toESM(require("sharp"));
 
 // src/ai-model/prompt/util.ts
 var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
@@ -4479,7 +4481,7 @@ function promptsOfSectionQuery(constraints) {
   }
   const instruction = "Use your segment_a_web_page skill to find the following section(s)";
   const singleSection = (c) => {
-    (0, import_assert3.default)(
+    (0, import_node_assert4.default)(
       c.name || c.description,
       "either `name` or `description` is required to define a section constraint"
     );
@@ -4559,34 +4561,45 @@ async function describeUserPage(context) {
       "elementInfos": ${JSON.stringify(elementInfosDescription)}
     }`,
     elementById(id) {
-      (0, import_assert3.default)(typeof id !== "undefined", "id is required for query");
+      (0, import_node_assert4.default)(typeof id !== "undefined", "id is required for query");
       const item = idElementMap[`${id}`];
       return item;
     }
   };
 }
 function cropfieldInformation(elementsInfo) {
-  const elementInfosDescription = elementsInfo.map((item) => {
-    const { id, attributes = {}, rect, content } = item;
-    const tailorContent = truncateText(content);
-    const tailorAttributes = Object.keys(attributes).reduce((res, currentKey) => {
-      const attributeVal = attributes[currentKey];
-      res[currentKey] = truncateText(attributeVal);
-      return res;
-    }, {});
-    return {
-      id,
-      attributes: tailorAttributes,
-      rect,
-      content: tailorContent
-    };
-  });
+  const elementInfosDescription = elementsInfo.map(
+    (item) => {
+      const { id, attributes = {}, rect, content } = item;
+      const tailorContent = truncateText(content);
+      const tailorAttributes = Object.keys(attributes).reduce(
+        (res, currentKey) => {
+          const attributeVal = attributes[currentKey];
+          res[currentKey] = truncateText(attributeVal);
+          return res;
+        },
+        {}
+      );
+      return {
+        id,
+        attributes: tailorAttributes,
+        rect,
+        content: tailorContent
+      };
+    }
+  );
   return JSON.stringify(elementInfosDescription);
 }
 
 // src/ai-model/inspect.ts
+var import_node_assert5 = __toESM(require("assert"));
 async function AiInspectElement(options) {
-  const { context, multi, findElementDescription, callAI = callToGetJSONObject } = options;
+  const {
+    context,
+    multi,
+    findElementDescription,
+    callAI = callToGetJSONObject
+  } = options;
   const { screenshotBase64 } = context;
   const { description, elementById } = await describeUserPage(context);
   const systemPrompt = systemPromptToFindElement(findElementDescription, multi);
@@ -4612,12 +4625,16 @@ async function AiInspectElement(options) {
   const parseResult = await callAI(msgs);
   return {
     parseResult,
-    elementById,
-    systemPrompt
+    elementById
   };
 }
 async function AiExtractElementInfo(options) {
-  const { dataQuery, sectionConstraints, context, callAI = callToGetJSONObject } = options;
+  const {
+    dataQuery,
+    sectionConstraints,
+    context,
+    callAI = callToGetJSONObject
+  } = options;
   const systemPrompt = systemPromptToExtract(dataQuery, sectionConstraints);
   const { screenshotBase64 } = context;
   const { description, elementById } = await describeUserPage(context);
@@ -4642,8 +4659,7 @@ async function AiExtractElementInfo(options) {
   const parseResult = await callAI(msgs);
   return {
     parseResult,
-    elementById,
-    systemPrompt
+    elementById
   };
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/lib/image.js

@@ -64,7 +64,8 @@ function base64Encoded(image, withHeader = true) {
   }
   if (image.endsWith("png")) {
     return `data:image/png;base64,${imageBuffer.toString("base64")}`;
-  } else if (image.endsWith("jpg") || image.endsWith("jpeg")) {
+  }
+  if (image.endsWith("jpg") || image.endsWith("jpeg")) {
     return `data:image/jpeg;base64,${imageBuffer.toString("base64")}`;
   }
   throw new Error("unsupported image type");
@@ -162,49 +163,48 @@ async function alignCoordByTrim(image, centerRect) {
 }
 
 // src/image/visualization.ts
-var import_buffer = require("buffer");
-var import_sharp3 = __toESM(require("sharp"));
+var import_node_buffer3 = require("buffer");
 
 // src/utils.ts
-var import_os = require("os");
-var import_path = require("path");
-var import_fs = require("fs");
-var import_crypto = require("crypto");
-var import_assert = __toESM(require("assert"));
+var import_node_assert2 = __toESM(require("assert"));
+var import_node_crypto = require("crypto");
+var import_node_fs2 = require("fs");
+var import_node_os = require("os");
+var import_node_path = require("path");
 var pkg;
 function getPkgInfo() {
   if (pkg) {
     return pkg;
   }
   let pkgJsonFile = "";
-  if ((0, import_fs.existsSync)((0, import_path.join)(__dirname, "../package.json"))) {
-    pkgJsonFile = (0, import_path.join)(__dirname, "../package.json");
-  } else if ((0, import_fs.existsSync)((0, import_path.join)(__dirname, "../../../package.json"))) {
-    pkgJsonFile = (0, import_path.join)(__dirname, "../../../package.json");
+  if ((0, import_node_fs2.existsSync)((0, import_node_path.join)(__dirname, "../package.json"))) {
+    pkgJsonFile = (0, import_node_path.join)(__dirname, "../package.json");
+  } else if ((0, import_node_fs2.existsSync)((0, import_node_path.join)(__dirname, "../../../package.json"))) {
+    pkgJsonFile = (0, import_node_path.join)(__dirname, "../../../package.json");
   }
   if (pkgJsonFile) {
-    const { name, version } = JSON.parse((0, import_fs.readFileSync)(pkgJsonFile, "utf-8"));
+    const { name, version } = JSON.parse((0, import_node_fs2.readFileSync)(pkgJsonFile, "utf-8"));
     pkg = { name, version };
     return pkg;
-  } else {
-    return {
-      name: "midscene-unknown-page-name",
-      version: "0.0.0"
-    };
   }
+  return {
+    name: "midscene-unknown-page-name",
+    version: "0.0.0"
+  };
 }
-var logDir = (0, import_path.join)(process.cwd(), "./midscene_run/");
+var logDir = (0, import_node_path.join)(process.cwd(), "./midscene_run/");
 function getTmpDir() {
-  const path = (0, import_path.join)((0, import_os.tmpdir)(), getPkgInfo().name);
-  (0, import_fs.mkdirSync)(path, { recursive: true });
+  const path = (0, import_node_path.join)((0, import_node_os.tmpdir)(), getPkgInfo().name);
+  (0, import_node_fs2.mkdirSync)(path, { recursive: true });
   return path;
 }
 function getTmpFile(fileExtWithoutDot) {
-  const filename = `${(0, import_crypto.randomUUID)()}.${fileExtWithoutDot}`;
-  return (0, import_path.join)(getTmpDir(), filename);
+  const filename = `${(0, import_node_crypto.randomUUID)()}.${fileExtWithoutDot}`;
+  return (0, import_node_path.join)(getTmpDir(), filename);
 }
 
 // src/image/visualization.ts
+var import_sharp3 = __toESM(require("sharp"));
 var colors = [
   {
     name: "Red",
@@ -291,7 +291,7 @@ async function composeSectionDiagram(sections, context) {
         ${rects.join("\n")}
         </svg>
     `;
-  const svgBuffer = import_buffer.Buffer.from(rectangles);
+  const svgBuffer = import_node_buffer3.Buffer.from(rectangles);
   const file = getTmpFile("png");
   await (0, import_sharp3.default)({
     create: {