@midscene/core 0.1.4 → 0.2.0

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024-present MidScene.js
+Copyright (c) 2024-present Midscene.js
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -0,0 +1,7 @@
+## Documentation
+
+See https://midscenejs.com/ for details.
+
+## License
+
+Midscene is MIT licensed.

package/dist/es/ai-model.js

@@ -1006,7 +1006,6 @@ var require_dist = __commonJS({
 
 // src/ai-model/openai.ts
 import assert from "assert";
-import OpenAI from "openai";
 
 // ../../node_modules/.pnpm/langsmith@0.1.36_openai@4.47.1/node_modules/langsmith/dist/traceable.js
 import { AsyncLocalStorage } from "async_hooks";
@@ -4221,11 +4220,12 @@ var wrapOpenAI = (openai, options) => {
 };
 
 // src/ai-model/openai.ts
+import OpenAI from "openai";
 var envConfigKey = "MIDSCENE_OPENAI_INIT_CONFIG_JSON";
 var envModelKey = "MIDSCENE_MODEL_NAME";
 var envSmithDebug = "MIDSCENE_LANGSMITH_DEBUG";
 var extraConfig = {};
-if (typeof process.env[envConfigKey] === "string") {
+if (typeof process.env[envConfigKey] === "string" && process.env[envConfigKey]) {
   console.log("config for openai loaded");
   extraConfig = JSON.parse(process.env[envConfigKey]);
 }
@@ -4294,9 +4294,9 @@ You are an expert in software page image (2D) and page element text analysis.
   "elements": [
     // If no matching elements are found, return an empty array []
     {
-      "reason": "xxx", // The thought process for finding the element, replace xxx with your thought process
-      "text": "xxx", // Replace xxx with the text of elementInfo, if none, leave empty
-      "id": "xxx" // Replace xxx with the ID of elementInfo
+      "reason": "PLACEHOLDER", // The thought process for finding the element, replace PLACEHOLDER with your thought process
+      "text": "PLACEHOLDER", // Replace PLACEHOLDER with the text of elementInfo, if none, leave empty
+      "id": "PLACEHOLDER" // Replace PLACEHOLDER with the ID of elementInfo
     }
     // More elements...
   ],
@@ -4415,16 +4415,24 @@ import Sharp2 from "sharp";
 
 // src/image/visualization.ts
 import { Buffer as Buffer4 } from "buffer";
-import Sharp3 from "sharp";
 
 // src/utils.ts
+import assert3 from "assert";
+import { randomUUID } from "crypto";
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync as readFileSync2,
+  writeFileSync
+} from "fs";
 import { tmpdir } from "os";
 import { basename, join } from "path";
-import { copyFileSync, existsSync, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
-import { randomUUID } from "crypto";
-import assert3 from "assert";
 var logDir = join(process.cwd(), "./midscene_run/");
 
+// src/image/visualization.ts
+import Sharp3 from "sharp";
+
 // src/ai-model/prompt/util.ts
 var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
 var contextFormatIntro = `
@@ -4549,27 +4557,38 @@ async function describeUserPage(context) {
   };
 }
 function cropfieldInformation(elementsInfo) {
-  const elementInfosDescription = elementsInfo.map((item) => {
-    const { id, attributes = {}, rect, content } = item;
-    const tailorContent = truncateText(content);
-    const tailorAttributes = Object.keys(attributes).reduce((res, currentKey) => {
-      const attributeVal = attributes[currentKey];
-      res[currentKey] = truncateText(attributeVal);
-      return res;
-    }, {});
-    return {
-      id,
-      attributes: tailorAttributes,
-      rect,
-      content: tailorContent
-    };
-  });
+  const elementInfosDescription = elementsInfo.map(
+    (item) => {
+      const { id, attributes = {}, rect, content } = item;
+      const tailorContent = truncateText(content);
+      const tailorAttributes = Object.keys(attributes).reduce(
+        (res, currentKey) => {
+          const attributeVal = attributes[currentKey];
+          res[currentKey] = truncateText(attributeVal);
+          return res;
+        },
+        {}
+      );
+      return {
+        id,
+        attributes: tailorAttributes,
+        rect,
+        content: tailorContent
+      };
+    }
+  );
   return JSON.stringify(elementInfosDescription);
 }
 
 // src/ai-model/inspect.ts
+import assert5 from "assert";
 async function AiInspectElement(options) {
-  const { context, multi, findElementDescription, callAI = callToGetJSONObject } = options;
+  const {
+    context,
+    multi,
+    findElementDescription,
+    callAI = callToGetJSONObject
+  } = options;
   const { screenshotBase64 } = context;
   const { description, elementById } = await describeUserPage(context);
   const systemPrompt = systemPromptToFindElement(findElementDescription, multi);
@@ -4595,12 +4614,16 @@ async function AiInspectElement(options) {
   const parseResult = await callAI(msgs);
   return {
     parseResult,
-    elementById,
-    systemPrompt
+    elementById
   };
 }
 async function AiExtractElementInfo(options) {
-  const { dataQuery, sectionConstraints, context, callAI = callToGetJSONObject } = options;
+  const {
+    dataQuery,
+    sectionConstraints,
+    context,
+    callAI = callToGetJSONObject
+  } = options;
   const systemPrompt = systemPromptToExtract(dataQuery, sectionConstraints);
   const { screenshotBase64 } = context;
   const { description, elementById } = await describeUserPage(context);
@@ -4625,8 +4648,7 @@ async function AiExtractElementInfo(options) {
   const parseResult = await callAI(msgs);
   return {
     parseResult,
-    elementById,
-    systemPrompt
+    elementById
   };
 }
 export {

package/dist/es/image.js

@@ -19,7 +19,8 @@ function base64Encoded(image, withHeader = true) {
   }
   if (image.endsWith("png")) {
     return `data:image/png;base64,${imageBuffer.toString("base64")}`;
-  } else if (image.endsWith("jpg") || image.endsWith("jpeg")) {
+  }
+  if (image.endsWith("jpg") || image.endsWith("jpeg")) {
     return `data:image/jpeg;base64,${imageBuffer.toString("base64")}`;
   }
   throw new Error("unsupported image type");
@@ -118,14 +119,19 @@ async function alignCoordByTrim(image, centerRect) {
 
 // src/image/visualization.ts
 import { Buffer as Buffer3 } from "buffer";
-import Sharp3 from "sharp";
 
 // src/utils.ts
+import assert2 from "assert";
+import { randomUUID } from "crypto";
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync as readFileSync2,
+  writeFileSync
+} from "fs";
 import { tmpdir } from "os";
 import { basename, join } from "path";
-import { copyFileSync, existsSync, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
-import { randomUUID } from "crypto";
-import assert2 from "assert";
 var pkg;
 function getPkgInfo() {
   if (pkg) {
@@ -141,12 +147,11 @@ function getPkgInfo() {
     const { name, version } = JSON.parse(readFileSync2(pkgJsonFile, "utf-8"));
     pkg = { name, version };
     return pkg;
-  } else {
-    return {
-      name: "midscene-unknown-page-name",
-      version: "0.0.0"
-    };
   }
+  return {
+    name: "midscene-unknown-page-name",
+    version: "0.0.0"
+  };
 }
 var logDir = join(process.cwd(), "./midscene_run/");
 function getTmpDir() {
@@ -160,6 +165,7 @@ function getTmpFile(fileExtWithoutDot) {
 }
 
 // src/image/visualization.ts
+import Sharp3 from "sharp";
 var colors = [
   {
     name: "Red",