@midscene/core 0.3.4 → 0.4.0

package/dist/es/image.js

@@ -1,289 +1,19 @@
-// src/image/info.ts
-import assert from "assert";
-import { Buffer } from "buffer";
-import { readFileSync } from "fs";
-import Sharp from "sharp";
-async function imageInfo(image) {
-  const { width, height } = await Sharp(image).metadata();
-  assert(width && height, `invalid image: ${image}`);
-  return { width, height };
-}
-async function imageInfoOfBase64(imageBase64) {
-  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
-  return imageInfo(Buffer.from(base64Data, "base64"));
-}
-function base64Encoded(image, withHeader = true) {
-  const imageBuffer = readFileSync(image);
-  if (!withHeader) {
-    return imageBuffer.toString("base64");
-  }
-  if (image.endsWith("png")) {
-    return `data:image/png;base64,${imageBuffer.toString("base64")}`;
-  }
-  if (image.endsWith("jpg") || image.endsWith("jpeg")) {
-    return `data:image/jpeg;base64,${imageBuffer.toString("base64")}`;
-  }
-  throw new Error("unsupported image type");
-}
-
-// src/image/transform.ts
-import { Buffer as Buffer2 } from "buffer";
-import Sharp2 from "sharp";
-async function saveBase64Image(options) {
-  const { base64Data, outputPath } = options;
-  const base64Image = base64Data.split(";base64,").pop() || base64Data;
-  const imageBuffer = Buffer2.from(base64Image, "base64");
-  await Sharp2(imageBuffer).toFile(outputPath);
-  console.log("Image successfully written to file.");
-}
-async function transformImgPathToBase64(inputPath) {
-  return await Sharp2(inputPath).toBuffer().then((data) => {
-    const base64Data = data.toString("base64");
-    return base64Data;
-  });
-}
-async function resizeImg(base64Data) {
-  const base64Image = base64Data.split(";base64,").pop() || base64Data;
-  const imageBuffer = Buffer2.from(base64Image, "base64");
-  const metadata = await Sharp2(imageBuffer).metadata();
-  const { width, height } = metadata;
-  if (!width || !height) {
-    throw Error("undefined width or height with url");
-  }
-  const newSize = calculateNewDimensions(width, height);
-  return await Sharp2(imageBuffer).resize(newSize.width, newSize.height).toBuffer().then((data) => {
-    const base64Data2 = data.toString("base64");
-    return base64Data2;
-  });
-}
-function calculateNewDimensions(originalWidth, originalHeight) {
-  const maxWidth = 768;
-  const maxHeight = 2048;
-  let newWidth = originalWidth;
-  let newHeight = originalHeight;
-  const aspectRatio = originalWidth / originalHeight;
-  if (originalWidth > maxWidth) {
-    newWidth = maxWidth;
-    newHeight = newWidth / aspectRatio;
-  }
-  if (newHeight > maxHeight) {
-    newHeight = maxHeight;
-    newWidth = newHeight * aspectRatio;
-  }
-  return {
-    width: Math.round(newWidth),
-    height: Math.round(newHeight)
-  };
-}
-async function trimImage(image) {
-  const imgInstance = Sharp2(image);
-  const instanceInfo = await imgInstance.metadata();
-  if (!instanceInfo.width || instanceInfo.width <= 3 || !instanceInfo.height || instanceInfo.height <= 3) {
-    return null;
-  }
-  const { info } = await imgInstance.trim().toBuffer({
-    resolveWithObject: true
-  });
-  if (typeof info.trimOffsetLeft === "undefined" || typeof info.trimOffsetTop === "undefined") {
-    return null;
-  }
-  return {
-    trimOffsetLeft: info.trimOffsetLeft,
-    trimOffsetTop: info.trimOffsetTop,
-    width: info.width,
-    height: info.height
-  };
-}
-async function alignCoordByTrim(image, centerRect) {
-  const imgInfo = await Sharp2(image).metadata();
-  if (!(imgInfo == null ? void 0 : imgInfo.width) || !imgInfo.height || imgInfo.width <= 3 || imgInfo.height <= 3) {
-    return centerRect;
-  }
-  try {
-    const img = await Sharp2(image).extract(centerRect).toBuffer();
-    const trimInfo = await trimImage(img);
-    if (!trimInfo) {
-      return centerRect;
-    }
-    return {
-      left: centerRect.left - trimInfo.trimOffsetLeft,
-      top: centerRect.top - trimInfo.trimOffsetTop,
-      width: trimInfo.width,
-      height: trimInfo.height
-    };
-  } catch (e) {
-    console.log(imgInfo);
-    throw e;
-  }
-}
-
-// src/image/visualization.ts
-import { Buffer as Buffer3 } from "buffer";
-
-// src/utils.ts
-import assert2 from "assert";
-import { randomUUID } from "crypto";
+// src/image/index.ts
 import {
-  existsSync,
-  mkdirSync,
-  readFileSync as readFileSync2,
-  writeFileSync
-} from "fs";
-import { tmpdir } from "os";
-import path, { basename, dirname, join } from "path";
-var pkg;
-function getPkgInfo() {
-  if (pkg) {
-    return pkg;
-  }
-  const pkgDir = findNearestPackageJson(__dirname);
-  assert2(pkgDir, "package.json not found");
-  const pkgJsonFile = join(pkgDir, "package.json");
-  if (pkgJsonFile) {
-    const { name, version } = JSON.parse(readFileSync2(pkgJsonFile, "utf-8"));
-    pkg = { name, version, dir: pkgDir };
-    return pkg;
-  }
-  return {
-    name: "midscene-unknown-page-name",
-    version: "0.0.0",
-    dir: pkgDir
-  };
-}
-var logDir = join(process.cwd(), "./midscene_run/");
-function getTmpDir() {
-  const path2 = join(tmpdir(), getPkgInfo().name);
-  mkdirSync(path2, { recursive: true });
-  return path2;
-}
-function getTmpFile(fileExtWithoutDot) {
-  const filename = `${randomUUID()}.${fileExtWithoutDot}`;
-  return join(getTmpDir(), filename);
-}
-function findNearestPackageJson(dir) {
-  const packageJsonPath = path.join(dir, "package.json");
-  if (existsSync(packageJsonPath)) {
-    return dir;
-  }
-  const parentDir = path.dirname(dir);
-  if (parentDir === dir) {
-    return null;
-  }
-  return findNearestPackageJson(parentDir);
-}
-
-// src/image/visualization.ts
-import Sharp3 from "sharp";
-var colors = [
-  {
-    name: "Red",
-    hex: "#FF0000"
-  },
-  {
-    name: "Green",
-    hex: "#00FF00"
-  },
-  {
-    name: "Blue",
-    hex: "#0000FF"
-  },
-  {
-    name: "Yellow",
-    hex: "#FFFF00"
-  },
-  {
-    name: "Cyan",
-    hex: "#00FFFF"
-  },
-  {
-    name: "Magenta",
-    hex: "#FF00FF"
-  },
-  {
-    name: "Orange",
-    hex: "#FFA500"
-  },
-  {
-    name: "Purple",
-    hex: "#800080"
-  },
-  {
-    name: "Brown",
-    hex: "#A52A2A"
-  },
-  {
-    name: "Pink",
-    hex: "#FFC0CB"
-  },
-  {
-    name: "Light Blue",
-    hex: "#ADD8E6"
-  },
-  {
-    name: "Lime",
-    hex: "#00FF00"
-  },
-  {
-    name: "Violet",
-    hex: "#EE82EE"
-  },
-  {
-    name: "Gold",
-    hex: "#FFD700"
-  },
-  {
-    name: "Teal",
-    hex: "#008080"
-  }
-];
-var sizeLimit = 512;
-var textFontSize = 12;
-async function composeSectionDiagram(sections, context) {
-  const { width, height } = await imageInfo(context.screenshotBase64);
-  const ratio = Math.min(sizeLimit / width, sizeLimit / height, 1);
-  const canvasWidth = width * ratio;
-  const canvasHeight = height * ratio;
-  const sectionNameColorMap = {};
-  const rects = sections.map((section, index) => {
-    const { left, top, width: width2, height: height2 } = section.rect;
-    const color = colors[index % colors.length];
-    sectionNameColorMap[section.name] = color;
-    return `
-            <rect x="${left * ratio}" y="${top * ratio}" width="${width2 * ratio}" height="${height2 * ratio}" fill="${color.hex}" />
-            <text x="${left * ratio}" y="${top * ratio + textFontSize}" font-family="Arial" font-size="${textFontSize}" fill="black">
-                ${section.name}
-            </text>
-        `;
-  });
-  const rectangles = `
-        <svg width="${canvasWidth}" height="${canvasHeight}">
-        ${rects.join("\n")}
-        </svg>
-    `;
-  const svgBuffer = Buffer3.from(rectangles);
-  const file = getTmpFile("png");
-  await Sharp3({
-    create: {
-      width: canvasWidth,
-      height: canvasHeight,
-      channels: 4,
-      background: { r: 255, g: 255, b: 255, alpha: 1 }
-    }
-  }).composite([{ input: svgBuffer }]).png().toFile(file);
-  return {
-    file,
-    sectionNameColorMap
-  };
-}
+  imageInfo,
+  imageInfoOfBase64,
+  base64Encoded,
+  calculateNewDimensions,
+  resizeImg,
+  transformImgPathToBase64,
+  saveBase64Image
+} from "@midscene/shared/img";
 export {
-  alignCoordByTrim,
   base64Encoded,
   calculateNewDimensions,
-  composeSectionDiagram,
   imageInfo,
   imageInfoOfBase64,
   resizeImg,
   saveBase64Image,
-  transformImgPathToBase64,
-  trimImage
+  transformImgPathToBase64
 };

package/dist/es/index.js

@@ -1251,12 +1251,13 @@ var Executor = class {
     }
     if (successfullyCompleted) {
       this.status = "completed";
-      if (this.tasks.length) {
-        return this.tasks[this.tasks.length - 1].output;
-      }
     } else {
       this.status = "error";
     }
+    if (this.tasks.length) {
+      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);
+      return this.tasks[outputIndex].output;
+    }
   }
   isInErrorState() {
     return this.status === "error";
@@ -1286,7 +1287,7 @@ var Executor = class {
 };
 
 // src/insight/index.ts
-import assert10 from "assert";
+import assert9 from "assert";
 
 // src/ai-model/openai/index.ts
 import assert3 from "assert";
@@ -4519,6 +4520,7 @@ import OpenAI from "openai";
 var MIDSCENE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_OPENAI_INIT_CONFIG_JSON";
 var MIDSCENE_MODEL_NAME = "MIDSCENE_MODEL_NAME";
 var MIDSCENE_LANGSMITH_DEBUG = "MIDSCENE_LANGSMITH_DEBUG";
+var MIDSCENE_DEBUG_AI_PROFILE = "MIDSCENE_DEBUG_AI_PROFILE";
 var OPENAI_API_KEY = "OPENAI_API_KEY";
 function useOpenAIModel(useModel) {
   if (useModel && useModel !== "openAI")
@@ -4548,12 +4550,16 @@ async function createOpenAI() {
 }
 async function call(messages, responseFormat) {
   const openai = await createOpenAI();
+  const shouldPrintTiming = typeof process.env[MIDSCENE_DEBUG_AI_PROFILE] === "string";
+  shouldPrintTiming && console.time("Midscene - AI call");
   const completion = await openai.chat.completions.create({
     model,
     messages,
     response_format: { type: responseFormat },
     temperature: 0.2
   });
+  shouldPrintTiming && console.timeEnd("Midscene - AI call");
+  shouldPrintTiming && console.log("Midscene - AI usage", completion.usage);
   const { content } = completion.choices[0].message;
   assert3(content, "empty content");
   return content;
@@ -4701,30 +4707,18 @@ function multiDescription(multi) {
 }
 
 // src/ai-model/prompt/util.ts
-import assert5 from "assert";
-
-// src/image/info.ts
 import assert4 from "assert";
-import { Buffer as Buffer2 } from "buffer";
-import { readFileSync as readFileSync2 } from "fs";
-import Sharp from "sharp";
-async function imageInfo(image) {
-  const { width, height } = await Sharp(image).metadata();
-  assert4(width && height, `invalid image: ${image}`);
-  return { width, height };
-}
-async function imageInfoOfBase64(imageBase64) {
-  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
-  return imageInfo(Buffer2.from(base64Data, "base64"));
-}
 
-// src/image/transform.ts
-import { Buffer as Buffer3 } from "buffer";
-import Sharp2 from "sharp";
-
-// src/image/visualization.ts
-import { Buffer as Buffer4 } from "buffer";
-import Sharp3 from "sharp";
+// src/image/index.ts
+import {
+  imageInfo,
+  imageInfoOfBase64,
+  base64Encoded,
+  calculateNewDimensions,
+  resizeImg,
+  transformImgPathToBase64,
+  saveBase64Image
+} from "@midscene/shared/img";
 
 // src/ai-model/prompt/util.ts
 var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
@@ -4818,7 +4812,7 @@ async function describeUserPage(context) {
       "elementInfos": ${JSON.stringify(elementInfosDescription)}
     }`,
     elementById(id) {
-      assert5(typeof id !== "undefined", "id is required for query");
+      assert4(typeof id !== "undefined", "id is required for query");
       const item = idElementMap[`${id}`];
       return item;
     }
@@ -4883,10 +4877,10 @@ function retrieveSection(prompt) {
 }
 
 // src/ai-model/inspect.ts
-import assert7 from "assert";
+import assert6 from "assert";
 
 // src/ai-model/coze/index.ts
-import assert6 from "assert";
+import assert5 from "assert";
 import fetch2 from "node-fetch";
 var COZE_INSPECT_ELEMENT_BOT_ID = process.env.COZE_INSPECT_ELEMENT_BOT_ID || "";
 var COZE_AI_ACTION_BOT_ID = process.env.COZE_AI_ACTION_BOT_ID || "";
@@ -4939,7 +4933,7 @@ async function callCozeAi(options) {
     throw new Error("aiResponse is undefined", aiResponse);
   }
   const parseContent = (_b = aiResponse == null ? void 0 : aiResponse.messages[0]) == null ? void 0 : _b.content;
-  assert6(parseContent, "empty content");
+  assert5(parseContent, "empty content");
   try {
     return JSON.parse(parseContent);
   } catch (err) {
@@ -5114,7 +5108,7 @@ DATA_DEMAND ends.
 }
 async function AiAssert(options) {
   const { assertion, context, useModel } = options;
-  assert7(assertion, "assertion should be a string");
+  assert6(assertion, "assertion should be a string");
   const { screenshotBase64 } = context;
   const { description, elementById } = await describeUserPage(context);
   const systemPrompt = systemPromptToAssert();
@@ -5158,7 +5152,7 @@ async function AiAssert(options) {
 }
 
 // src/ai-model/automation/index.ts
-import assert8 from "assert";
+import assert7 from "assert";
 
 // src/ai-model/automation/planning.ts
 function systemPromptToTaskPlanning() {
@@ -5264,8 +5258,8 @@ async function plan(userPrompt, opts, useModel) {
     });
   }
   const actions = (planFromAI == null ? void 0 : planFromAI.actions) || [];
-  assert8(planFromAI, "can't get planFromAI");
-  assert8(actions && actions.length > 0, "no actions in ai plan");
+  assert7(planFromAI, "can't get planFromAI");
+  assert7(actions && actions.length > 0, "no actions in ai plan");
   if (planFromAI.error) {
     throw new Error(planFromAI.error);
   }
@@ -5273,7 +5267,7 @@ async function plan(userPrompt, opts, useModel) {
 }
 
 // src/insight/utils.ts
-import assert9 from "assert";
+import assert8 from "assert";
 import { randomUUID as randomUUID2 } from "crypto";
 import { existsSync as existsSync2 } from "fs";
 import { join as join2 } from "path";
@@ -5284,7 +5278,7 @@ var { pid } = process;
 var logFileExt = insightDumpFileExt;
 function writeInsightDump(data, logId, dumpSubscriber) {
   const logDir2 = getLogDir();
-  assert9(logDir2, "logDir should be set before writing dump file");
+  assert8(logDir2, "logDir should be set before writing dump file");
   const id = logId || randomUUID2();
   const baseData = {
     sdkVersion: getPkgInfo().version,
@@ -5380,7 +5374,7 @@ var Insight = class {
     __publicField(this, "aiVendorFn", callAiFn);
     __publicField(this, "onceDumpUpdatedFn");
     __publicField(this, "taskInfo");
-    assert10(context, "context is required for Insight");
+    assert9(context, "context is required for Insight");
     if (typeof context === "function") {
       this.contextRetrieverFn = context;
     } else {
@@ -5396,7 +5390,7 @@ var Insight = class {
   async locate(queryPrompt, opt) {
     var _a;
     const { callAI: callAI2, multi = false } = opt || {};
-    assert10(queryPrompt, "query is required for located");
+    assert9(queryPrompt, "query is required for located");
     const dumpSubscriber = this.onceDumpUpdatedFn;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn();
@@ -5468,7 +5462,7 @@ ${parseResult.errors.join("\n")}`;
   }
   async extract(dataDemand) {
     var _a;
-    assert10(
+    assert9(
       typeof dataDemand === "object" || typeof dataDemand === "string",
       `dataDemand should be object or string, but get ${typeof dataDemand}`
     );