@midscene/core 0.3.1-beta-20240821105917.0 → 0.3.1

package/dist/es/ai-model.js

@@ -4440,7 +4440,7 @@ import {
   writeFileSync
 } from "fs";
 import { tmpdir } from "os";
-import path, { basename, join } from "path";
+import path, { basename, dirname, join } from "path";
 var logDir = join(process.cwd(), "./midscene_run/");
 
 // src/image/visualization.ts
@@ -4954,11 +4954,6 @@ async function plan(userPrompt, opts, useModel) {
   if (planFromAI.error) {
     throw new Error(planFromAI.error);
   }
-  actions.forEach((task) => {
-    if (task.type === "Error") {
-      throw new Error(task.thought);
-    }
-  });
   return { plans: actions };
 }
 export {

package/dist/es/image.js

@@ -1,23 +1,6 @@
-var __defProp = Object.defineProperty;
-var __getOwnPropSymbols = Object.getOwnPropertySymbols;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __propIsEnum = Object.prototype.propertyIsEnumerable;
-var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
-var __spreadValues = (a, b) => {
-  for (var prop in b || (b = {}))
-    if (__hasOwnProp.call(b, prop))
-      __defNormalProp(a, prop, b[prop]);
-  if (__getOwnPropSymbols)
-    for (var prop of __getOwnPropSymbols(b)) {
-      if (__propIsEnum.call(b, prop))
-        __defNormalProp(a, prop, b[prop]);
-    }
-  return a;
-};
-
 // src/image/info.ts
 import assert from "assert";
-import { Buffer as Buffer2 } from "buffer";
+import { Buffer } from "buffer";
 import { readFileSync } from "fs";
 import Sharp from "sharp";
 async function imageInfo(image) {
@@ -27,7 +10,7 @@ async function imageInfo(image) {
 }
 async function imageInfoOfBase64(imageBase64) {
   const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
-  return imageInfo(Buffer2.from(base64Data, "base64"));
+  return imageInfo(Buffer.from(base64Data, "base64"));
 }
 function base64Encoded(image, withHeader = true) {
   const imageBuffer = readFileSync(image);
@@ -44,12 +27,12 @@ function base64Encoded(image, withHeader = true) {
 }
 
 // src/image/transform.ts
-import { Buffer as Buffer3 } from "buffer";
+import { Buffer as Buffer2 } from "buffer";
 import Sharp2 from "sharp";
 async function saveBase64Image(options) {
   const { base64Data, outputPath } = options;
   const base64Image = base64Data.split(";base64,").pop() || base64Data;
-  const imageBuffer = Buffer3.from(base64Image, "base64");
+  const imageBuffer = Buffer2.from(base64Image, "base64");
   await Sharp2(imageBuffer).toFile(outputPath);
   console.log("Image successfully written to file.");
 }
@@ -61,7 +44,7 @@ async function transformImgPathToBase64(inputPath) {
 }
 async function resizeImg(base64Data) {
   const base64Image = base64Data.split(";base64,").pop() || base64Data;
-  const imageBuffer = Buffer3.from(base64Image, "base64");
+  const imageBuffer = Buffer2.from(base64Image, "base64");
   const metadata = await Sharp2(imageBuffer).metadata();
   const { width, height } = metadata;
   if (!width || !height) {
@@ -116,42 +99,26 @@ async function alignCoordByTrim(image, centerRect) {
   if (!(imgInfo == null ? void 0 : imgInfo.width) || !imgInfo.height || imgInfo.width <= 3 || imgInfo.height <= 3) {
     return centerRect;
   }
-  const zeroSize = {
-    left: 0,
-    top: 0,
-    width: -1,
-    height: -1
-  };
-  const finalCenterRect = __spreadValues({}, centerRect);
-  if (centerRect.left > imgInfo.width || centerRect.top > imgInfo.height) {
-    return zeroSize;
-  }
-  if (centerRect.left + centerRect.width > imgInfo.width) {
-    finalCenterRect.width = imgInfo.width - centerRect.left;
-  }
-  if (centerRect.top + centerRect.height > imgInfo.height) {
-    finalCenterRect.height = imgInfo.height - centerRect.top;
-  }
   try {
-    const img = await Sharp2(image).extract(finalCenterRect).toBuffer();
+    const img = await Sharp2(image).extract(centerRect).toBuffer();
     const trimInfo = await trimImage(img);
     if (!trimInfo) {
-      return finalCenterRect;
+      return centerRect;
     }
     return {
-      left: finalCenterRect.left - trimInfo.trimOffsetLeft,
-      top: finalCenterRect.top - trimInfo.trimOffsetTop,
+      left: centerRect.left - trimInfo.trimOffsetLeft,
+      top: centerRect.top - trimInfo.trimOffsetTop,
       width: trimInfo.width,
       height: trimInfo.height
     };
   } catch (e) {
-    console.warn(imgInfo, finalCenterRect);
+    console.log(imgInfo);
     throw e;
   }
 }
 
 // src/image/visualization.ts
-import { Buffer as Buffer4 } from "buffer";
+import { Buffer as Buffer3 } from "buffer";
 
 // src/utils.ts
 import assert2 from "assert";
@@ -163,7 +130,7 @@ import {
   writeFileSync
 } from "fs";
 import { tmpdir } from "os";
-import path, { basename, join } from "path";
+import path, { basename, dirname, join } from "path";
 var pkg;
 function getPkgInfo() {
   if (pkg) {
@@ -293,7 +260,7 @@ async function composeSectionDiagram(sections, context) {
         ${rects.join("\n")}
         </svg>
     `;
-  const svgBuffer = Buffer4.from(rectangles);
+  const svgBuffer = Buffer3.from(rectangles);
   const file = getTmpFile("png");
   await Sharp3({
     create: {

package/dist/es/index.js

@@ -1021,7 +1021,7 @@ import {
   writeFileSync
 } from "fs";
 import { tmpdir } from "os";
-import path, { basename, join } from "path";
+import path, { basename, dirname, join } from "path";
 var pkg;
 function getPkgInfo() {
   if (pkg) {
@@ -1106,6 +1106,10 @@ ${logDirName}/dump
     logEnvReady = true;
   }
   const filePath = join(targetDir, `${fileName}.${fileExt}`);
+  const outputResourceDir = dirname(filePath);
+  if (!existsSync(outputResourceDir)) {
+    mkdirSync(outputResourceDir, { recursive: true });
+  }
   writeFileSync(filePath, fileContent);
   if (opts == null ? void 0 : opts.generateReport) {
     return writeDumpReport(fileName, fileContent);
@@ -1247,13 +1251,12 @@ var Executor = class {
     }
     if (successfullyCompleted) {
       this.status = "completed";
+      if (this.tasks.length) {
+        return this.tasks[this.tasks.length - 1].output;
+      }
     } else {
       this.status = "error";
     }
-    if (this.tasks.length) {
-      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);
-      return this.tasks[outputIndex].output;
-    }
   }
   isInErrorState() {
     return this.status === "error";
@@ -5266,11 +5269,6 @@ async function plan(userPrompt, opts, useModel) {
   if (planFromAI.error) {
     throw new Error(planFromAI.error);
   }
-  actions.forEach((task) => {
-    if (task.type === "Error") {
-      throw new Error(task.thought);
-    }
-  });
   return { plans: actions };
 }
 

package/dist/es/utils.js

@@ -8,7 +8,7 @@ import {
   writeFileSync
 } from "fs";
 import { tmpdir } from "os";
-import path, { basename, join } from "path";
+import path, { basename, dirname, join } from "path";
 var pkg;
 function getPkgInfo() {
   if (pkg) {
@@ -94,6 +94,10 @@ ${logDirName}/dump
     logEnvReady = true;
   }
   const filePath = join(targetDir, `${fileName}.${fileExt}`);
+  const outputResourceDir = dirname(filePath);
+  if (!existsSync(outputResourceDir)) {
+    mkdirSync(outputResourceDir, { recursive: true });
+  }
   writeFileSync(filePath, fileContent);
   if (opts == null ? void 0 : opts.generateReport) {
     return writeDumpReport(fileName, fileContent);

package/dist/lib/ai-model.js

@@ -4969,11 +4969,6 @@ async function plan(userPrompt, opts, useModel) {
   if (planFromAI.error) {
     throw new Error(planFromAI.error);
   }
-  actions.forEach((task) => {
-    if (task.type === "Error") {
-      throw new Error(task.thought);
-    }
-  });
   return { plans: actions };
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/lib/image.js

@@ -3,22 +3,8 @@ var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
-var __getOwnPropSymbols = Object.getOwnPropertySymbols;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __propIsEnum = Object.prototype.propertyIsEnumerable;
-var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
-var __spreadValues = (a, b) => {
-  for (var prop in b || (b = {}))
-    if (__hasOwnProp.call(b, prop))
-      __defNormalProp(a, prop, b[prop]);
-  if (__getOwnPropSymbols)
-    for (var prop of __getOwnPropSymbols(b)) {
-      if (__propIsEnum.call(b, prop))
-        __defNormalProp(a, prop, b[prop]);
-    }
-  return a;
-};
 var __export = (target, all) => {
   for (var name in all)
     __defProp(target, name, { get: all[name], enumerable: true });
@@ -158,36 +144,20 @@ async function alignCoordByTrim(image, centerRect) {
   if (!(imgInfo == null ? void 0 : imgInfo.width) || !imgInfo.height || imgInfo.width <= 3 || imgInfo.height <= 3) {
     return centerRect;
   }
-  const zeroSize = {
-    left: 0,
-    top: 0,
-    width: -1,
-    height: -1
-  };
-  const finalCenterRect = __spreadValues({}, centerRect);
-  if (centerRect.left > imgInfo.width || centerRect.top > imgInfo.height) {
-    return zeroSize;
-  }
-  if (centerRect.left + centerRect.width > imgInfo.width) {
-    finalCenterRect.width = imgInfo.width - centerRect.left;
-  }
-  if (centerRect.top + centerRect.height > imgInfo.height) {
-    finalCenterRect.height = imgInfo.height - centerRect.top;
-  }
   try {
-    const img = await (0, import_sharp2.default)(image).extract(finalCenterRect).toBuffer();
+    const img = await (0, import_sharp2.default)(image).extract(centerRect).toBuffer();
     const trimInfo = await trimImage(img);
     if (!trimInfo) {
-      return finalCenterRect;
+      return centerRect;
     }
     return {
-      left: finalCenterRect.left - trimInfo.trimOffsetLeft,
-      top: finalCenterRect.top - trimInfo.trimOffsetTop,
+      left: centerRect.left - trimInfo.trimOffsetLeft,
+      top: centerRect.top - trimInfo.trimOffsetTop,
       width: trimInfo.width,
       height: trimInfo.height
     };
   } catch (e) {
-    console.warn(imgInfo, finalCenterRect);
+    console.log(imgInfo);
     throw e;
   }
 }

package/dist/lib/index.js

@@ -1122,6 +1122,10 @@ ${logDirName}/dump
     logEnvReady = true;
   }
   const filePath = (0, import_node_path.join)(targetDir, `${fileName}.${fileExt}`);
+  const outputResourceDir = (0, import_node_path.dirname)(filePath);
+  if (!(0, import_node_fs.existsSync)(outputResourceDir)) {
+    (0, import_node_fs.mkdirSync)(outputResourceDir, { recursive: true });
+  }
   (0, import_node_fs.writeFileSync)(filePath, fileContent);
   if (opts == null ? void 0 : opts.generateReport) {
     return writeDumpReport(fileName, fileContent);
@@ -1263,13 +1267,12 @@ var Executor = class {
     }
     if (successfullyCompleted) {
       this.status = "completed";
+      if (this.tasks.length) {
+        return this.tasks[this.tasks.length - 1].output;
+      }
     } else {
       this.status = "error";
     }
-    if (this.tasks.length) {
-      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);
-      return this.tasks[outputIndex].output;
-    }
   }
   isInErrorState() {
     return this.status === "error";
@@ -5282,11 +5285,6 @@ async function plan(userPrompt, opts, useModel) {
   if (planFromAI.error) {
     throw new Error(planFromAI.error);
   }
-  actions.forEach((task) => {
-    if (task.type === "Error") {
-      throw new Error(task.thought);
-    }
-  });
   return { plans: actions };
 }
 

package/dist/lib/utils.js

@@ -138,6 +138,10 @@ ${logDirName}/dump
     logEnvReady = true;
   }
   const filePath = (0, import_node_path.join)(targetDir, `${fileName}.${fileExt}`);
+  const outputResourceDir = (0, import_node_path.dirname)(filePath);
+  if (!(0, import_node_fs.existsSync)(outputResourceDir)) {
+    (0, import_node_fs.mkdirSync)(outputResourceDir, { recursive: true });
+  }
   (0, import_node_fs.writeFileSync)(filePath, fileContent);
   if (opts == null ? void 0 : opts.generateReport) {
     return writeDumpReport(fileName, fileContent);

package/dist/types/ai-model.d.ts

@@ -1,8 +1,8 @@
 import { ChatCompletionMessageParam } from 'openai/resources';
 export { ChatCompletionMessageParam } from 'openai/resources';
-import { c as callAiFn } from './index-0479d487.js';
-export { d as describeUserPage, p as plan } from './index-0479d487.js';
-import { B as BaseElement, U as UIContext, e as AIElementParseResponse, f as AISectionParseResponse, g as AIAssertionResponse } from './types-3eb61b5c.js';
+import { c as callAiFn } from './index-7a9ec3e1.js';
+export { d as describeUserPage, p as plan } from './index-7a9ec3e1.js';
+import { B as BaseElement, U as UIContext, e as AIElementParseResponse, f as AISectionParseResponse, g as AIAssertionResponse } from './types-ed68710b.js';
 
 declare function AiInspectElement<ElementType extends BaseElement = BaseElement>(options: {
     context: UIContext<ElementType>;

package/dist/types/image.d.ts

@@ -1,5 +1,5 @@
 import { Buffer } from 'node:buffer';
-import { S as Size, R as Rect, h as UISection, U as UIContext, G as Color } from './types-3eb61b5c.js';
+import { S as Size, R as Rect, h as UISection, U as UIContext, G as Color } from './types-ed68710b.js';
 import 'openai/resources';
 
 /**

package/dist/types/{index-0479d487.d.ts → index-7a9ec3e1.d.ts}

@@ -1,4 +1,4 @@
-import { B as BaseElement, U as UIContext, r as PlanningAction } from './types-3eb61b5c.js';
+import { B as BaseElement, U as UIContext, r as PlanningAction } from './types-ed68710b.js';
 import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
 
 type AIArgs = [

package/dist/types/index.d.ts

@@ -1,7 +1,7 @@
-import { E as ExecutionTask, a as ExecutionTaskApply, b as ExecutionDump, B as BaseElement, U as UIContext, D as DumpSubscriber, I as InsightTaskInfo, c as InsightOptions, d as InsightAssertionResponse } from './types-3eb61b5c.js';
-export { g as AIAssertionResponse, e as AIElementParseResponse, A as AIResponseFormat, f as AISectionParseResponse, q as AgentWaitForOpt, H as BaseAgentParserOpt, j as BasicSectionQuery, C as CallAIFn, G as Color, l as DumpMeta, p as ElementById, i as EnsureObject, M as ExecutionRecorderItem, a6 as ExecutionTaskAction, a5 as ExecutionTaskActionApply, a4 as ExecutionTaskInsightAssertion, a3 as ExecutionTaskInsightAssertionApply, a2 as ExecutionTaskInsightAssertionParam, X as ExecutionTaskInsightDumpLog, Z as ExecutionTaskInsightLocate, Y as ExecutionTaskInsightLocateApply, W as ExecutionTaskInsightLocateOutput, V as ExecutionTaskInsightLocateParam, a1 as ExecutionTaskInsightQuery, a0 as ExecutionTaskInsightQueryApply, $ as ExecutionTaskInsightQueryOutput, _ as ExecutionTaskInsightQueryParam, a8 as ExecutionTaskPlanning, a7 as ExecutionTaskPlanningApply, Q as ExecutionTaskReturn, N as ExecutionTaskType, O as ExecutorContext, a9 as GroupedActionDump, n as InsightDump, k as InsightExtractParam, L as LiteUISection, o as PartialInsightDumpFromSDK, s as PlanningAIResponse, r as PlanningAction, x as PlanningActionParamAssert, z as PlanningActionParamError, u as PlanningActionParamHover, v as PlanningActionParamInputOrKeyPress, w as PlanningActionParamScroll, y as PlanningActionParamSleep, t as PlanningActionParamTap, F as PlanningActionParamWaitFor, K as PlaywrightParserOpt, P as Point, J as PuppeteerParserOpt, R as Rect, m as ReportDumpWithAttributes, S as Size, T as TaskCacheInfo, h as UISection } from './types-3eb61b5c.js';
-import { c as callAiFn, r as retrieveElement, a as retrieveSection } from './index-0479d487.js';
-export { p as plan } from './index-0479d487.js';
+import { E as ExecutionTask, a as ExecutionTaskApply, b as ExecutionDump, B as BaseElement, U as UIContext, D as DumpSubscriber, I as InsightTaskInfo, c as InsightOptions, d as InsightAssertionResponse } from './types-ed68710b.js';
+export { g as AIAssertionResponse, e as AIElementParseResponse, A as AIResponseFormat, f as AISectionParseResponse, q as AgentWaitForOpt, H as BaseAgentParserOpt, j as BasicSectionQuery, C as CallAIFn, G as Color, l as DumpMeta, p as ElementById, i as EnsureObject, M as ExecutionRecorderItem, a6 as ExecutionTaskAction, a5 as ExecutionTaskActionApply, a4 as ExecutionTaskInsightAssertion, a3 as ExecutionTaskInsightAssertionApply, a2 as ExecutionTaskInsightAssertionParam, X as ExecutionTaskInsightDumpLog, Z as ExecutionTaskInsightLocate, Y as ExecutionTaskInsightLocateApply, W as ExecutionTaskInsightLocateOutput, V as ExecutionTaskInsightLocateParam, a1 as ExecutionTaskInsightQuery, a0 as ExecutionTaskInsightQueryApply, $ as ExecutionTaskInsightQueryOutput, _ as ExecutionTaskInsightQueryParam, a8 as ExecutionTaskPlanning, a7 as ExecutionTaskPlanningApply, Q as ExecutionTaskReturn, N as ExecutionTaskType, O as ExecutorContext, a9 as GroupedActionDump, n as InsightDump, k as InsightExtractParam, L as LiteUISection, o as PartialInsightDumpFromSDK, s as PlanningAIResponse, r as PlanningAction, x as PlanningActionParamAssert, z as PlanningActionParamError, u as PlanningActionParamHover, v as PlanningActionParamInputOrKeyPress, w as PlanningActionParamScroll, y as PlanningActionParamSleep, t as PlanningActionParamTap, F as PlanningActionParamWaitFor, K as PlaywrightParserOpt, P as Point, J as PuppeteerParserOpt, R as Rect, m as ReportDumpWithAttributes, S as Size, T as TaskCacheInfo, h as UISection } from './types-ed68710b.js';
+import { c as callAiFn, r as retrieveElement, a as retrieveSection } from './index-7a9ec3e1.js';
+export { p as plan } from './index-7a9ec3e1.js';
 export { setLogDir } from './utils.js';
 import 'openai/resources';
 

package/dist/types/{types-3eb61b5c.d.ts → types-ed68710b.d.ts}

@@ -10,7 +10,7 @@ interface Size {
 }
 type Rect = Point & Size;
 declare enum NodeType {
-    FORM_ITEM = "FORM_ITEM Node",
+    INPUT = "INPUT Node",
     BUTTON = "BUTTON Node",
     IMG = "IMG Node",
     TEXT = "TEXT Node"

package/dist/types/utils.d.ts

@@ -1,4 +1,4 @@
-import { m as ReportDumpWithAttributes, R as Rect } from './types-3eb61b5c.js';
+import { m as ReportDumpWithAttributes, R as Rect } from './types-ed68710b.js';
 import 'openai/resources';
 
 interface PkgInfo {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Hello, It's Midscene",
-  "version": "0.3.1-beta-20240821105917.0",
+  "version": "0.3.1",
   "jsnext:source": "./src/index.ts",
   "main": "./dist/lib/index.js",
   "module": "./dist/es/index.js",