@midscene/shared 0.7.2 → 0.7.3-beta-20241104100519.0

package/package.json

@@ -1,46 +1,63 @@
 {
   "name": "@midscene/shared",
-  "version": "0.7.2",
+  "version": "0.7.3-beta-20241104100519.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
-  "types": "./dist/types/index.d.ts",
+  "types": "./src/index.ts",
+  "type": "commonjs",
   "main": "./dist/lib/index.js",
   "module": "./dist/es/index.js",
   "exports": {
     ".": {
-      "types": "./dist/types/index.d.ts",
-      "import": "./dist/es/index.js",
-      "require": "./dist/lib/index.js"
+      "types": "./dist/lib/index.d.ts",
+      "require": "./dist/lib/index.js",
+      "import": "./dist/es/index.js"
     },
     "./constants": {
-      "types": "./dist/types/constants.d.ts",
-      "import": "./dist/es/constants.js",
-      "require": "./dist/lib/constants.js"
+      "types": "./dist/lib/constants.d.ts",
+      "require": "./dist/lib/constants.js",
+      "import": "./dist/es/constants.js"
+    },
+    "./fs": {
+      "types": "./dist/lib/fs.d.ts",
+      "require": "./dist/lib/fs.js",
+      "import": "./dist/es/fs.js"
     },
     "./img": {
-      "types": "./dist/types/img.d.ts",
-      "import": "./dist/es/img.js",
-      "require": "./dist/lib/img.js"
+      "types": "./dist/lib/img.d.ts",
+      "require": "./dist/lib/img.js",
+      "import": "./dist/es/img.js"
     },
-    "./fs": {
-      "types": "./dist/types/fs.d.ts",
-      "import": "./dist/es/fs.js",
-      "require": "./dist/lib/fs.js"
+    "./browser/img": {
+      "types": "./dist/browser/img.d.ts",
+      "require": "./dist/browser/img.js",
+      "import": "./dist/browser/img.js"
+    },
+    "./utils": {
+      "types": "./dist/lib/utils.d.ts",
+      "require": "./dist/lib/utils.js",
+      "import": "./dist/es/utils.js"
     }
   },
   "typesVersions": {
     "*": {
       ".": [
-        "./dist/types/index.d.ts"
+        "./dist/lib/index.d.ts"
       ],
       "constants": [
-        "./dist/types/constants.d.ts"
+        "./dist/lib/constants.d.ts"
       ],
       "img": [
-        "./dist/types/img.d.ts"
+        "./dist/lib/img.d.ts"
+      ],
+      "browser/img": [
+        "./dist/browser/img.d.ts"
       ],
       "fs": [
-        "./dist/types/fs.d.ts"
+        "./dist/lib/fs.d.ts"
+      ],
+      "utils": [
+        "./dist/lib/utils.d.ts"
       ]
     }
   },
@@ -54,7 +71,7 @@
   },
   "peerDependencies": {},
   "devDependencies": {
-    "@modern-js/module-tools": "2.58.2",
+    "@modern-js/module-tools": "2.60.6",
     "typescript": "~5.0.4",
     "@types/node": "^18.0.0",
     "rimraf": "~3.0.2",

package/src/fs/index.ts

@@ -8,25 +8,29 @@ interface PkgInfo {
   dir: string;
 }
 
-let pkg: PkgInfo | undefined;
-export function getMidscenePkgInfo(dir: string): PkgInfo {
-  if (pkg) {
-    return pkg;
+const pkgCacheMap: Record<string, PkgInfo> = {};
+const ifInBrowser = typeof window !== 'undefined';
+export function getRunningPkgInfo(dir?: string): PkgInfo | null {
+  if (ifInBrowser) {
+    return null;
+  }
+  const dirToCheck = dir || process.cwd();
+  if (pkgCacheMap[dirToCheck]) {
+    return pkgCacheMap[dirToCheck];
   }
 
-  const pkgDir = findNearestPackageJson(dir || process.cwd());
-  assert(pkgDir, 'package.json not found');
-  const pkgJsonFile = join(pkgDir, 'package.json');
+  const pkgDir = findNearestPackageJson(dirToCheck);
+  const pkgJsonFile = pkgDir ? join(pkgDir, 'package.json') : null;
 
-  if (pkgJsonFile) {
+  if (pkgDir && pkgJsonFile) {
     const { name, version } = JSON.parse(readFileSync(pkgJsonFile, 'utf-8'));
-    pkg = { name, version, dir: pkgDir };
-    return pkg;
+    pkgCacheMap[dirToCheck] = { name, version, dir: pkgDir };
+    return pkgCacheMap[dirToCheck];
   }
   return {
-    name: 'midscene-unknown-page-name',
+    name: 'midscene-unknown-package-name',
     version: '0.0.0',
-    dir: pkgDir,
+    dir: dirToCheck,
   };
 }
 

package/src/img/box-select.ts

@@ -1,231 +1,246 @@
 import assert from 'node:assert';
-import { Buffer } from 'node:buffer';
+import type { Buffer } from 'node:buffer';
 import type { Rect } from '@/types';
-import Jimp from 'jimp';
+import type Jimp from 'jimp';
 import type { NodeType } from '../constants';
+import getJimp from './get-jimp';
+import { bufferFromBase64, imageInfo, imageInfoOfBase64 } from './index';
 
 // Define picture path
 type ElementType = {
-  locator: string;
+  locator?: string;
 
   rect: Rect;
 
-  center: [number, number];
+  center?: [number, number];
 
-  id: string;
+  id?: string;
 
   indexId: number;
 
-  attributes: {
+  attributes?: {
     nodeType: NodeType;
     [key: string]: string;
   };
 };
 
-const createSvgOverlay = (
+let cachedFont: any = null;
+
+const createSvgOverlay = async (
   elements: Array<ElementType>,
   imageWidth: number,
   imageHeight: number,
-) => {
-  const createPngOverlay = async (
-    elements: Array<ElementType>,
-    imageWidth: number,
-    imageHeight: number,
-  ) => {
-    const image = new Jimp(imageWidth, imageHeight, 0x00000000);
-
-    // Define color array
-    const colors = [
-      { rect: 0xff0000ff, text: 0xffffffff }, // red, white
-      // { rect: 0x0000ffff, text: 0xffffffff }, // blue, white
-      // { rect: 0x8b4513ff, text: 0xffffffff }, // brown, white
-    ];
+): Promise<Jimp> => {
+  const Jimp = await getJimp();
+  const image = new Jimp(imageWidth, imageHeight, 0x00000000);
 
-    const boxPadding = 5;
-    for (let index = 0; index < elements.length; index++) {
-      const element = elements[index];
-      const color = colors[index % colors.length];
+  // Define color array
+  const colors = [
+    { rect: 0xff0000ff, text: 0xffffffff }, // red, white
+    // { rect: 0x0000ffff, text: 0xffffffff }, // blue, white
+    // { rect: 0x8b4513ff, text: 0xffffffff }, // brown, white
+  ];
 
-      // Add 5px padding to the rect
-      const paddedRect = {
-        left: Math.max(0, element.rect.left - boxPadding),
-        top: Math.max(0, element.rect.top - boxPadding),
-        width: Math.min(
-          imageWidth - element.rect.left,
-          element.rect.width + boxPadding * 2,
-        ),
-        height: Math.min(
-          imageHeight - element.rect.top,
-          element.rect.height + boxPadding * 2,
-        ),
-      };
+  const boxPadding = 5;
+  for (let index = 0; index < elements.length; index++) {
+    const element = elements[index];
+    const color = colors[index % colors.length];
 
-      // Draw rectangle
-      image.scan(
-        paddedRect.left,
-        paddedRect.top,
-        paddedRect.width,
-        paddedRect.height,
-        function (x, y, idx) {
-          if (
-            x === paddedRect.left ||
-            x === paddedRect.left + paddedRect.width - 1 ||
-            y === paddedRect.top ||
-            y === paddedRect.top + paddedRect.height - 1
-          ) {
-            this.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
-            this.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
-            this.bitmap.data[idx + 2] = (color.rect >> 8) & 0xff; // B
-            this.bitmap.data[idx + 3] = color.rect & 0xff; // A
-          }
-        },
-      );
+    // Add 5px padding to the rect
+    const paddedRect = {
+      left: Math.max(0, element.rect.left - boxPadding),
+      top: Math.max(0, element.rect.top - boxPadding),
+      width: Math.min(
+        imageWidth - element.rect.left,
+        element.rect.width + boxPadding * 2,
+      ),
+      height: Math.min(
+        imageHeight - element.rect.top,
+        element.rect.height + boxPadding * 2,
+      ),
+    };
 
-      // Calculate text position
-      const textWidth = element.indexId.toString().length * 8;
-      const textHeight = 12;
-      const rectWidth = textWidth + 5;
-      const rectHeight = textHeight + 4;
-      let rectX = paddedRect.left - rectWidth;
-      let rectY = paddedRect.top + paddedRect.height / 2 - textHeight / 2 - 2;
+    // Draw rectangle
+    image.scan(
+      paddedRect.left,
+      paddedRect.top,
+      paddedRect.width,
+      paddedRect.height,
+      function (x, y, idx) {
+        if (
+          x === paddedRect.left ||
+          x === paddedRect.left + paddedRect.width - 1 ||
+          y === paddedRect.top ||
+          y === paddedRect.top + paddedRect.height - 1
+        ) {
+          this.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
+          this.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
+          this.bitmap.data[idx + 2] = (color.rect >> 8) & 0xff; // B
+          this.bitmap.data[idx + 3] = color.rect & 0xff; // A
+        }
+      },
+    );
 
-      // Check if this new position overlaps with any existing boxes
-      // Function to check if a given position overlaps with any existing boxes
-      const checkOverlap = (x: number, y: number) => {
-        // Check against all previously processed elements
-        return elements.slice(0, index).some((otherElement) => {
-          // Check if the rectangles overlap
-          return (
-            x < otherElement.rect.left + otherElement.rect.width &&
-            x + rectWidth > otherElement.rect.left &&
-            y < otherElement.rect.top + otherElement.rect.height &&
-            y + rectHeight > otherElement.rect.top
-          );
-        });
-      };
+    // Calculate text position
+    const textWidth = element.indexId.toString().length * 8;
+    const textHeight = 12;
+    const rectWidth = textWidth + 5;
+    const rectHeight = textHeight + 4;
+    let rectX = paddedRect.left - rectWidth;
+    let rectY = paddedRect.top + paddedRect.height / 2 - textHeight / 2 - 2;
 
-      // Function to check if a given position is within the image bounds
-      const isWithinBounds = (x: number, y: number) => {
+    // Check if this new position overlaps with any existing boxes
+    // Function to check if a given position overlaps with any existing boxes
+    const checkOverlap = (x: number, y: number) => {
+      // Check against all previously processed elements
+      return elements.slice(0, index).some((otherElement) => {
+        // Check if the rectangles overlap
         return (
-          x >= 0 &&
-          x + rectWidth <= imageWidth &&
-          y >= 0 &&
-          y + rectHeight <= imageHeight
+          x < otherElement.rect.left + otherElement.rect.width &&
+          x + rectWidth > otherElement.rect.left &&
+          y < otherElement.rect.top + otherElement.rect.height &&
+          y + rectHeight > otherElement.rect.top
         );
-      };
+      });
+    };
 
-      // Check left side (original position)
-      if (checkOverlap(rectX, rectY) || !isWithinBounds(rectX, rectY)) {
-        // If the original position overlaps or is out of bounds, try alternative positions
+    // Function to check if a given position is within the image bounds
+    const isWithinBounds = (x: number, y: number) => {
+      return (
+        x >= 0 &&
+        x + rectWidth <= imageWidth &&
+        y >= 0 &&
+        y + rectHeight <= imageHeight
+      );
+    };
 
-        // Check top position
-        if (
-          !checkOverlap(paddedRect.left, paddedRect.top - rectHeight - 2) &&
-          isWithinBounds(paddedRect.left, paddedRect.top - rectHeight - 2)
-        ) {
-          rectX = paddedRect.left;
-          rectY = paddedRect.top - rectHeight - 2;
-        }
-        // Check bottom position
-        else if (
-          !checkOverlap(
-            paddedRect.left,
-            paddedRect.top + paddedRect.height + 2,
-          ) &&
-          isWithinBounds(
-            paddedRect.left,
-            paddedRect.top + paddedRect.height + 2,
-          )
-        ) {
-          rectX = paddedRect.left;
-          rectY = paddedRect.top + paddedRect.height + 2;
-        }
-        // Check right position
-        else if (
-          !checkOverlap(
-            paddedRect.left + paddedRect.width + 2,
-            paddedRect.top,
-          ) &&
-          isWithinBounds(paddedRect.left + paddedRect.width + 2, paddedRect.top)
-        ) {
-          rectX = paddedRect.left + paddedRect.width + 2;
-          rectY = paddedRect.top;
-        }
-        // If all sides are overlapped or out of bounds, place it inside the box at the top
-        else {
-          rectX = paddedRect.left;
-          rectY = paddedRect.top + 2;
-        }
-      }
-      // Note: If the original left position doesn't overlap and is within bounds, we keep it as is
+    // Check left side (original position)
+    if (checkOverlap(rectX, rectY) || !isWithinBounds(rectX, rectY)) {
+      // If the original position overlaps or is out of bounds, try alternative positions
 
-      // Draw text background
-      image.scan(rectX, rectY, rectWidth, rectHeight, function (x, y, idx) {
-        this.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
-        this.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
-        this.bitmap.data[idx + 2] = (color.rect >> 8) & 0xff; // B
-        this.bitmap.data[idx + 3] = color.rect & 0xff; // A
-      });
-      // Draw text (simplified, as Jimp doesn't have built-in text drawing)
-      const font = await Jimp.loadFont(Jimp.FONT_SANS_16_WHITE);
-      image.print(
-        font,
-        rectX,
-        rectY,
-        {
-          text: element.indexId.toString(),
-          alignmentX: Jimp.HORIZONTAL_ALIGN_CENTER,
-          alignmentY: Jimp.VERTICAL_ALIGN_MIDDLE,
-        },
-        rectWidth,
-        rectHeight,
-      );
+      // Check top position
+      if (
+        !checkOverlap(paddedRect.left, paddedRect.top - rectHeight - 2) &&
+        isWithinBounds(paddedRect.left, paddedRect.top - rectHeight - 2)
+      ) {
+        rectX = paddedRect.left;
+        rectY = paddedRect.top - rectHeight - 2;
+      }
+      // Check bottom position
+      else if (
+        !checkOverlap(
+          paddedRect.left,
+          paddedRect.top + paddedRect.height + 2,
+        ) &&
+        isWithinBounds(paddedRect.left, paddedRect.top + paddedRect.height + 2)
+      ) {
+        rectX = paddedRect.left;
+        rectY = paddedRect.top + paddedRect.height + 2;
+      }
+      // Check right position
+      else if (
+        !checkOverlap(paddedRect.left + paddedRect.width + 2, paddedRect.top) &&
+        isWithinBounds(paddedRect.left + paddedRect.width + 2, paddedRect.top)
+      ) {
+        rectX = paddedRect.left + paddedRect.width + 2;
+        rectY = paddedRect.top;
+      }
+      // If all sides are overlapped or out of bounds, place it inside the box at the top
+      else {
+        rectX = paddedRect.left;
+        rectY = paddedRect.top + 2;
+      }
     }
+    // Note: If the original left position doesn't overlap and is within bounds, we keep it as is
 
-    return image.getBufferAsync(Jimp.MIME_PNG);
-  };
+    // Draw text background
+    image.scan(rectX, rectY, rectWidth, rectHeight, function (x, y, idx) {
+      this.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
+      this.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
+      this.bitmap.data[idx + 2] = (color.rect >> 8) & 0xff; // B
+      this.bitmap.data[idx + 3] = color.rect & 0xff; // A
+    });
+    // Draw text (simplified, as Jimp doesn't have built-in text drawing)
+    try {
+      cachedFont = cachedFont || (await Jimp.loadFont(Jimp.FONT_SANS_16_WHITE));
+    } catch (error) {
+      console.error('Error loading font', error);
+    }
+    image.print(
+      cachedFont,
+      rectX,
+      rectY,
+      {
+        text: element.indexId.toString(),
+        alignmentX: Jimp.HORIZONTAL_ALIGN_CENTER,
+        alignmentY: Jimp.VERTICAL_ALIGN_MIDDLE,
+      },
+      rectWidth,
+      rectHeight,
+    );
+  }
 
-  return createPngOverlay(elements, imageWidth, imageHeight);
+  return image;
 };
 
 export const compositeElementInfoImg = async (options: {
   inputImgBase64: string;
   elementsPositionInfo: Array<ElementType>;
+  size?: { width: number; height: number };
 }) => {
-  const { inputImgBase64, elementsPositionInfo } = options;
-  const imageBuffer = Buffer.from(inputImgBase64, 'base64');
-  const image = await Jimp.read(imageBuffer);
-  const { width, height } = image.bitmap;
+  assert(options.inputImgBase64, 'inputImgBase64 is required');
+  let width = 0;
+  let height = 0;
+  let jimpImage: Jimp;
+
+  const Jimp = await getJimp();
+
+  if (options.size) {
+    width = options.size.width;
+    height = options.size.height;
+  }
+
+  if (!width || !height) {
+    const info = await imageInfoOfBase64(options.inputImgBase64);
+    width = info.width;
+    height = info.height;
+    jimpImage = info.jimpImage;
+  } else {
+    const imageBuffer = await bufferFromBase64(options.inputImgBase64);
+    jimpImage = await Jimp.read(imageBuffer);
+  }
 
   if (!width || !height) {
     throw Error('Image processing failed because width or height is undefined');
   }
 
-  // Create svg overlay
-  const svgOverlay = await createSvgOverlay(
-    elementsPositionInfo,
-    width,
-    height,
-  );
+  const { elementsPositionInfo } = options;
 
-  return await Jimp.read(imageBuffer)
+  const result = await Promise.resolve(jimpImage)
     .then(async (image: Jimp) => {
+      // Create svg overlay
+      const svgOverlay = await createSvgOverlay(
+        elementsPositionInfo,
+        width,
+        height,
+      );
       const svgImage = await Jimp.read(svgOverlay);
-      return image.composite(svgImage, 0, 0, {
+      const compositeImage = await image.composite(svgImage, 0, 0, {
         mode: Jimp.BLEND_SOURCE_OVER,
         opacitySource: 1,
         opacityDest: 1,
       });
+      return compositeImage;
     })
-    .then((compositeImage: Jimp) => {
-      return compositeImage.getBufferAsync(Jimp.MIME_PNG);
-    })
-    .then((buffer: Buffer) => {
-      return buffer.toString('base64');
+    .then(async (compositeImage: Jimp) => {
+      const base64 = await compositeImage.getBase64Async(Jimp.MIME_PNG);
+      return base64;
     })
     .catch((error: unknown) => {
       throw error;
     });
+
+  return result;
 };
 
 export const processImageElementInfo = async (options: {

package/src/img/get-jimp.ts

@@ -0,0 +1,11 @@
+import type Jimp from 'jimp/browser/lib/jimp.js';
+
+const ifInBrowser = typeof window !== 'undefined';
+export default async function getJimp(): Promise<typeof Jimp> {
+  if (ifInBrowser) {
+    await import('jimp/browser/lib/jimp.js');
+    return (window as any).Jimp;
+  }
+  // return Jimp;
+  return (await import('jimp')).default;
+}

package/src/img/index.ts

@@ -1,6 +1,7 @@
 export {
   imageInfo,
   imageInfoOfBase64,
+  bufferFromBase64,
   base64Encoded,
   base64ToPngFormat,
 } from './info';
@@ -8,6 +9,7 @@ export {
   trimImage,
   calculateNewDimensions,
   resizeImg,
+  resizeImgBase64,
   transformImgPathToBase64,
   saveBase64Image,
 } from './transform';

package/src/img/info.ts

@@ -1,13 +1,18 @@
 import assert from 'node:assert';
 import { Buffer } from 'node:buffer';
 import { readFileSync } from 'node:fs';
-import Jimp from 'jimp';
+import type Jimp from 'jimp';
+import getJimp from './get-jimp';
 
 export interface Size {
   width: number;
   height: number;
 }
 
+export interface ImageInfo extends Size {
+  jimpImage: Jimp;
+}
+
 /**
  * Retrieves the dimensions of an image asynchronously
  *
@@ -15,12 +20,17 @@ export interface Size {
  * @returns A Promise that resolves to an object containing the width and height of the image
  * @throws Error if the image data is invalid
  */
-export async function imageInfo(image: string | Buffer): Promise<Size> {
+export async function imageInfo(
+  image: string | Buffer | Jimp,
+): Promise<ImageInfo> {
+  const Jimp = await getJimp();
   let jimpImage: Jimp;
   if (typeof image === 'string') {
     jimpImage = await Jimp.read(image);
   } else if (Buffer.isBuffer(image)) {
     jimpImage = await Jimp.read(image);
+  } else if (image instanceof Jimp) {
+    jimpImage = image;
   } else {
     throw new Error('Invalid image input: must be a string path or a Buffer');
   }
@@ -29,7 +39,7 @@ export async function imageInfo(image: string | Buffer): Promise<Size> {
     width && height,
     `Invalid image: ${typeof image === 'string' ? image : 'Buffer'}`,
   );
-  return { width, height };
+  return { width, height, jimpImage };
 }
 
 /**
@@ -39,10 +49,18 @@ export async function imageInfo(image: string | Buffer): Promise<Size> {
  * @returns A Promise that resolves to an object containing the width and height of the image
  * @throws Error if the image data is invalid
  */
-export async function imageInfoOfBase64(imageBase64: string): Promise<Size> {
-  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '');
+export async function imageInfoOfBase64(
+  imageBase64: string,
+): Promise<ImageInfo> {
+  // const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '');
   // Call the imageInfo function to get the dimensions of the image
-  return imageInfo(Buffer.from(base64Data, 'base64'));
+  const buffer = await bufferFromBase64(imageBase64);
+  return imageInfo(buffer);
+}
+
+export async function bufferFromBase64(imageBase64: string): Promise<Buffer> {
+  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '');
+  return Buffer.from(base64Data, 'base64');
 }
 
 /**

package/src/img/jimp.d.ts

@@ -0,0 +1,4 @@
+declare module 'jimp/browser/lib/jimp.js' {
+  import Jimp from 'jimp';
+  export default Jimp;
+}