@midscene/shared 0.4.0

package/dist/types/img.d.ts

@@ -0,0 +1,111 @@
+import { Buffer } from 'node:buffer';
+import { NodeType } from './constants.js';
+
+interface Size {
+    width: number;
+    height: number;
+}
+/**
+ * Retrieves the dimensions of an image asynchronously
+ *
+ * @param image - The image data, which can be a string path or a buffer
+ * @returns A Promise that resolves to an object containing the width and height of the image
+ * @throws Error if the image data is invalid
+ */
+declare function imageInfo(image: string | Buffer): Promise<Size>;
+/**
+ * Retrieves the dimensions of an image from a base64-encoded string
+ *
+ * @param imageBase64 - The base64-encoded image data
+ * @returns A Promise that resolves to an object containing the width and height of the image
+ * @throws Error if the image data is invalid
+ */
+declare function imageInfoOfBase64(imageBase64: string): Promise<Size>;
+/**
+ * Encodes an image file to a base64 encoded string
+ *
+ * @param image The path of the image file
+ * @param withHeader Determine whether to return data including the file header information, the default is true
+ *
+ * @returns The base64 encoded string of the image file, which may or may not include header information depending on the withHeader parameter
+ *
+ * @throws When the image type is not supported, an error will be thrown
+ */
+declare function base64Encoded(image: string, withHeader?: boolean): string;
+
+/**
+ * Saves a Base64-encoded image to a file
+ *
+ * @param options - An object containing the Base64-encoded image data and the output file path
+ * @param options.base64Data - The Base64-encoded image data
+ * @param options.outputPath - The path where the image will be saved
+ * @throws Error if there is an error during the saving process
+ */
+declare function saveBase64Image(options: {
+    base64Data: string;
+    outputPath: string;
+}): Promise<void>;
+/**
+ * Transforms an image path into a base64-encoded string
+ * @param inputPath - The path of the image file to be encoded
+ * @returns A Promise that resolves to a base64-encoded string representing the image file
+ */
+declare function transformImgPathToBase64(inputPath: string): Promise<string>;
+/**
+ * Resizes an image from a base64-encoded string
+ *
+ * @param base64Data - A base64-encoded string representing the image
+ * @returns A Promise that resolves to a base64-encoded string representing the resized image
+ * @throws An error if the width or height cannot be determined from the metadata
+ */
+declare function resizeImg(base64Data: string): Promise<string>;
+/**
+ * Calculates new dimensions for an image while maintaining its aspect ratio.
+ *
+ * This function is designed to resize an image to fit within a specified maximum width and height
+ * while maintaining the original aspect ratio. If the original width or height exceeds the maximum
+ * dimensions, the image will be scaled down to fit.
+ *
+ * @param {number} originalWidth - The original width of the image.
+ * @param {number} originalHeight - The original height of the image.
+ * @returns {Object} An object containing the new width and height.
+ * @throws {Error} Throws an error if the width or height is not a positive number.
+ */
+declare function calculateNewDimensions(originalWidth: number, originalHeight: number): {
+    width: number;
+    height: number;
+};
+/**
+ * Trims an image and returns the trimming information, including the offset from the left and top edges, and the trimmed width and height
+ *
+ * @param image - The image to be trimmed. This can be a file path or a Buffer object containing the image data
+ * @returns A Promise that resolves to an object containing the trimming information. If the image does not need to be trimmed, this object will be null
+ */
+declare function trimImage(image: string | Buffer): Promise<{
+    trimOffsetLeft: number;
+    trimOffsetTop: number;
+    width: number;
+    height: number;
+} | null>;
+
+type ElementType = {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    label: string;
+    attributes: {
+        [key: string]: string;
+        nodeType: NodeType;
+    };
+};
+declare const processImageElementInfo: (options: {
+    inputImgBase64: string;
+    elementsPositionInfo: Array<ElementType>;
+    elementsPositionInfoWithoutText: Array<ElementType>;
+}) => Promise<{
+    compositeElementInfoImgBase64: string;
+    compositeElementInfoImgWithoutTextBase64: string;
+}>;
+
+export { base64Encoded, calculateNewDimensions, imageInfo, imageInfoOfBase64, processImageElementInfo, resizeImg, saveBase64Image, transformImgPathToBase64, trimImage };

package/dist/types/index.d.ts

@@ -0,0 +1,3 @@
+declare function export_default(): string;
+
+export { export_default as default };

package/package.json

@@ -0,0 +1,74 @@
+{
+  "name": "@midscene/shared",
+  "version": "0.4.0",
+  "types": "./dist/types/index.d.ts",
+  "main": "./dist/lib/index.js",
+  "module": "./dist/es/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/es/index.js",
+      "require": "./dist/lib/index.js"
+    },
+    "./constants": {
+      "types": "./dist/types/constants.d.ts",
+      "import": "./dist/es/constants.js",
+      "require": "./dist/lib/constants.js"
+    },
+    "./img": {
+      "types": "./dist/types/img.d.ts",
+      "import": "./dist/es/img.js",
+      "require": "./dist/lib/img.js"
+    }
+  },
+  "typesVersions": {
+    "*": {
+      ".": [
+        "./dist/types/index.d.ts"
+      ],
+      "constants": [
+        "./dist/types/constants.d.ts"
+      ],
+      "img": [
+        "./dist/types/img.d.ts"
+      ]
+    }
+  },
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "dependencies": {
+    "jimp": "0.22.12"
+  },
+  "peerDependencies": {},
+  "devDependencies": {
+    "@modern-js/module-tools": "2.58.2",
+    "typescript": "~5.0.4",
+    "@types/node": "^18.0.0",
+    "rimraf": "~3.0.2",
+    "vitest": "^1.6.0"
+  },
+  "sideEffects": [],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "scripts": {
+    "dev": "modern dev",
+    "build": "modern build",
+    "build:watch": "modern build -w",
+    "reset": "rimraf ./**/node_modules",
+    "lint": "modern lint",
+    "bump": "modern bump",
+    "pre": "modern pre",
+    "change-status": "modern change-status",
+    "gen-release-note": "modern gen-release-note",
+    "release": "modern release",
+    "new": "modern new",
+    "upgrade": "modern upgrade",
+    "test": "vitest --run",
+    "test:ai": "AITEST=true npm run test"
+  }
+}

package/src/constants/index.ts

@@ -0,0 +1,10 @@
+export const TEXT_SIZE_THRESHOLD = 9;
+
+export const TEXT_MAX_SIZE = 40;
+
+export enum NodeType {
+  FORM_ITEM = 'FORM_ITEM Node',
+  BUTTON = 'BUTTON Node',
+  IMG = 'IMG Node',
+  TEXT = 'TEXT Node',
+}

package/src/img/box-select.ts

@@ -0,0 +1,179 @@
+import assert from 'node:assert';
+import { Buffer } from 'node:buffer';
+import Jimp from 'jimp';
+import type { NodeType } from '../constants';
+
+// Define picture path
+type ElementType = {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  label: string;
+  attributes: {
+    [key: string]: string;
+    nodeType: NodeType;
+  };
+};
+
+const createSvgOverlay = (
+  elements: Array<ElementType>,
+  imageWidth: number,
+  imageHeight: number,
+) => {
+  const createPngOverlay = async (
+    elements: Array<ElementType>,
+    imageWidth: number,
+    imageHeight: number,
+  ) => {
+    const image = new Jimp(imageWidth, imageHeight, 0x00000000);
+
+    // Define color array
+    const colors = [
+      { rect: 0x0000ffff, text: 0xffffffff }, // blue, white
+      { rect: 0x8b4513ff, text: 0xffffffff }, // brown, white
+    ];
+
+    for (let index = 0; index < elements.length; index++) {
+      const element = elements[index];
+      const color = colors[index % colors.length];
+
+      // Draw rectangle
+      image.scan(
+        element.x,
+        element.y,
+        element.width,
+        element.height,
+        function (x, y, idx) {
+          if (
+            x === element.x ||
+            x === element.x + element.width - 1 ||
+            y === element.y ||
+            y === element.y + element.height - 1
+          ) {
+            this.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
+            this.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
+            this.bitmap.data[idx + 2] = (color.rect >> 8) & 0xff; // B
+            this.bitmap.data[idx + 3] = color.rect & 0xff; // A
+          }
+        },
+      );
+
+      // Calculate text position
+      const textWidth = element.label.length * 8;
+      const textHeight = 12;
+      const rectWidth = textWidth + 5;
+      const rectHeight = textHeight + 4;
+      let rectX = element.x - rectWidth;
+      let rectY = element.y + element.height / 2 - textHeight / 2 - 2;
+
+      // Check if obscured by the left
+      if (rectX < 0) {
+        rectX = element.x;
+        rectY = element.y - rectHeight;
+      }
+
+      // Draw text background
+      image.scan(rectX, rectY, rectWidth, rectHeight, function (x, y, idx) {
+        this.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
+        this.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
+        this.bitmap.data[idx + 2] = (color.rect >> 8) & 0xff; // B
+        this.bitmap.data[idx + 3] = color.rect & 0xff; // A
+      });
+      // Draw text (simplified, as Jimp doesn't have built-in text drawing)
+      const font = await Jimp.loadFont(Jimp.FONT_SANS_16_WHITE);
+      image.print(
+        font,
+        rectX,
+        rectY,
+        {
+          text: element.label,
+          alignmentX: Jimp.HORIZONTAL_ALIGN_CENTER,
+          alignmentY: Jimp.VERTICAL_ALIGN_MIDDLE,
+        },
+        rectWidth,
+        rectHeight,
+      );
+    }
+
+    return image.getBufferAsync(Jimp.MIME_PNG);
+  };
+
+  return createPngOverlay(elements, imageWidth, imageHeight);
+};
+
+export const processImageElementInfo = async (options: {
+  inputImgBase64: string;
+  elementsPositionInfo: Array<ElementType>;
+  elementsPositionInfoWithoutText: Array<ElementType>;
+}) => {
+  // Get the size of the original image
+  const base64Image = options.inputImgBase64.split(';base64,').pop();
+  assert(base64Image, 'base64Image is undefined');
+
+  const imageBuffer = Buffer.from(base64Image, 'base64');
+  const image = await Jimp.read(imageBuffer);
+  const { width, height } = image.bitmap;
+
+  if (width && height) {
+    // Create svg overlay
+    const svgOverlay = await createSvgOverlay(
+      options.elementsPositionInfo,
+      width,
+      height,
+    );
+    const svgOverlayWithoutText = await createSvgOverlay(
+      options.elementsPositionInfoWithoutText,
+      width,
+      height,
+    );
+
+    // Composite picture
+    const compositeElementInfoImgBase64 = await Jimp.read(imageBuffer)
+      .then(async (image: Jimp) => {
+        const svgImage = await Jimp.read(svgOverlay);
+        return image.composite(svgImage, 0, 0, {
+          mode: Jimp.BLEND_SOURCE_OVER,
+          opacitySource: 1,
+          opacityDest: 1,
+        });
+      })
+      .then((compositeImage: Jimp) => {
+        return compositeImage.getBufferAsync(Jimp.MIME_PNG);
+      })
+      .then((buffer: Buffer) => {
+        return buffer.toString('base64');
+      })
+      .catch((error: unknown) => {
+        throw error;
+      });
+
+    // Composite picture withoutText
+    const compositeElementInfoImgWithoutTextBase64 = await Jimp.read(
+      imageBuffer,
+    )
+      .then(async (image: Jimp) => {
+        const svgImage = await Jimp.read(svgOverlayWithoutText);
+        return image.composite(svgImage, 0, 0, {
+          mode: Jimp.BLEND_SOURCE_OVER,
+          opacitySource: 1,
+          opacityDest: 1,
+        });
+      })
+      .then((compositeImage: Jimp) => {
+        return compositeImage.getBufferAsync(Jimp.MIME_PNG);
+      })
+      .then((buffer: Buffer) => {
+        return buffer.toString('base64');
+      })
+      .catch((error: unknown) => {
+        throw error;
+      });
+
+    return {
+      compositeElementInfoImgBase64,
+      compositeElementInfoImgWithoutTextBase64,
+    };
+  }
+  throw Error('Image processing failed because width or height is undefined');
+};

package/src/img/index.ts

@@ -0,0 +1,9 @@
+export { imageInfo, imageInfoOfBase64, base64Encoded } from './info';
+export {
+  trimImage,
+  calculateNewDimensions,
+  resizeImg,
+  transformImgPathToBase64,
+  saveBase64Image,
+} from './transform';
+export { processImageElementInfo } from './box-select';

package/src/img/info.ts

@@ -0,0 +1,71 @@
+import assert from 'node:assert';
+import { Buffer } from 'node:buffer';
+import { readFileSync } from 'node:fs';
+import Jimp from 'jimp';
+
+export interface Size {
+  width: number;
+  height: number;
+}
+
+/**
+ * Retrieves the dimensions of an image asynchronously
+ *
+ * @param image - The image data, which can be a string path or a buffer
+ * @returns A Promise that resolves to an object containing the width and height of the image
+ * @throws Error if the image data is invalid
+ */
+export async function imageInfo(image: string | Buffer): Promise<Size> {
+  let jimpImage: Jimp;
+  if (typeof image === 'string') {
+    jimpImage = await Jimp.read(image);
+  } else if (Buffer.isBuffer(image)) {
+    jimpImage = await Jimp.read(image);
+  } else {
+    throw new Error('Invalid image input: must be a string path or a Buffer');
+  }
+  const { width, height } = jimpImage.bitmap;
+  assert(
+    width && height,
+    `Invalid image: ${typeof image === 'string' ? image : 'Buffer'}`,
+  );
+  return { width, height };
+}
+
+/**
+ * Retrieves the dimensions of an image from a base64-encoded string
+ *
+ * @param imageBase64 - The base64-encoded image data
+ * @returns A Promise that resolves to an object containing the width and height of the image
+ * @throws Error if the image data is invalid
+ */
+export async function imageInfoOfBase64(imageBase64: string): Promise<Size> {
+  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '');
+  // Call the imageInfo function to get the dimensions of the image
+  return imageInfo(Buffer.from(base64Data, 'base64'));
+}
+
+/**
+ * Encodes an image file to a base64 encoded string
+ *
+ * @param image The path of the image file
+ * @param withHeader Determine whether to return data including the file header information, the default is true
+ *
+ * @returns The base64 encoded string of the image file, which may or may not include header information depending on the withHeader parameter
+ *
+ * @throws When the image type is not supported, an error will be thrown
+ */
+export function base64Encoded(image: string, withHeader = true) {
+  // get base64 encoded image
+  const imageBuffer = readFileSync(image);
+  if (!withHeader) {
+    return imageBuffer.toString('base64');
+  }
+  if (image.endsWith('png')) {
+    return `data:image/png;base64,${imageBuffer.toString('base64')}`;
+  }
+  if (image.endsWith('jpg') || image.endsWith('jpeg')) {
+    return `data:image/jpeg;base64,${imageBuffer.toString('base64')}`;
+  }
+  throw new Error('unsupported image type');
+}