@midscene/shared 0.12.4 → 0.12.5-beta-20250306111616.0

package/dist/es/img/box-select.js

@@ -0,0 +1,187 @@
+import assert from "node:assert";
+import getJimp from "./get-jimp";
+import { bufferFromBase64, imageInfoOfBase64 } from "./index";
+let cachedFont = null;
+const createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5) => {
+  const Jimp = await getJimp();
+  const image = new Jimp(imageWidth, imageHeight, 0);
+  const colors = [
+    { rect: 3324182783, text: 4294967295 },
+    // red, white
+    { rect: 65535, text: 4294967295 },
+    // blue, white
+    { rect: 2336560127, text: 4294967295 },
+    // brown, white
+    { rect: 1048258559, text: 4294967295 },
+    // green, white
+    { rect: 1342206975, text: 4294967295 }
+    // purple, white
+  ];
+  for (let index = 0; index < elements.length; index++) {
+    const element = elements[index];
+    const color = colors[index % colors.length];
+    const paddedRect = {
+      left: Math.max(0, element.rect.left - boxPadding),
+      top: Math.max(0, element.rect.top - boxPadding),
+      width: Math.min(
+        imageWidth - element.rect.left,
+        element.rect.width + boxPadding * 2
+      ),
+      height: Math.min(
+        imageHeight - element.rect.top,
+        element.rect.height + boxPadding * 2
+      )
+    };
+    image.scan(
+      paddedRect.left,
+      paddedRect.top,
+      paddedRect.width,
+      paddedRect.height,
+      (x, y, idx) => {
+        if (x === paddedRect.left || x === paddedRect.left + paddedRect.width - 1 || y === paddedRect.top || y === paddedRect.top + paddedRect.height - 1) {
+          image.bitmap.data[idx + 0] = color.rect >> 24 & 255;
+          image.bitmap.data[idx + 1] = color.rect >> 16 & 255;
+          image.bitmap.data[idx + 2] = color.rect >> 8 & 255;
+          image.bitmap.data[idx + 3] = color.rect & 255;
+        }
+      }
+    );
+    const textWidth = element.indexId.toString().length * 8;
+    const textHeight = 12;
+    const rectWidth = textWidth + 5;
+    const rectHeight = textHeight + 4;
+    let rectX = paddedRect.left - rectWidth;
+    let rectY = paddedRect.top + paddedRect.height / 2 - textHeight / 2 - 2;
+    const checkOverlap = (x, y) => {
+      return elements.slice(0, index).some((otherElement) => {
+        return x < otherElement.rect.left + otherElement.rect.width && x + rectWidth > otherElement.rect.left && y < otherElement.rect.top + otherElement.rect.height && y + rectHeight > otherElement.rect.top;
+      });
+    };
+    const isWithinBounds = (x, y) => {
+      return x >= 0 && x + rectWidth <= imageWidth && y >= 0 && y + rectHeight <= imageHeight;
+    };
+    if (checkOverlap(rectX, rectY) || !isWithinBounds(rectX, rectY)) {
+      if (!checkOverlap(paddedRect.left, paddedRect.top - rectHeight - 2) && isWithinBounds(paddedRect.left, paddedRect.top - rectHeight - 2)) {
+        rectX = paddedRect.left;
+        rectY = paddedRect.top - rectHeight - 2;
+      } else if (!checkOverlap(
+        paddedRect.left,
+        paddedRect.top + paddedRect.height + 2
+      ) && isWithinBounds(paddedRect.left, paddedRect.top + paddedRect.height + 2)) {
+        rectX = paddedRect.left;
+        rectY = paddedRect.top + paddedRect.height + 2;
+      } else if (!checkOverlap(paddedRect.left + paddedRect.width + 2, paddedRect.top) && isWithinBounds(paddedRect.left + paddedRect.width + 2, paddedRect.top)) {
+        rectX = paddedRect.left + paddedRect.width + 2;
+        rectY = paddedRect.top;
+      } else {
+        rectX = paddedRect.left;
+        rectY = paddedRect.top + 2;
+      }
+    }
+    image.scan(
+      rectX,
+      rectY,
+      rectWidth,
+      rectHeight,
+      (x, y, idx) => {
+        image.bitmap.data[idx + 0] = color.rect >> 24 & 255;
+        image.bitmap.data[idx + 1] = color.rect >> 16 & 255;
+        image.bitmap.data[idx + 2] = color.rect >> 8 & 255;
+        image.bitmap.data[idx + 3] = color.rect & 255;
+      }
+    );
+    try {
+      cachedFont = cachedFont || await Jimp.loadFont(Jimp.FONT_SANS_16_WHITE);
+    } catch (error) {
+      console.error("Error loading font", error);
+    }
+    image.print(
+      cachedFont,
+      rectX,
+      rectY,
+      {
+        text: element.indexId.toString(),
+        alignmentX: Jimp.HORIZONTAL_ALIGN_CENTER,
+        alignmentY: Jimp.VERTICAL_ALIGN_MIDDLE
+      },
+      rectWidth,
+      rectHeight
+    );
+  }
+  return image;
+};
+const compositeElementInfoImg = async (options) => {
+  assert(options.inputImgBase64, "inputImgBase64 is required");
+  let width = 0;
+  let height = 0;
+  let jimpImage;
+  const Jimp = await getJimp();
+  if (options.size) {
+    width = options.size.width;
+    height = options.size.height;
+  }
+  if (!width || !height) {
+    const info = await imageInfoOfBase64(options.inputImgBase64);
+    width = info.width;
+    height = info.height;
+    jimpImage = info.jimpImage;
+  } else {
+    const imageBuffer = await bufferFromBase64(options.inputImgBase64);
+    jimpImage = await Jimp.read(imageBuffer);
+    const imageBitmap = jimpImage.bitmap;
+    if (imageBitmap.width !== width || imageBitmap.height !== height) {
+      jimpImage.resize(width, height, Jimp.RESIZE_NEAREST_NEIGHBOR);
+    }
+  }
+  if (!width || !height) {
+    throw Error("Image processing failed because width or height is undefined");
+  }
+  const { elementsPositionInfo } = options;
+  const result = await Promise.resolve(jimpImage).then(async (image) => {
+    const svgOverlay = await createSvgOverlay(
+      elementsPositionInfo,
+      width,
+      height,
+      options.annotationPadding
+    );
+    const svgImage = await Jimp.read(svgOverlay);
+    const compositeImage = await image.composite(svgImage, 0, 0, {
+      mode: Jimp.BLEND_SOURCE_OVER,
+      opacitySource: 1,
+      opacityDest: 1
+    });
+    return compositeImage;
+  }).then(async (compositeImage) => {
+    compositeImage.quality(90);
+    const base64 = await compositeImage.getBase64Async(Jimp.MIME_JPEG);
+    return base64;
+  }).catch((error) => {
+    throw error;
+  });
+  return result;
+};
+const processImageElementInfo = async (options) => {
+  const base64Image = options.inputImgBase64.split(";base64,").pop();
+  assert(base64Image, "base64Image is undefined");
+  const [
+    compositeElementInfoImgBase64,
+    compositeElementInfoImgWithoutTextBase64
+  ] = await Promise.all([
+    compositeElementInfoImg({
+      inputImgBase64: options.inputImgBase64,
+      elementsPositionInfo: options.elementsPositionInfo
+    }),
+    compositeElementInfoImg({
+      inputImgBase64: options.inputImgBase64,
+      elementsPositionInfo: options.elementsPositionInfoWithoutText
+    })
+  ]);
+  return {
+    compositeElementInfoImgBase64,
+    compositeElementInfoImgWithoutTextBase64
+  };
+};
+export {
+  compositeElementInfoImg,
+  processImageElementInfo
+};

package/dist/es/img/draw-box.js

@@ -0,0 +1,43 @@
+import getJimp from "./get-jimp";
+import { bufferFromBase64 } from "./info";
+import { saveBase64Image } from "./transform";
+async function drawBoxOnImage(options) {
+  const { inputImgBase64, rect } = options;
+  const color = { r: 255, g: 0, b: 0, a: 255 };
+  const Jimp = await getJimp();
+  const imageBuffer = await bufferFromBase64(inputImgBase64);
+  const image = await Jimp.read(imageBuffer);
+  const centerX = rect.x;
+  const centerY = rect.y;
+  const radius = 5;
+  image.scan(
+    Math.floor(centerX - radius),
+    Math.floor(centerY - radius),
+    radius * 2,
+    radius * 2,
+    (x, y, idx) => {
+      const distance = Math.sqrt((x - centerX) ** 2 + (y - centerY) ** 2);
+      if (distance <= radius) {
+        image.bitmap.data[idx + 0] = color.r;
+        image.bitmap.data[idx + 1] = color.g;
+        image.bitmap.data[idx + 2] = color.b;
+        image.bitmap.data[idx + 3] = color.a;
+      }
+    }
+  );
+  image.quality(90);
+  const resultBase64 = await image.getBase64Async(Jimp.MIME_JPEG);
+  return resultBase64;
+}
+async function savePositionImg(options) {
+  const { inputImgBase64, rect, outputPath } = options;
+  const imgBase64 = await drawBoxOnImage({ inputImgBase64, rect });
+  await saveBase64Image({
+    base64Data: imgBase64,
+    outputPath
+  });
+}
+export {
+  drawBoxOnImage,
+  savePositionImg
+};

package/dist/es/img/get-jimp.js

@@ -0,0 +1,11 @@
+const ifInBrowser = typeof window !== "undefined";
+async function getJimp() {
+  if (ifInBrowser) {
+    await import("jimp/browser/lib/jimp.js");
+    return window.Jimp;
+  }
+  return (await import("jimp")).default;
+}
+export {
+  getJimp as default
+};

package/dist/es/img/index.js

@@ -0,0 +1,34 @@
+import {
+  imageInfo,
+  imageInfoOfBase64,
+  bufferFromBase64,
+  base64Encoded
+} from "./info";
+import {
+  trimImage,
+  resizeImg,
+  resizeImgBase64,
+  transformImgPathToBase64,
+  zoomForGPT4o,
+  saveBase64Image,
+  paddingToMatchBlock
+} from "./transform";
+import { processImageElementInfo, compositeElementInfoImg } from "./box-select";
+import { drawBoxOnImage, savePositionImg } from "./draw-box";
+export {
+  base64Encoded,
+  bufferFromBase64,
+  compositeElementInfoImg,
+  drawBoxOnImage,
+  imageInfo,
+  imageInfoOfBase64,
+  paddingToMatchBlock,
+  processImageElementInfo,
+  resizeImg,
+  resizeImgBase64,
+  saveBase64Image,
+  savePositionImg,
+  transformImgPathToBase64,
+  trimImage,
+  zoomForGPT4o
+};

package/dist/es/img/info.js

@@ -0,0 +1,50 @@
+import assert from "node:assert";
+import { Buffer } from "node:buffer";
+import { readFileSync } from "node:fs";
+import getJimp from "./get-jimp";
+async function imageInfo(image) {
+  const Jimp = await getJimp();
+  let jimpImage;
+  if (typeof image === "string") {
+    jimpImage = await Jimp.read(image);
+  } else if (Buffer.isBuffer(image)) {
+    jimpImage = await Jimp.read(image);
+  } else if (image instanceof Jimp) {
+    jimpImage = image;
+  } else {
+    throw new Error("Invalid image input: must be a string path or a Buffer");
+  }
+  const { width, height } = jimpImage.bitmap;
+  assert(
+    width && height,
+    `Invalid image: ${typeof image === "string" ? image : "Buffer"}`
+  );
+  return { width, height, jimpImage };
+}
+async function imageInfoOfBase64(imageBase64) {
+  const buffer = await bufferFromBase64(imageBase64);
+  return imageInfo(buffer);
+}
+async function bufferFromBase64(imageBase64) {
+  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
+  return Buffer.from(base64Data, "base64");
+}
+function base64Encoded(image, withHeader = true) {
+  const imageBuffer = readFileSync(image);
+  if (!withHeader) {
+    return imageBuffer.toString("base64");
+  }
+  if (image.endsWith("png")) {
+    return `data:image/png;base64,${imageBuffer.toString("base64")}`;
+  }
+  if (image.endsWith("jpg") || image.endsWith("jpeg")) {
+    return `data:image/jpeg;base64,${imageBuffer.toString("base64")}`;
+  }
+  throw new Error("unsupported image type");
+}
+export {
+  base64Encoded,
+  bufferFromBase64,
+  imageInfo,
+  imageInfoOfBase64
+};

package/dist/es/img/transform.js

@@ -0,0 +1,128 @@
+import assert from "node:assert";
+import { Buffer } from "node:buffer";
+import getJimp from "./get-jimp";
+async function saveBase64Image(options) {
+  const { base64Data, outputPath } = options;
+  const base64Image = base64Data.split(";base64,").pop() || base64Data;
+  const imageBuffer = Buffer.from(base64Image, "base64");
+  const Jimp = await getJimp();
+  const image = await Jimp.read(imageBuffer);
+  await image.writeAsync(outputPath);
+}
+async function transformImgPathToBase64(inputPath) {
+  const Jimp = await getJimp();
+  const image = await Jimp.read(inputPath);
+  const buffer = await image.getBufferAsync(Jimp.MIME_JPEG);
+  return buffer.toString("base64");
+}
+async function resizeImg(inputData, newSize) {
+  if (typeof inputData === "string")
+    throw Error("inputData is base64, use resizeImgBase64 instead");
+  assert(
+    newSize && newSize.width > 0 && newSize.height > 0,
+    "newSize must be positive"
+  );
+  const Jimp = await getJimp();
+  const image = await Jimp.read(inputData);
+  const { width, height } = image.bitmap;
+  if (!width || !height) {
+    throw Error("Undefined width or height from the input image.");
+  }
+  if (newSize.width === width && newSize.height === height) {
+    return inputData;
+  }
+  image.resize(newSize.width, newSize.height, Jimp.RESIZE_NEAREST_NEIGHBOR);
+  image.quality(90);
+  const resizedBuffer = await image.getBufferAsync(Jimp.MIME_JPEG);
+  return resizedBuffer;
+}
+async function bufferFromBase64(base64) {
+  const splitFlag = ";base64,";
+  const dataSplitted = base64.split(splitFlag);
+  if (dataSplitted.length !== 2) {
+    throw Error("Invalid base64 data");
+  }
+  return Buffer.from(dataSplitted[1], "base64");
+}
+async function resizeImgBase64(inputBase64, newSize) {
+  const splitFlag = ";base64,";
+  const dataSplitted = inputBase64.split(splitFlag);
+  if (dataSplitted.length !== 2) {
+    throw Error("Invalid base64 data");
+  }
+  const imageBuffer = Buffer.from(dataSplitted[1], "base64");
+  const buffer = await resizeImg(imageBuffer, newSize);
+  const content = buffer.toString("base64");
+  return `${dataSplitted[0]}${splitFlag}${content}`;
+}
+function zoomForGPT4o(originalWidth, originalHeight) {
+  const maxWidth = 2048;
+  const maxHeight = 768;
+  let newWidth = originalWidth;
+  let newHeight = originalHeight;
+  const aspectRatio = originalWidth / originalHeight;
+  if (originalWidth > maxWidth) {
+    newWidth = maxWidth;
+    newHeight = newWidth / aspectRatio;
+  }
+  if (newHeight > maxHeight) {
+    newHeight = maxHeight;
+    newWidth = newHeight * aspectRatio;
+  }
+  return {
+    width: Math.round(newWidth),
+    height: Math.round(newHeight)
+  };
+}
+async function trimImage(image) {
+  const Jimp = await getJimp();
+  const jimpImage = await Jimp.read(
+    Buffer.isBuffer(image) ? image : Buffer.from(image)
+  );
+  const { width, height } = jimpImage.bitmap;
+  if (width <= 3 || height <= 3) {
+    return null;
+  }
+  const trimmedImage = jimpImage.autocrop();
+  const { width: trimmedWidth, height: trimmedHeight } = trimmedImage.bitmap;
+  const trimOffsetLeft = (width - trimmedWidth) / 2;
+  const trimOffsetTop = (height - trimmedHeight) / 2;
+  if (trimOffsetLeft === 0 && trimOffsetTop === 0) {
+    return null;
+  }
+  return {
+    trimOffsetLeft: -trimOffsetLeft,
+    trimOffsetTop: -trimOffsetTop,
+    width: trimmedWidth,
+    height: trimmedHeight
+  };
+}
+function prependBase64Header(base64, mimeType = "image/png") {
+  return `data:${mimeType};base64,${base64}`;
+}
+async function paddingToMatchBlock(imageBase64, blockSize = 28) {
+  const Jimp = await getJimp();
+  const imageBuffer = await bufferFromBase64(imageBase64);
+  const image = await Jimp.read(imageBuffer);
+  const { width, height } = image.bitmap;
+  const targetWidth = Math.ceil(width / blockSize) * blockSize;
+  const targetHeight = Math.ceil(height / blockSize) * blockSize;
+  if (targetWidth === width && targetHeight === height) {
+    return imageBase64;
+  }
+  const paddedImage = new Jimp(targetWidth, targetHeight, 4294967295);
+  paddedImage.composite(image, 0, 0);
+  const base64 = await paddedImage.getBase64Async(Jimp.MIME_JPEG);
+  return base64;
+}
+export {
+  bufferFromBase64,
+  paddingToMatchBlock,
+  prependBase64Header,
+  resizeImg,
+  resizeImgBase64,
+  saveBase64Image,
+  transformImgPathToBase64,
+  trimImage,
+  zoomForGPT4o
+};

package/dist/es/index.js

@@ -1,4 +1,3 @@
-// src/index.ts
 var src_default = {};
 export {
   src_default as default

package/dist/es/{fs.js → node/fs.js}

@@ -1,8 +1,10 @@
-// src/fs/index.ts
-import { existsSync, readFileSync } from "fs";
-import { dirname, join } from "path";
-var pkgCacheMap = {};
-var ifInBrowser = typeof window !== "undefined";
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { assert } from "../utils";
+const pkgCacheMap = {};
+const ifInBrowser = typeof window !== "undefined";
 function getRunningPkgInfo(dir) {
   if (ifInBrowser) {
     return null;
@@ -39,7 +41,22 @@ function findNearestPackageJson(dir) {
   }
   return findNearestPackageJson(parentDir);
 }
+async function getExtraReturnLogic(tree = false) {
+  if (ifInBrowser) {
+    return null;
+  }
+  const filename = fileURLToPath(import.meta.url);
+  const pathDir = findNearestPackageJson(dirname(filename));
+  assert(pathDir, `can't find pathDir, with ${dirname}`);
+  const scriptPath = path.join(pathDir, "./dist/script/htmlElement.js");
+  const elementInfosScriptContent = readFileSync(scriptPath, "utf-8");
+  if (tree) {
+    return `${elementInfosScriptContent}midscene_element_inspector.webExtractNodeTree()`;
+  }
+  return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
+}
 export {
   findNearestPackageJson,
+  getExtraReturnLogic,
   getRunningPkgInfo
 };

package/dist/es/types/index.js

@@ -0,0 +1,5 @@
+class BaseElement {
+}
+export {
+  BaseElement
+};

package/dist/es/us-keyboard-layout.js

@@ -1,5 +1,9 @@
-// src/us-keyboard-layout.ts
-var _keyDefinitions = {
+/**
+ * @license
+ * Copyright 2017 Google Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+const _keyDefinitions = {
   "0": { keyCode: 48, key: "0", code: "Digit0" },
   "1": { keyCode: 49, key: "1", code: "Digit1" },
   "2": { keyCode: 50, key: "2", code: "Digit2" },
@@ -384,7 +388,7 @@ var _keyDefinitions = {
   },
   VolumeUp: { keyCode: 183, key: "VolumeUp", code: "VolumeUp", location: 4 }
 };
-var lowerCaseKeyDefinitions = Object.entries(_keyDefinitions).reduce(
+const lowerCaseKeyDefinitions = Object.entries(_keyDefinitions).reduce(
   (acc, [key, definition]) => {
     const lowerKey = key.toLowerCase();
     if (lowerKey !== key) {
@@ -394,15 +398,15 @@ var lowerCaseKeyDefinitions = Object.entries(_keyDefinitions).reduce(
   },
   {}
 );
-var getKeyDefinition = (key) => {
+const getKeyDefinition = (key) => {
   const lowerKey = key.toLowerCase();
   if (lowerCaseKeyDefinitions[lowerKey]) {
     return lowerCaseKeyDefinitions[lowerKey].key;
   }
   return key;
 };
-var isMac = typeof window !== "undefined" ? /Mac|iPod|iPhone|iPad/.test(window.navigator.platform) : process.platform === "darwin";
-var keyMap = {
+const isMac = typeof window !== "undefined" ? /Mac|iPod|iPhone|iPad/.test(window.navigator.platform) : process.platform === "darwin";
+const keyMap = {
   return: _keyDefinitions.Enter.key,
   enter: _keyDefinitions.Enter.key,
   ctrl: isMac ? _keyDefinitions.Meta.key : _keyDefinitions.Control.key,
@@ -428,8 +432,3 @@ export {
   isMac,
   transformHotkeyInput
 };
-/**
- * @license
- * Copyright 2017 Google Inc.
- * SPDX-License-Identifier: Apache-2.0
- */