@midscene/shared 1.3.5-beta-20260202030636.0 → 1.3.5-beta-20260202075348.0

package/dist/es/img/box-select.mjs

@@ -1,64 +1,698 @@
 import node_assert from "node:assert";
-import get_jimp from "./get-jimp.mjs";
-import { bufferFromBase64, imageInfoOfBase64 } from "./index.mjs";
-import { readImageBuffer } from "./safe-jimp.mjs";
-let cachedFont = null;
-const loadFonts = async ()=>{
-    const Jimp = await get_jimp();
-    try {
-        const fonts = await Jimp.loadFont(Jimp.FONT_SANS_16_WHITE);
-        return fonts;
-    } catch (error) {
-        console.warn('Error loading font, will try to load online fonts', error);
-        const onlineFonts = 'https://cdn.jsdelivr.net/npm/jimp-compact@0.16.1-2/fonts/open-sans/open-sans-16-white/open-sans-16-white.fnt';
-        const fonts = await Jimp.loadFont(onlineFonts);
-        return fonts;
-    }
+import { NodeType } from "../constants/index.mjs";
+import get_photon from "./get-photon.mjs";
+import { photonFromBase64, photonToBase64 } from "./transform.mjs";
+const DIGIT_FONT = {
+    0: [
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ]
+    ],
+    1: [
+        [
+            0,
+            0,
+            1,
+            0,
+            0
+        ],
+        [
+            0,
+            1,
+            1,
+            0,
+            0
+        ],
+        [
+            0,
+            0,
+            1,
+            0,
+            0
+        ],
+        [
+            0,
+            0,
+            1,
+            0,
+            0
+        ],
+        [
+            0,
+            0,
+            1,
+            0,
+            0
+        ],
+        [
+            0,
+            0,
+            1,
+            0,
+            0
+        ],
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ]
+    ],
+    2: [
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            0,
+            1,
+            1,
+            0
+        ],
+        [
+            0,
+            1,
+            0,
+            0,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            0
+        ],
+        [
+            1,
+            1,
+            1,
+            1,
+            1
+        ]
+    ],
+    3: [
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            0,
+            1,
+            1,
+            0
+        ],
+        [
+            0,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ]
+    ],
+    4: [
+        [
+            0,
+            0,
+            0,
+            1,
+            0
+        ],
+        [
+            0,
+            0,
+            1,
+            1,
+            0
+        ],
+        [
+            0,
+            1,
+            0,
+            1,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            1,
+            0
+        ],
+        [
+            1,
+            1,
+            1,
+            1,
+            1
+        ],
+        [
+            0,
+            0,
+            0,
+            1,
+            0
+        ],
+        [
+            0,
+            0,
+            0,
+            1,
+            0
+        ]
+    ],
+    5: [
+        [
+            1,
+            1,
+            1,
+            1,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            0
+        ],
+        [
+            1,
+            1,
+            1,
+            1,
+            0
+        ],
+        [
+            0,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ]
+    ],
+    6: [
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            0
+        ],
+        [
+            1,
+            1,
+            1,
+            1,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ]
+    ],
+    7: [
+        [
+            1,
+            1,
+            1,
+            1,
+            1
+        ],
+        [
+            0,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            0,
+            0,
+            1,
+            0
+        ],
+        [
+            0,
+            0,
+            1,
+            0,
+            0
+        ],
+        [
+            0,
+            0,
+            1,
+            0,
+            0
+        ],
+        [
+            0,
+            0,
+            1,
+            0,
+            0
+        ],
+        [
+            0,
+            0,
+            1,
+            0,
+            0
+        ]
+    ],
+    8: [
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ]
+    ],
+    9: [
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            1,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            1,
+            1,
+            1,
+            1
+        ],
+        [
+            0,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            0,
+            0,
+            0,
+            1
+        ],
+        [
+            0,
+            1,
+            1,
+            1,
+            0
+        ]
+    ]
 };
+const FONT_WIDTH = 5;
+const FONT_HEIGHT = 7;
+const FONT_SCALE = 2;
+function drawDigit(pixels, width, height, digit, startX, startY, color) {
+    const bitmap = DIGIT_FONT[digit];
+    if (!bitmap) return;
+    for(let row = 0; row < FONT_HEIGHT; row++)for(let col = 0; col < FONT_WIDTH; col++)if (1 === bitmap[row][col]) for(let sy = 0; sy < FONT_SCALE; sy++)for(let sx = 0; sx < FONT_SCALE; sx++){
+        const x = startX + col * FONT_SCALE + sx;
+        const y = startY + row * FONT_SCALE + sy;
+        if (x >= 0 && x < width && y >= 0 && y < height) {
+            const idx = (y * width + x) * 4;
+            pixels[idx + 0] = color.r;
+            pixels[idx + 1] = color.g;
+            pixels[idx + 2] = color.b;
+            pixels[idx + 3] = color.a;
+        }
+    }
+}
+function drawNumber(pixels, width, height, num, startX, startY, color) {
+    const str = num.toString();
+    let x = startX;
+    for (const digit of str){
+        drawDigit(pixels, width, height, digit, x, startY, color);
+        x += FONT_WIDTH * FONT_SCALE + 1;
+    }
+}
+function getNumberWidth(num) {
+    return num.toString().length * (FONT_WIDTH * FONT_SCALE + 1) - 1;
+}
+function drawRect(pixels, width, height, rect, color, thickness) {
+    const x = Math.floor(rect.x);
+    const y = Math.floor(rect.y);
+    const w = Math.floor(rect.w);
+    const h = Math.floor(rect.h);
+    for(let py = y; py < y + h && py < height; py++)for(let px = x; px < x + w && px < width; px++){
+        if (px < 0 || py < 0) continue;
+        const isLeftBorder = px >= x && px < x + thickness;
+        const isRightBorder = px <= x + w - 1 && px > x + w - thickness - 1;
+        const isTopBorder = py >= y && py < y + thickness;
+        const isBottomBorder = py <= y + h - 1 && py > y + h - thickness - 1;
+        if (isLeftBorder || isRightBorder || isTopBorder || isBottomBorder) {
+            const idx = (py * width + px) * 4;
+            pixels[idx + 0] = color.r;
+            pixels[idx + 1] = color.g;
+            pixels[idx + 2] = color.b;
+            pixels[idx + 3] = color.a;
+        }
+    }
+}
+function fillRect(pixels, width, height, rect, color) {
+    const x = Math.floor(rect.x);
+    const y = Math.floor(rect.y);
+    const w = Math.floor(rect.w);
+    const h = Math.floor(rect.h);
+    for(let py = y; py < y + h && py < height; py++)for(let px = x; px < x + w && px < width; px++){
+        if (px < 0 || py < 0) continue;
+        const idx = (py * width + px) * 4;
+        pixels[idx + 0] = color.r;
+        pixels[idx + 1] = color.g;
+        pixels[idx + 2] = color.b;
+        pixels[idx + 3] = color.a;
+    }
+}
+function blendPixels(basePixels, overlayPixels, width, height) {
+    const result = new Uint8Array(basePixels.length);
+    for(let i = 0; i < basePixels.length; i += 4){
+        const overlayAlpha = overlayPixels[i + 3] / 255;
+        const baseAlpha = basePixels[i + 3] / 255;
+        if (0 === overlayAlpha) {
+            result[i + 0] = basePixels[i + 0];
+            result[i + 1] = basePixels[i + 1];
+            result[i + 2] = basePixels[i + 2];
+            result[i + 3] = basePixels[i + 3];
+        } else {
+            const outAlpha = overlayAlpha + baseAlpha * (1 - overlayAlpha);
+            if (0 === outAlpha) {
+                result[i + 0] = overlayPixels[i + 0];
+                result[i + 1] = overlayPixels[i + 1];
+                result[i + 2] = overlayPixels[i + 2];
+                result[i + 3] = overlayPixels[i + 3];
+            } else {
+                result[i + 0] = Math.round((overlayPixels[i + 0] * overlayAlpha + basePixels[i + 0] * baseAlpha * (1 - overlayAlpha)) / outAlpha);
+                result[i + 1] = Math.round((overlayPixels[i + 1] * overlayAlpha + basePixels[i + 1] * baseAlpha * (1 - overlayAlpha)) / outAlpha);
+                result[i + 2] = Math.round((overlayPixels[i + 2] * overlayAlpha + basePixels[i + 2] * baseAlpha * (1 - overlayAlpha)) / outAlpha);
+                result[i + 3] = Math.round(255 * outAlpha);
+            }
+        }
+    }
+    return result;
+}
 const createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5, borderThickness = 2, prompt)=>{
-    const Jimp = await get_jimp();
-    const image = new Jimp(imageWidth, imageHeight, 0x00000000);
+    const overlayPixels = new Uint8Array(imageWidth * imageHeight * 4);
     const colors = [
         {
-            rect: 0xc62300ff,
-            text: 0xffffffff
+            rect: {
+                r: 0xc6,
+                g: 0x23,
+                b: 0x00,
+                a: 0xff
+            },
+            text: {
+                r: 0xff,
+                g: 0xff,
+                b: 0xff,
+                a: 0xff
+            }
         },
         {
-            rect: 0x0000ffff,
-            text: 0xffffffff
+            rect: {
+                r: 0x00,
+                g: 0x00,
+                b: 0xff,
+                a: 0xff
+            },
+            text: {
+                r: 0xff,
+                g: 0xff,
+                b: 0xff,
+                a: 0xff
+            }
         },
         {
-            rect: 0x8b4513ff,
-            text: 0xffffffff
+            rect: {
+                r: 0x8b,
+                g: 0x45,
+                b: 0x13,
+                a: 0xff
+            },
+            text: {
+                r: 0xff,
+                g: 0xff,
+                b: 0xff,
+                a: 0xff
+            }
         },
         {
-            rect: 0x3e7b27ff,
-            text: 0xffffffff
+            rect: {
+                r: 0x3e,
+                g: 0x7b,
+                b: 0x27,
+                a: 0xff
+            },
+            text: {
+                r: 0xff,
+                g: 0xff,
+                b: 0xff,
+                a: 0xff
+            }
         },
         {
-            rect: 0x500073ff,
-            text: 0xffffffff
+            rect: {
+                r: 0x50,
+                g: 0x00,
+                b: 0x73,
+                a: 0xff
+            },
+            text: {
+                r: 0xff,
+                g: 0xff,
+                b: 0xff,
+                a: 0xff
+            }
         }
     ];
-    if (prompt) try {
-        cachedFont = cachedFont || await loadFonts();
-        const promptPadding = 10;
+    if (prompt) {
         const promptMargin = 20;
         const promptHeight = 30;
         const promptY = imageHeight - promptHeight - promptMargin;
-        image.scan(0, promptY, imageWidth, promptHeight, (x, y, idx)=>{
-            image.bitmap.data[idx + 0] = 0x00;
-            image.bitmap.data[idx + 1] = 0x00;
-            image.bitmap.data[idx + 2] = 0x00;
-            image.bitmap.data[idx + 3] = 0xcc;
+        fillRect(overlayPixels, imageWidth, imageHeight, {
+            x: 0,
+            y: promptY,
+            w: imageWidth,
+            h: promptHeight
+        }, {
+            r: 0x00,
+            g: 0x00,
+            b: 0x00,
+            a: 0xcc
         });
-        image.print(cachedFont, promptPadding, promptY, {
-            text: prompt,
-            alignmentX: Jimp.HORIZONTAL_ALIGN_LEFT,
-            alignmentY: Jimp.VERTICAL_ALIGN_MIDDLE
-        }, imageWidth - 2 * promptPadding, promptHeight);
-    } catch (error) {
-        console.error('Error drawing prompt text', error);
     }
     for(let index = 0; index < elements.length; index++){
         const element = elements[index];
@@ -68,101 +702,85 @@ const createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding =
         const paddedWidth = Math.min(imageWidth - paddedLeft, element.rect.width + 2 * boxPadding);
         const paddedHeight = Math.min(imageHeight - paddedTop, element.rect.height + 2 * boxPadding);
         const paddedRect = {
-            left: paddedLeft,
-            top: paddedTop,
-            width: paddedWidth,
-            height: paddedHeight
+            x: paddedLeft,
+            y: paddedTop,
+            w: paddedWidth,
+            h: paddedHeight
         };
-        image.scan(paddedRect.left, paddedRect.top, paddedRect.width, paddedRect.height, (x, y, idx)=>{
-            if (x >= paddedRect.left && x < paddedRect.left + borderThickness || x <= paddedRect.left + paddedRect.width - 1 && x > paddedRect.left + paddedRect.width - borderThickness || y >= paddedRect.top && y < paddedRect.top + borderThickness || y <= paddedRect.top + paddedRect.height - 1 && y > paddedRect.top + paddedRect.height - borderThickness) {
-                image.bitmap.data[idx + 0] = color.rect >> 24 & 0xff;
-                image.bitmap.data[idx + 1] = color.rect >> 16 & 0xff;
-                image.bitmap.data[idx + 2] = color.rect >> 8 & 0xff;
-                image.bitmap.data[idx + 3] = 0xff & color.rect;
-            }
-        });
+        drawRect(overlayPixels, imageWidth, imageHeight, paddedRect, color.rect, borderThickness);
         const indexId = element.indexId;
         if ('number' != typeof indexId) continue;
-        const textWidth = 8 * indexId.toString().length;
-        const textHeight = 12;
+        const textWidth = getNumberWidth(indexId);
+        const textHeight = FONT_HEIGHT * FONT_SCALE;
         const rectWidth = textWidth + 5;
         const rectHeight = textHeight + 4;
-        let rectX = paddedRect.left - rectWidth;
-        let rectY = paddedRect.top + paddedRect.height / 2 - textHeight / 2 - 2;
+        let rectX = paddedLeft - rectWidth;
+        let rectY = paddedTop + Math.floor(paddedHeight / 2) - Math.floor(textHeight / 2) - 2;
         const checkOverlap = (x, y)=>elements.slice(0, index).some((otherElement)=>x < otherElement.rect.left + otherElement.rect.width && x + rectWidth > otherElement.rect.left && y < otherElement.rect.top + otherElement.rect.height && y + rectHeight > otherElement.rect.top);
         const isWithinBounds = (x, y)=>x >= 0 && x + rectWidth <= imageWidth && y >= 0 && y + rectHeight <= imageHeight;
-        if (checkOverlap(rectX, rectY) || !isWithinBounds(rectX, rectY)) if (!checkOverlap(paddedRect.left, paddedRect.top - rectHeight - 2) && isWithinBounds(paddedRect.left, paddedRect.top - rectHeight - 2)) {
-            rectX = paddedRect.left;
-            rectY = paddedRect.top - rectHeight - 2;
-        } else if (!checkOverlap(paddedRect.left, paddedRect.top + paddedRect.height + 2) && isWithinBounds(paddedRect.left, paddedRect.top + paddedRect.height + 2)) {
-            rectX = paddedRect.left;
-            rectY = paddedRect.top + paddedRect.height + 2;
-        } else if (!checkOverlap(paddedRect.left + paddedRect.width + 2, paddedRect.top) && isWithinBounds(paddedRect.left + paddedRect.width + 2, paddedRect.top)) {
-            rectX = paddedRect.left + paddedRect.width + 2;
-            rectY = paddedRect.top;
+        if (checkOverlap(rectX, rectY) || !isWithinBounds(rectX, rectY)) if (!checkOverlap(paddedLeft, paddedTop - rectHeight - 2) && isWithinBounds(paddedLeft, paddedTop - rectHeight - 2)) {
+            rectX = paddedLeft;
+            rectY = paddedTop - rectHeight - 2;
+        } else if (!checkOverlap(paddedLeft, paddedTop + paddedHeight + 2) && isWithinBounds(paddedLeft, paddedTop + paddedHeight + 2)) {
+            rectX = paddedLeft;
+            rectY = paddedTop + paddedHeight + 2;
+        } else if (!checkOverlap(paddedLeft + paddedWidth + 2, paddedTop) && isWithinBounds(paddedLeft + paddedWidth + 2, paddedTop)) {
+            rectX = paddedLeft + paddedWidth + 2;
+            rectY = paddedTop;
         } else {
-            rectX = paddedRect.left;
-            rectY = paddedRect.top + 2;
-        }
-        image.scan(rectX, rectY, rectWidth, rectHeight, (x, y, idx)=>{
-            image.bitmap.data[idx + 0] = color.rect >> 24 & 0xff;
-            image.bitmap.data[idx + 1] = color.rect >> 16 & 0xff;
-            image.bitmap.data[idx + 2] = color.rect >> 8 & 0xff;
-            image.bitmap.data[idx + 3] = 0xff & color.rect;
-        });
-        try {
-            cachedFont = cachedFont || await loadFonts();
-        } catch (error) {
-            console.error('Error loading font', error);
+            rectX = paddedLeft;
+            rectY = paddedTop + 2;
         }
-        image.print(cachedFont, rectX, rectY, {
-            text: indexId.toString(),
-            alignmentX: Jimp.HORIZONTAL_ALIGN_CENTER,
-            alignmentY: Jimp.VERTICAL_ALIGN_MIDDLE
-        }, rectWidth, rectHeight);
+        fillRect(overlayPixels, imageWidth, imageHeight, {
+            x: rectX,
+            y: rectY,
+            w: rectWidth,
+            h: rectHeight
+        }, color.rect);
+        const textX = rectX + Math.floor((rectWidth - textWidth) / 2);
+        const textY = rectY + Math.floor((rectHeight - textHeight) / 2);
+        drawNumber(overlayPixels, imageWidth, imageHeight, indexId, textX, textY, color.text);
     }
-    return image;
+    return overlayPixels;
 };
 const compositeElementInfoImg = async (options)=>{
     node_assert(options.inputImgBase64, 'inputImgBase64 is required');
+    const { PhotonImage, SamplingFilter, resize } = await get_photon();
     let width = 0;
     let height = 0;
-    let jimpImage;
-    const Jimp = await get_jimp();
     if (options.size) {
         width = options.size.width;
         height = options.size.height;
     }
+    let photonImage = await photonFromBase64(options.inputImgBase64);
     if (width && height) {
-        const imageBuffer = await bufferFromBase64(options.inputImgBase64);
-        jimpImage = await readImageBuffer(imageBuffer, Jimp);
-        const imageBitmap = jimpImage.bitmap;
-        if (imageBitmap.width !== width || imageBitmap.height !== height) jimpImage.resize(width, height, Jimp.RESIZE_NEAREST_NEIGHBOR);
+        const imageWidth = photonImage.get_width();
+        const imageHeight = photonImage.get_height();
+        if (imageWidth !== width || imageHeight !== height) {
+            const resized = resize(photonImage, width, height, SamplingFilter.Nearest);
+            photonImage.free();
+            photonImage = resized;
+        }
     } else {
-        const info = await imageInfoOfBase64(options.inputImgBase64);
-        width = info.width;
-        height = info.height;
-        jimpImage = info.jimpImage;
+        width = photonImage.get_width();
+        height = photonImage.get_height();
+    }
+    if (!width || !height) {
+        photonImage.free();
+        throw Error('Image processing failed because width or height is undefined');
     }
-    if (!width || !height) throw Error('Image processing failed because width or height is undefined');
     const { elementsPositionInfo, prompt } = options;
-    const result = await Promise.resolve(jimpImage).then(async (image)=>{
-        const svgOverlay = await createSvgOverlay(elementsPositionInfo, width, height, options.annotationPadding, options.borderThickness, prompt);
-        const svgImage = await Jimp.read(svgOverlay);
-        const compositeImage = await image.composite(svgImage, 0, 0, {
-            mode: Jimp.BLEND_SOURCE_OVER,
-            opacitySource: 1,
-            opacityDest: 1
-        });
-        return compositeImage;
-    }).then(async (compositeImage)=>{
-        compositeImage.quality(90);
-        const base64 = await compositeImage.getBase64Async(Jimp.MIME_JPEG);
+    try {
+        const basePixels = photonImage.get_raw_pixels();
+        const overlayPixels = await createSvgOverlay(elementsPositionInfo, width, height, options.annotationPadding, options.borderThickness, prompt);
+        const blendedPixels = blendPixels(basePixels, overlayPixels, width, height);
+        const resultImage = new PhotonImage(blendedPixels, width, height);
+        const base64 = await photonToBase64(resultImage, 90);
+        resultImage.free();
         return base64;
-    }).catch((error)=>{
-        throw error;
-    });
-    return result;
+    } finally{
+        photonImage.free();
+    }
 };
 const processImageElementInfo = async (options)=>{
     const base64Image = options.inputImgBase64.split(';base64,').pop();
@@ -182,4 +800,25 @@ const processImageElementInfo = async (options)=>{
         compositeElementInfoImgWithoutTextBase64
     };
 };
-export { compositeElementInfoImg, processImageElementInfo };
+async function annotateRects(imgBase64, rects, prompt) {
+    const markedImage = await compositeElementInfoImg({
+        inputImgBase64: imgBase64,
+        elementsPositionInfo: rects.map((rect, index)=>({
+                id: `rect-${index}`,
+                rect,
+                indexId: index + 1,
+                attributes: {
+                    nodeType: NodeType.CONTAINER
+                },
+                content: '',
+                center: [
+                    rect.left + rect.width / 2,
+                    rect.top + rect.height / 2
+                ]
+            })),
+        annotationPadding: 0,
+        prompt
+    });
+    return markedImage;
+}
+export { annotateRects, compositeElementInfoImg, processImageElementInfo };