@xiuchang-midscene/shared 2.0.2

package/src/img/canvas-fallback.ts

@@ -0,0 +1,393 @@
+/**
+ * Canvas-based fallback for image processing when Photon WASM fails to load.
+ * Provides a compatible API with Photon for browser environments.
+ */
+
+import { getDebug } from '../logger';
+
+const debug = getDebug('img:canvas-fallback');
+
+/**
+ * Canvas-based image class that mimics PhotonImage API
+ */
+export class CanvasImage {
+  private canvas: HTMLCanvasElement;
+  private ctx: CanvasRenderingContext2D;
+  private _width: number;
+  private _height: number;
+
+  constructor(canvas: HTMLCanvasElement) {
+    this.canvas = canvas;
+    const ctx = canvas.getContext('2d');
+    if (!ctx) {
+      throw new Error('Failed to get 2d context');
+    }
+    this.ctx = ctx;
+    this._width = canvas.width;
+    this._height = canvas.height;
+  }
+
+  get_width(): number {
+    return this._width;
+  }
+
+  get_height(): number {
+    return this._height;
+  }
+
+  get_raw_pixels(): Uint8Array {
+    const imageData = this.ctx.getImageData(0, 0, this._width, this._height);
+    return new Uint8Array(imageData.data.buffer);
+  }
+
+  get_bytes_jpeg(quality: number): Uint8Array {
+    const dataUrl = this.canvas.toDataURL('image/jpeg', quality / 100);
+    const base64 = dataUrl.split(',')[1];
+    const binary = atob(base64);
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) {
+      bytes[i] = binary.charCodeAt(i);
+    }
+    return bytes;
+  }
+
+  free(): void {
+    // No-op for Canvas, garbage collector will handle it
+  }
+
+  // Internal method to get canvas for composition
+  _getCanvas(): HTMLCanvasElement {
+    return this.canvas;
+  }
+
+  _getContext(): CanvasRenderingContext2D {
+    return this.ctx;
+  }
+
+  /**
+   * Create a CanvasImage from a base64 string
+   */
+  static async new_from_base64(base64Body: string): Promise<CanvasImage> {
+    return new Promise((resolve, reject) => {
+      const img = new Image();
+      img.onload = () => {
+        const canvas = document.createElement('canvas');
+        canvas.width = img.width;
+        canvas.height = img.height;
+        const ctx = canvas.getContext('2d');
+        if (!ctx) {
+          reject(new Error('Failed to get 2d context'));
+          return;
+        }
+        ctx.drawImage(img, 0, 0);
+        resolve(new CanvasImage(canvas));
+      };
+      img.onerror = () => {
+        reject(new Error('Failed to load image'));
+      };
+      // Handle both with and without data URL prefix
+      if (base64Body.startsWith('data:')) {
+        img.src = base64Body;
+      } else {
+        img.src = `data:image/png;base64,${base64Body}`;
+      }
+    });
+  }
+
+  /**
+   * Create a CanvasImage from a byte array (async version)
+   */
+  static async new_from_byteslice(bytes: Uint8Array): Promise<CanvasImage> {
+    return new Promise((resolve, reject) => {
+      const blob = new Blob([bytes], { type: 'image/png' });
+      const url = URL.createObjectURL(blob);
+      const img = new Image();
+
+      img.onload = () => {
+        const canvas = document.createElement('canvas');
+        canvas.width = img.width;
+        canvas.height = img.height;
+        const ctx = canvas.getContext('2d');
+        if (!ctx) {
+          URL.revokeObjectURL(url);
+          reject(new Error('Failed to get 2d context'));
+          return;
+        }
+        ctx.drawImage(img, 0, 0);
+        URL.revokeObjectURL(url);
+        resolve(new CanvasImage(canvas));
+      };
+
+      img.onerror = () => {
+        URL.revokeObjectURL(url);
+        reject(new Error('Failed to load image from bytes'));
+      };
+
+      img.src = url;
+    });
+  }
+}
+
+/**
+ * Sampling filter enum (compatible with Photon)
+ */
+export const CanvasSamplingFilter = {
+  Nearest: 'nearest',
+  Triangle: 'triangle',
+  CatmullRom: 'catmullrom',
+  Gaussian: 'gaussian',
+  Lanczos3: 'lanczos3',
+} as const;
+
+/**
+ * RGBA color class (compatible with Photon)
+ */
+export class CanvasRgba {
+  r: number;
+  g: number;
+  b: number;
+  a: number;
+
+  constructor(r: number, g: number, b: number, a: number) {
+    this.r = r;
+    this.g = g;
+    this.b = b;
+    this.a = a;
+  }
+}
+
+/**
+ * Resize an image
+ */
+export function canvasResize(
+  image: CanvasImage,
+  newWidth: number,
+  newHeight: number,
+  _filter: string,
+): CanvasImage {
+  const canvas = document.createElement('canvas');
+  canvas.width = newWidth;
+  canvas.height = newHeight;
+  const ctx = canvas.getContext('2d');
+  if (!ctx) {
+    throw new Error('Failed to get 2d context');
+  }
+
+  // Enable image smoothing for better quality
+  ctx.imageSmoothingEnabled = true;
+  ctx.imageSmoothingQuality = 'high';
+
+  ctx.drawImage(image._getCanvas(), 0, 0, newWidth, newHeight);
+  return new CanvasImage(canvas);
+}
+
+/**
+ * Crop an image
+ */
+export function canvasCrop(
+  image: CanvasImage,
+  x1: number,
+  y1: number,
+  x2: number,
+  y2: number,
+): CanvasImage {
+  const width = x2 - x1;
+  const height = y2 - y1;
+  const canvas = document.createElement('canvas');
+  canvas.width = width;
+  canvas.height = height;
+  const ctx = canvas.getContext('2d');
+  if (!ctx) {
+    throw new Error('Failed to get 2d context');
+  }
+  ctx.drawImage(image._getCanvas(), x1, y1, width, height, 0, 0, width, height);
+  return new CanvasImage(canvas);
+}
+
+/**
+ * Add padding to the right of an image
+ */
+export function canvasPaddingRight(
+  image: CanvasImage,
+  padding: number,
+  color: CanvasRgba,
+): CanvasImage {
+  const newWidth = image.get_width() + padding;
+  const height = image.get_height();
+  const canvas = document.createElement('canvas');
+  canvas.width = newWidth;
+  canvas.height = height;
+  const ctx = canvas.getContext('2d');
+  if (!ctx) {
+    throw new Error('Failed to get 2d context');
+  }
+
+  // Fill with color
+  ctx.fillStyle = `rgba(${color.r}, ${color.g}, ${color.b}, ${color.a / 255})`;
+  ctx.fillRect(0, 0, newWidth, height);
+
+  // Draw original image
+  ctx.drawImage(image._getCanvas(), 0, 0);
+  return new CanvasImage(canvas);
+}
+
+/**
+ * Add padding to the bottom of an image
+ */
+export function canvasPaddingBottom(
+  image: CanvasImage,
+  padding: number,
+  color: CanvasRgba,
+): CanvasImage {
+  const width = image.get_width();
+  const newHeight = image.get_height() + padding;
+  const canvas = document.createElement('canvas');
+  canvas.width = width;
+  canvas.height = newHeight;
+  const ctx = canvas.getContext('2d');
+  if (!ctx) {
+    throw new Error('Failed to get 2d context');
+  }
+
+  // Fill with color
+  ctx.fillStyle = `rgba(${color.r}, ${color.g}, ${color.b}, ${color.a / 255})`;
+  ctx.fillRect(0, 0, width, newHeight);
+
+  // Draw original image
+  ctx.drawImage(image._getCanvas(), 0, 0);
+  return new CanvasImage(canvas);
+}
+
+/**
+ * Add uniform padding to an image
+ */
+export function canvasPaddingUniform(
+  image: CanvasImage,
+  padding: number,
+  color: CanvasRgba,
+): CanvasImage {
+  const newWidth = image.get_width() + padding * 2;
+  const newHeight = image.get_height() + padding * 2;
+  const canvas = document.createElement('canvas');
+  canvas.width = newWidth;
+  canvas.height = newHeight;
+  const ctx = canvas.getContext('2d');
+  if (!ctx) {
+    throw new Error('Failed to get 2d context');
+  }
+
+  // Fill with color
+  ctx.fillStyle = `rgba(${color.r}, ${color.g}, ${color.b}, ${color.a / 255})`;
+  ctx.fillRect(0, 0, newWidth, newHeight);
+
+  // Draw original image centered
+  ctx.drawImage(image._getCanvas(), padding, padding);
+  return new CanvasImage(canvas);
+}
+
+/**
+ * Add padding to the left of an image
+ */
+export function canvasPaddingLeft(
+  image: CanvasImage,
+  padding: number,
+  color: CanvasRgba,
+): CanvasImage {
+  const newWidth = image.get_width() + padding;
+  const height = image.get_height();
+  const canvas = document.createElement('canvas');
+  canvas.width = newWidth;
+  canvas.height = height;
+  const ctx = canvas.getContext('2d');
+  if (!ctx) {
+    throw new Error('Failed to get 2d context');
+  }
+
+  // Fill with color
+  ctx.fillStyle = `rgba(${color.r}, ${color.g}, ${color.b}, ${color.a / 255})`;
+  ctx.fillRect(0, 0, newWidth, height);
+
+  // Draw original image offset by padding
+  ctx.drawImage(image._getCanvas(), padding, 0);
+  return new CanvasImage(canvas);
+}
+
+/**
+ * Add padding to the top of an image
+ */
+export function canvasPaddingTop(
+  image: CanvasImage,
+  padding: number,
+  color: CanvasRgba,
+): CanvasImage {
+  const width = image.get_width();
+  const newHeight = image.get_height() + padding;
+  const canvas = document.createElement('canvas');
+  canvas.width = width;
+  canvas.height = newHeight;
+  const ctx = canvas.getContext('2d');
+  if (!ctx) {
+    throw new Error('Failed to get 2d context');
+  }
+
+  // Fill with color
+  ctx.fillStyle = `rgba(${color.r}, ${color.g}, ${color.b}, ${color.a / 255})`;
+  ctx.fillRect(0, 0, width, newHeight);
+
+  // Draw original image offset by padding
+  ctx.drawImage(image._getCanvas(), 0, padding);
+  return new CanvasImage(canvas);
+}
+
+/**
+ * Watermark an image (overlay one image on another)
+ */
+export function canvasWatermark(
+  base: CanvasImage,
+  overlay: CanvasImage,
+  x: number,
+  y: number,
+): CanvasImage {
+  const canvas = document.createElement('canvas');
+  canvas.width = base.get_width();
+  canvas.height = base.get_height();
+  const ctx = canvas.getContext('2d');
+  if (!ctx) {
+    throw new Error('Failed to get 2d context');
+  }
+
+  // Draw base image
+  ctx.drawImage(base._getCanvas(), 0, 0);
+
+  // Draw overlay at position
+  ctx.drawImage(overlay._getCanvas(), x, y);
+  return new CanvasImage(canvas);
+}
+
+/**
+ * Create and return the canvas fallback module with Photon-compatible API
+ */
+export function createCanvasFallbackModule() {
+  debug('Creating Canvas fallback module');
+  console.log(
+    '[midscene:img] Using Canvas fallback (Photon WASM not available)',
+  );
+
+  return {
+    PhotonImage: CanvasImage,
+    SamplingFilter: CanvasSamplingFilter,
+    resize: canvasResize,
+    crop: canvasCrop,
+    open_image: () => {
+      throw new Error('open_image not supported in Canvas fallback');
+    },
+    base64_to_image: CanvasImage.new_from_base64,
+    padding_uniform: canvasPaddingUniform,
+    padding_left: canvasPaddingLeft,
+    padding_right: canvasPaddingRight,
+    padding_top: canvasPaddingTop,
+    padding_bottom: canvasPaddingBottom,
+    watermark: canvasWatermark,
+    Rgba: CanvasRgba,
+  };
+}

package/src/img/get-photon.ts

@@ -0,0 +1,108 @@
+import { getDebug } from '../logger';
+import { ifInBrowser, ifInNode, ifInWorker } from '../utils';
+
+const debug = getDebug('img');
+
+let photonModule: any = null;
+let isInitialized = false;
+let usingCanvasFallback = false;
+
+export default async function getPhoton(): Promise<{
+  PhotonImage: typeof import('@silvia-odwyer/photon-node').PhotonImage;
+  SamplingFilter: typeof import('@silvia-odwyer/photon-node').SamplingFilter;
+  resize: typeof import('@silvia-odwyer/photon-node').resize;
+  crop: typeof import('@silvia-odwyer/photon-node').crop;
+  open_image: typeof import('@silvia-odwyer/photon-node').open_image;
+  base64_to_image: typeof import('@silvia-odwyer/photon-node').base64_to_image;
+  padding_uniform: typeof import('@silvia-odwyer/photon-node').padding_uniform;
+  padding_left: typeof import('@silvia-odwyer/photon-node').padding_left;
+  padding_right: typeof import('@silvia-odwyer/photon-node').padding_right;
+  padding_top: typeof import('@silvia-odwyer/photon-node').padding_top;
+  padding_bottom: typeof import('@silvia-odwyer/photon-node').padding_bottom;
+  watermark: typeof import('@silvia-odwyer/photon-node').watermark;
+  Rgba: typeof import('@silvia-odwyer/photon-node').Rgba;
+}> {
+  if (photonModule && isInitialized) {
+    return photonModule;
+  }
+
+  const env = ifInBrowser
+    ? 'browser'
+    : ifInWorker
+      ? 'worker'
+      : ifInNode
+        ? 'node'
+        : 'unknown';
+  debug(`Loading photon module in ${env} environment`);
+
+  // Try to load Photon first
+  try {
+    if (ifInBrowser || ifInWorker) {
+      // Regular browser environment: use @silvia-odwyer/photon
+      const photon = await import('@silvia-odwyer/photon');
+      if (typeof photon.default === 'function') {
+        // for browser environment, ensure WASM module is correctly initialized
+        await photon.default();
+      }
+      debug('Photon loaded: @silvia-odwyer/photon (browser/worker)');
+      photonModule = photon;
+    } else if (ifInNode) {
+      // Node.js environment: use @silvia-odwyer/photon-node
+      photonModule = await import('@silvia-odwyer/photon-node');
+      debug('Photon loaded: @silvia-odwyer/photon-node (node)');
+    }
+
+    // verify that the critical functions exist (only for Photon, not Canvas fallback)
+    if (!photonModule?.PhotonImage) {
+      throw new Error('PhotonImage is not available');
+    }
+    // new_from_byteslice may be sync (Photon) or async (Canvas), both are acceptable
+    if (
+      !photonModule.PhotonImage.new_from_byteslice &&
+      !photonModule.PhotonImage.new_from_base64
+    ) {
+      throw new Error(
+        'PhotonImage.new_from_byteslice or new_from_base64 is not available',
+      );
+    }
+
+    isInitialized = true;
+    return photonModule;
+  } catch (error) {
+    debug(
+      `Photon load failed: ${error instanceof Error ? error.message : String(error)}`,
+    );
+
+    // In browser environment, fall back to Canvas API
+    if (ifInBrowser) {
+      console.warn(
+        `[midscene:img] Photon WASM failed to load, falling back to Canvas API. Error: ${error instanceof Error ? error.message : String(error)}`,
+      );
+
+      try {
+        const { createCanvasFallbackModule } = await import(
+          './canvas-fallback'
+        );
+        photonModule = createCanvasFallbackModule();
+        usingCanvasFallback = true;
+        isInitialized = true;
+        return photonModule;
+      } catch (fallbackError) {
+        debug(
+          `Canvas fallback also failed: ${fallbackError instanceof Error ? fallbackError.message : String(fallbackError)}`,
+        );
+      }
+    }
+
+    throw new Error(
+      `Failed to load photon module: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+}
+
+/**
+ * Check if we're using the Canvas fallback instead of Photon
+ */
+export function isUsingCanvasFallback(): boolean {
+  return usingCanvasFallback;
+}

package/src/img/get-sharp.ts

@@ -0,0 +1,18 @@
+import { ifInNode } from '../utils';
+type TSharpModule = typeof import('sharp');
+
+export default async function getSharp(): Promise<TSharpModule> {
+  if (!ifInNode) {
+    throw new Error('Sharp is only available in Node.js environment');
+  }
+
+  try {
+    // @ts-ignore
+    const sharp = await import('sharp');
+    return sharp.default;
+  } catch (error) {
+    throw new Error(
+      `Failed to load sharp module: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+}

package/src/img/index.ts

@@ -0,0 +1,26 @@
+export {
+  imageInfoOfBase64,
+  isValidPNGImageBuffer,
+  isValidJPEGImageBuffer,
+  isValidImageBuffer,
+} from './info';
+export {
+  resizeAndConvertImgBuffer,
+  resizeImgBase64,
+  convertToJpegBase64,
+  zoomForGPT4o,
+  saveBase64Image,
+  paddingToMatchBlockByBase64,
+  cropByRect,
+  scaleImage,
+  localImg2Base64,
+  httpImg2Base64,
+  preProcessImageUrl,
+  parseBase64,
+  createImgBase64ByFormat,
+} from './transform';
+export {
+  processImageElementInfo,
+  compositeElementInfoImg,
+  annotateRects,
+} from './box-select';

package/src/img/info.ts

@@ -0,0 +1,75 @@
+import assert from 'node:assert';
+import type { Size } from '../types';
+import getPhoton from './get-photon';
+
+export interface ImageInfo extends Size {}
+
+/**
+ * Retrieves the dimensions of an image from a base64-encoded string
+ *
+ * @param imageBase64 - The base64-encoded image data
+ * @returns A Promise that resolves to an object containing the width and height of the image
+ * @throws Error if the image data is invalid
+ */
+export async function imageInfoOfBase64(
+  imageBase64: string,
+): Promise<ImageInfo> {
+  const { PhotonImage } = await getPhoton();
+  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '');
+  // Support both sync (Photon) and async (Canvas fallback) versions
+  const result = PhotonImage.new_from_base64(base64Data);
+  const image = result instanceof Promise ? await result : result;
+  const width = image.get_width();
+  const height = image.get_height();
+  image.free();
+  assert(width && height, 'Invalid image: cannot get width or height');
+  return { width, height };
+}
+
+/**
+ * Check if the Buffer is a valid PNG image
+ * @param buffer The Buffer to check
+ * @returns true if the Buffer is a valid PNG image, otherwise false
+ */
+export function isValidPNGImageBuffer(buffer: Buffer): boolean {
+  if (!buffer || buffer.length < 8) {
+    return false;
+  }
+
+  // Check PNG signature (8 bytes): 89 50 4E 47 0D 0A 1A 0A
+  // This is more robust than just checking the first 4 bytes
+  const isPNG =
+    buffer[0] === 0x89 &&
+    buffer[1] === 0x50 &&
+    buffer[2] === 0x4e &&
+    buffer[3] === 0x47 &&
+    buffer[4] === 0x0d &&
+    buffer[5] === 0x0a &&
+    buffer[6] === 0x1a &&
+    buffer[7] === 0x0a;
+
+  return isPNG;
+}
+
+/**
+ * Check if the Buffer is a valid JPEG image
+ * @param buffer The Buffer to check
+ * @returns true if the Buffer is a valid JPEG image, otherwise false
+ */
+export function isValidJPEGImageBuffer(buffer: Buffer): boolean {
+  if (!buffer || buffer.length < 3) {
+    return false;
+  }
+
+  // Check JPEG signature (3 bytes): FF D8 FF
+  return buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff;
+}
+
+/**
+ * Check if the Buffer is a valid image (PNG or JPEG)
+ * @param buffer The Buffer to check
+ * @returns true if the Buffer is a valid PNG or JPEG image, otherwise false
+ */
+export function isValidImageBuffer(buffer: Buffer): boolean {
+  return isValidPNGImageBuffer(buffer) || isValidJPEGImageBuffer(buffer);
+}