@midscene/shared 1.10.5 → 1.10.6-beta-20260717061640.0

package/src/img/box-select.ts

@@ -1,9 +1,16 @@
 import assert from 'node:assert';
-import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon-node';
+import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon';
 import { NodeType } from '../constants';
 import type { BaseElement, Rect } from '../types';
+import { ifInNode } from '../utils';
 import getPhoton from './get-photon';
-import { photonFromBase64, photonToBase64 } from './transform';
+import getSharp from './get-sharp';
+import {
+  createImgBase64ByFormat,
+  parseBase64,
+  photonFromBase64,
+  photonToBase64,
+} from './transform';
 
 // Simple 5x7 bitmap font for digits 0-9
 const DIGIT_FONT: Record<string, number[][]> = {
@@ -715,6 +722,47 @@ const createSvgOverlay = async (
   return overlayPixels;
 };
 
+async function decodeImageWithSharp(
+  inputImgBase64: string,
+  size?: { width: number; height: number },
+) {
+  const { body } = parseBase64(inputImgBase64);
+  const Sharp = await getSharp();
+  let pipeline = Sharp(Buffer.from(body, 'base64'));
+  if (size) {
+    pipeline = pipeline.resize(size.width, size.height, {
+      fit: 'fill',
+      kernel: 'nearest',
+    });
+  }
+  const { data, info } = await pipeline
+    .ensureAlpha()
+    .raw()
+    .toBuffer({ resolveWithObject: true });
+  if (!info.width || !info.height || info.channels !== 4) {
+    throw new Error('Image processing failed to produce RGBA pixels');
+  }
+  return {
+    pixels: new Uint8Array(data),
+    width: info.width,
+    height: info.height,
+  };
+}
+
+async function encodeRgbaWithSharp(
+  pixels: Uint8Array,
+  width: number,
+  height: number,
+) {
+  const Sharp = await getSharp();
+  const output = await Sharp(Buffer.from(pixels), {
+    raw: { width, height, channels: 4 },
+  })
+    .jpeg({ quality: 90, chromaSubsampling: '4:4:4' })
+    .toBuffer();
+  return createImgBase64ByFormat('jpeg', output.toString('base64'));
+}
+
 export const compositeElementInfoImg = async (options: {
   inputImgBase64: string;
   elementsPositionInfo: Array<ElementForOverlay>;
@@ -725,6 +773,27 @@ export const compositeElementInfoImg = async (options: {
   prompt?: string;
 }) => {
   assert(options.inputImgBase64, 'inputImgBase64 is required');
+  if (ifInNode) {
+    const { pixels, width, height } = await decodeImageWithSharp(
+      options.inputImgBase64,
+      options.size,
+    );
+    const overlayPixels = await createSvgOverlay(
+      options.elementsPositionInfo,
+      width,
+      height,
+      options.annotationPadding,
+      options.borderThickness,
+      options.prompt,
+      options.centerPoint,
+    );
+    return encodeRgbaWithSharp(
+      blendPixels(pixels, overlayPixels, width, height),
+      width,
+      height,
+    );
+  }
+
   const { PhotonImage, SamplingFilter, resize } = await getPhoton();
 
   let width = 0;
@@ -800,6 +869,27 @@ export const compositePointMarkerImg = async (options: {
   indexId?: number;
 }) => {
   assert(options.inputImgBase64, 'inputImgBase64 is required');
+  if (ifInNode) {
+    const { pixels, width, height } = await decodeImageWithSharp(
+      options.inputImgBase64,
+      options.size,
+    );
+    const overlayPixels = new Uint8Array(width * height * 4);
+    drawPointMarker(
+      overlayPixels,
+      width,
+      height,
+      options.point,
+      options.radius ?? 14,
+      options.indexId ?? 1,
+    );
+    return encodeRgbaWithSharp(
+      blendPixels(pixels, overlayPixels, width, height),
+      width,
+      height,
+    );
+  }
+
   const { PhotonImage, SamplingFilter, resize } = await getPhoton();
 
   let width = 0;

package/src/img/get-photon.ts

@@ -1,5 +1,5 @@
 import { getDebug } from '../logger';
-import { ifInBrowser, ifInNode, ifInWorker } from '../utils';
+import { ifInBrowser, ifInWorker } from '../utils';
 
 const debug = getDebug('img');
 
@@ -8,31 +8,25 @@ let isInitialized = false;
 let usingCanvasFallback = false;
 
 export default async function getPhoton(): Promise<{
-  PhotonImage: typeof import('@silvia-odwyer/photon-node').PhotonImage;
-  SamplingFilter: typeof import('@silvia-odwyer/photon-node').SamplingFilter;
-  resize: typeof import('@silvia-odwyer/photon-node').resize;
-  crop: typeof import('@silvia-odwyer/photon-node').crop;
-  open_image: typeof import('@silvia-odwyer/photon-node').open_image;
-  base64_to_image: typeof import('@silvia-odwyer/photon-node').base64_to_image;
-  padding_uniform: typeof import('@silvia-odwyer/photon-node').padding_uniform;
-  padding_left: typeof import('@silvia-odwyer/photon-node').padding_left;
-  padding_right: typeof import('@silvia-odwyer/photon-node').padding_right;
-  padding_top: typeof import('@silvia-odwyer/photon-node').padding_top;
-  padding_bottom: typeof import('@silvia-odwyer/photon-node').padding_bottom;
-  watermark: typeof import('@silvia-odwyer/photon-node').watermark;
-  Rgba: typeof import('@silvia-odwyer/photon-node').Rgba;
+  PhotonImage: typeof import('@silvia-odwyer/photon').PhotonImage;
+  SamplingFilter: typeof import('@silvia-odwyer/photon').SamplingFilter;
+  resize: typeof import('@silvia-odwyer/photon').resize;
+  crop: typeof import('@silvia-odwyer/photon').crop;
+  open_image: typeof import('@silvia-odwyer/photon').open_image;
+  base64_to_image: typeof import('@silvia-odwyer/photon').base64_to_image;
+  padding_uniform: typeof import('@silvia-odwyer/photon').padding_uniform;
+  padding_left: typeof import('@silvia-odwyer/photon').padding_left;
+  padding_right: typeof import('@silvia-odwyer/photon').padding_right;
+  padding_top: typeof import('@silvia-odwyer/photon').padding_top;
+  padding_bottom: typeof import('@silvia-odwyer/photon').padding_bottom;
+  watermark: typeof import('@silvia-odwyer/photon').watermark;
+  Rgba: typeof import('@silvia-odwyer/photon').Rgba;
 }> {
   if (photonModule && isInitialized) {
     return photonModule;
   }
 
-  const env = ifInBrowser
-    ? 'browser'
-    : ifInWorker
-      ? 'worker'
-      : ifInNode
-        ? 'node'
-        : 'unknown';
+  const env = ifInBrowser ? 'browser' : ifInWorker ? 'worker' : 'unknown';
   debug(`Loading photon module in ${env} environment`);
 
   // Try to load Photon first
@@ -46,10 +40,8 @@ export default async function getPhoton(): Promise<{
       }
       debug('Photon loaded: @silvia-odwyer/photon (browser/worker)');
       photonModule = photon;
-    } else if (ifInNode) {
-      // Node.js environment: use @silvia-odwyer/photon-node
-      photonModule = await import('@silvia-odwyer/photon-node');
-      debug('Photon loaded: @silvia-odwyer/photon-node (node)');
+    } else {
+      throw new Error('Photon is only available in browser environments');
     }
 
     // verify that the critical functions exist (only for Photon, not Canvas fallback)

package/src/img/info.ts

@@ -1,7 +1,9 @@
 import assert from 'node:assert';
 import { Buffer } from 'node:buffer';
 import type { Size } from '../types';
+import { ifInNode } from '../utils';
 import getPhoton from './get-photon';
+import getSharp from './get-sharp';
 
 export interface ImageInfo extends Size {}
 
@@ -15,7 +17,6 @@ export interface ImageInfo extends Size {}
 export async function imageInfoOfBase64(
   imageBase64: string,
 ): Promise<ImageInfo> {
-  const { PhotonImage } = await getPhoton();
   const base64Data = imageBase64
     .replace(/^data:image\/\w+;base64,/, '')
     .replace(/\s/g, '');
@@ -26,6 +27,25 @@ export async function imageInfoOfBase64(
   );
   const imageBuffer = Buffer.from(base64Data, 'base64');
   assert(isValidImageBuffer(imageBuffer), 'Invalid image: unsupported format');
+  if (ifInNode) {
+    let metadata;
+    try {
+      const Sharp = await getSharp();
+      metadata = await Sharp(imageBuffer).metadata();
+    } catch (error) {
+      throw new Error(
+        `Invalid image: failed to decode base64 data (${error instanceof Error ? error.message : String(error)})`,
+        { cause: error },
+      );
+    }
+    assert(
+      metadata.width && metadata.height,
+      'Invalid image: cannot get width or height',
+    );
+    return { width: metadata.width, height: metadata.height };
+  }
+
+  const { PhotonImage } = await getPhoton();
   // Support both sync (Photon) and async (Canvas fallback) versions
   let result: ReturnType<typeof PhotonImage.new_from_base64>;
   try {

package/src/img/transform.ts

@@ -3,7 +3,7 @@ import { Buffer } from 'node:buffer';
 import { readFileSync } from 'node:fs';
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
-import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon-node';
+import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon';
 import { getDebug } from '../logger';
 import type { Rect } from '../types';
 import { ifInNode } from '../utils';
@@ -61,44 +61,36 @@ export async function resizeAndConvertImgBuffer(
   imgDebug(`resizeImg start, target size: ${newSize.width}x${newSize.height}`);
 
   if (ifInNode) {
-    // Node.js environment: use Sharp
-    try {
-      const Sharp = await getSharp();
-      const metadata = await Sharp(inputData).metadata();
-      const { width: originalWidth, height: originalHeight } = metadata;
-
-      if (!originalWidth || !originalHeight) {
-        throw Error('Undefined width or height from the input image.');
-      }
-
-      if (
-        newSize.width === originalWidth &&
-        newSize.height === originalHeight
-      ) {
-        return {
-          buffer: inputData,
-          format: inputFormat,
-        };
-      }
+    const Sharp = await getSharp();
+    const metadata = await Sharp(inputData).metadata();
+    const { width: originalWidth, height: originalHeight } = metadata;
 
-      const resizedBuffer = await Sharp(inputData)
-        .resize(newSize.width, newSize.height)
-        .jpeg({ quality: 90 })
-        .toBuffer();
-
-      const resizeEndTime = Date.now();
-      imgDebug(
-        `resizeImg done (Sharp), target size: ${newSize.width}x${newSize.height}, cost: ${resizeEndTime - resizeStartTime}ms`,
-      );
+    if (!originalWidth || !originalHeight) {
+      throw Error('Undefined width or height from the input image.');
+    }
 
+    if (newSize.width === originalWidth && newSize.height === originalHeight) {
       return {
-        buffer: resizedBuffer,
-        // by Sharp.jpeg()
-        format: 'jpeg',
+        buffer: inputData,
+        format: inputFormat,
       };
-    } catch (error) {
-      imgDebug('Sharp failed, falling back to Photon:', error);
     }
+
+    const resizedBuffer = await Sharp(inputData)
+      .resize(newSize.width, newSize.height)
+      .jpeg({ quality: 90 })
+      .toBuffer();
+
+    const resizeEndTime = Date.now();
+    imgDebug(
+      `resizeImg done (Sharp), target size: ${newSize.width}x${newSize.height}, cost: ${resizeEndTime - resizeStartTime}ms`,
+    );
+
+    return {
+      buffer: resizedBuffer,
+      // by Sharp.jpeg()
+      format: 'jpeg',
+    };
   }
 
   // browser environment: use Photon (or Canvas fallback)
@@ -410,6 +402,38 @@ export async function paddingToMatchBlockByBase64(
   height: number;
   imageBase64: string;
 }> {
+  if (ifInNode) {
+    const { body } = parseBase64(imageBase64);
+    const inputBuffer = Buffer.from(body, 'base64');
+    const Sharp = await getSharp();
+    const metadata = await Sharp(inputBuffer).metadata();
+    const width = metadata.width;
+    const height = metadata.height;
+    if (!width || !height) {
+      throw new Error('Failed to get image dimensions');
+    }
+
+    const targetWidth = Math.ceil(width / blockSize) * blockSize;
+    const targetHeight = Math.ceil(height / blockSize) * blockSize;
+    if (targetWidth === width && targetHeight === height) {
+      return { width, height, imageBase64 };
+    }
+
+    const output = await Sharp(inputBuffer)
+      .extend({
+        right: targetWidth - width,
+        bottom: targetHeight - height,
+        background: { r: 255, g: 255, b: 255, alpha: 1 },
+      })
+      .jpeg({ quality: 90 })
+      .toBuffer();
+    return {
+      width: targetWidth,
+      height: targetHeight,
+      imageBase64: createImgBase64ByFormat('jpeg', output.toString('base64')),
+    };
+  }
+
   const photonImage = await photonFromBase64(imageBase64);
   try {
     const paddedResult = await paddingToMatchBlock(photonImage, blockSize);
@@ -435,6 +459,29 @@ export async function cropByRect(
   height: number;
   imageBase64: string;
 }> {
+  if (ifInNode) {
+    const { body } = parseBase64(imageBase64);
+    const Sharp = await getSharp();
+    const left = Math.trunc(rect.left);
+    const top = Math.trunc(rect.top);
+    const width = Math.trunc(rect.left + rect.width) - left;
+    const height = Math.trunc(rect.top + rect.height) - top;
+    const output = await Sharp(Buffer.from(body, 'base64'))
+      .extract({
+        left,
+        top,
+        width,
+        height,
+      })
+      .jpeg({ quality: 90 })
+      .toBuffer();
+    return {
+      width,
+      height,
+      imageBase64: createImgBase64ByFormat('jpeg', output.toString('base64')),
+    };
+  }
+
   const { crop } = await getPhoton();
   const photonImage = await photonFromBase64(imageBase64);
   const { left, top, width, height } = rect;
@@ -593,48 +640,43 @@ export async function scaleImage(
   imgDebug(`scaleImage start, scale factor: ${scale}`);
 
   if (ifInNode) {
-    // Node.js environment: use Sharp
-    try {
-      const Sharp = await getSharp();
-      const metadata = await Sharp(buffer).metadata();
-      const originalWidth = metadata.width || 0;
-      const originalHeight = metadata.height || 0;
+    const Sharp = await getSharp();
+    const metadata = await Sharp(buffer).metadata();
+    const originalWidth = metadata.width || 0;
+    const originalHeight = metadata.height || 0;
 
-      if (originalWidth === 0 || originalHeight === 0) {
-        throw new Error('Failed to get image dimensions');
-      }
+    if (originalWidth === 0 || originalHeight === 0) {
+      throw new Error('Failed to get image dimensions');
+    }
 
-      const newWidth = Math.round(originalWidth * scale);
-      const newHeight = Math.round(originalHeight * scale);
-
-      const resizedBuffer = await Sharp(buffer)
-        .resize(newWidth, newHeight, {
-          kernel: 'lanczos3',
-          fit: 'fill',
-        })
-        .jpeg({
-          quality: 90,
-        })
-        .toBuffer();
-
-      const scaleEndTime = Date.now();
-      imgDebug(
-        `scaleImage done (Sharp): ${originalWidth}x${originalHeight} -> ${newWidth}x${newHeight} (scale=${scale}), cost: ${scaleEndTime - scaleStartTime}ms`,
-      );
+    const newWidth = Math.round(originalWidth * scale);
+    const newHeight = Math.round(originalHeight * scale);
+
+    const resizedBuffer = await Sharp(buffer)
+      .resize(newWidth, newHeight, {
+        kernel: 'lanczos3',
+        fit: 'fill',
+      })
+      .jpeg({
+        quality: 90,
+      })
+      .toBuffer();
+
+    const scaleEndTime = Date.now();
+    imgDebug(
+      `scaleImage done (Sharp): ${originalWidth}x${originalHeight} -> ${newWidth}x${newHeight} (scale=${scale}), cost: ${scaleEndTime - scaleStartTime}ms`,
+    );
 
-      const base64 = `data:image/jpeg;base64,${resizedBuffer.toString('base64')}`;
+    const base64 = `data:image/jpeg;base64,${resizedBuffer.toString('base64')}`;
 
-      return {
-        width: newWidth,
-        height: newHeight,
-        imageBase64: base64,
-      };
-    } catch (error) {
-      imgDebug('Sharp failed, falling back to Photon:', error);
-    }
+    return {
+      width: newWidth,
+      height: newHeight,
+      imageBase64: base64,
+    };
   }
 
-  // Browser environment or Sharp failed: use Photon (or Canvas fallback)
+  // Browser environment: use Photon (or Canvas fallback)
   const { PhotonImage, SamplingFilter, resize } = await getPhoton();
   const inputBytes = new Uint8Array(buffer);
   // Support both sync (Photon) and async (Canvas fallback) versions

package/src/logger.ts

@@ -8,28 +8,53 @@ import { ifInNode } from './utils';
 const topicPrefix = 'midscene';
 // Map to store file streams
 const logStreams = new Map<string, fs.WriteStream>();
+// A WriteStream queues every write made after it reports backpressure. The
+// main process can keep running while macOS has deprioritized a backgrounded
+// app, so retaining diagnostic logs here can otherwise grow without bound.
+// Drop best-effort logs until the stream drains instead.
+const backpressuredLogStreams = new Set<string>();
+const unavailableLogStreams = new Set<string>();
 // Map to store debug instances
 const debugInstances = new Map<string, DebugFunction>();
 
 // Function to get or create a log stream
-function getLogStream(topic: string): fs.WriteStream {
+function getLogStream(topic: string): fs.WriteStream | null {
   const topicFileName = topic.replace(/:/g, '-');
+  if (unavailableLogStreams.has(topicFileName)) {
+    return null;
+  }
   if (!logStreams.has(topicFileName)) {
     const logFile = path.join(
       getMidsceneRunSubDir('log'),
       `${topicFileName}.log`,
     );
     const stream = fs.createWriteStream(logFile, { flags: 'a' });
+    // A stream error without a listener terminates the Electron main process.
+    // Logging must remain best-effort, so disable this topic after a file error
+    // rather than repeatedly queuing writes to a broken stream.
+    stream.on('error', () => {
+      unavailableLogStreams.add(topicFileName);
+      backpressuredLogStreams.delete(topicFileName);
+      if (logStreams.get(topicFileName) === stream) {
+        logStreams.delete(topicFileName);
+      }
+    });
     logStreams.set(topicFileName, stream);
   }
-  return logStreams.get(topicFileName)!;
+  return logStreams.get(topicFileName) ?? null;
 }
 
 // Function to write log to file
 function writeLogToFile(topic: string, message: string): void {
   if (!ifInNode) return;
 
+  const topicFileName = topic.replace(/:/g, '-');
+  if (backpressuredLogStreams.has(topicFileName)) {
+    return;
+  }
+
   const stream = getLogStream(topic);
+  if (!stream) return;
   // Generate ISO format timestamp with local timezone
   const now = new Date();
   // Use sv-SE locale to get ISO-like format (YYYY-MM-DD HH:mm:ss)
@@ -47,7 +72,17 @@ function writeLogToFile(topic: string, message: string): void {
     .padStart(2, '0');
   const timezoneString = `${sign}${hours}:${minutes}`;
   const localISOTime = `${isoDate}T${isoTime}.${milliseconds}${timezoneString}`;
-  stream.write(`[${localISOTime}] ${message}\n`);
+  try {
+    if (!stream.write(`[${localISOTime}] ${message}\n`)) {
+      backpressuredLogStreams.add(topicFileName);
+      stream.once('drain', () => {
+        backpressuredLogStreams.delete(topicFileName);
+      });
+    }
+  } catch {
+    unavailableLogStreams.add(topicFileName);
+    backpressuredLogStreams.delete(topicFileName);
+  }
 }
 
 export type DebugFunction = (...args: unknown[]) => void;

package/src/recorder.ts

@@ -34,6 +34,19 @@ export interface MidsceneRecorderPageInfo {
   height: number;
 }
 
+/**
+ * A screenshot stored outside the recording event payload.
+ *
+ * Recorder events are persisted in the Studio renderer. Keeping full data
+ * URLs there makes long recordings retain every screenshot in the renderer
+ * heap, so screenshot bytes live in the Playground run directory instead.
+ */
+export interface MidsceneRecorderScreenshotAssetRef {
+  id: string;
+  mimeType: string;
+  bytes: number;
+}
+
 export interface MidsceneRecorderEvent {
   type: MidsceneRecorderEventType;
   source?: MidsceneRecorderSourceKind;
@@ -46,6 +59,8 @@ export interface MidsceneRecorderEvent {
   pageInfo: MidsceneRecorderPageInfo;
   screenshotBefore?: string;
   screenshotAfter?: string;
+  /** The single screenshot retained for AI description and Markdown export. */
+  screenshotAsset?: MidsceneRecorderScreenshotAssetRef;
   semantic?: MidsceneRecorderSemantic;
   elementDescription?: string;
   descriptionLoading?: boolean;

package/src/zod-schema-utils.ts

@@ -33,9 +33,7 @@ export function unwrapZodField(field: unknown): unknown {
 
 /**
  * Check if a field is a Midscene locator field
- * Checks for either:
- * 1. midscene_location_field_flag in shape (result schema)
- * 2. prompt field in shape (input schema)
+ * Locator input schemas are identified by their prompt field.
  */
 export function isMidsceneLocatorField(field: unknown): boolean {
   const actualField = unwrapZodField(field) as {
@@ -45,11 +43,6 @@ export function isMidsceneLocatorField(field: unknown): boolean {
   if (actualField._def?.typeName === 'ZodObject') {
     const shape = actualField._def.shape?.();
     if (shape) {
-      // Method 1: Check for the location field flag (for result schema)
-      if ('midscene_location_field_flag' in shape) {
-        return true;
-      }
-      // Method 2: Check if it's the input schema by checking for 'prompt' field
       if ('prompt' in shape && shape.prompt) {
         return true;
       }