@midscene/shared 1.10.5-beta-20260716133905.0 → 1.10.5

package/src/img/transform.ts

@@ -3,7 +3,7 @@ import { Buffer } from 'node:buffer';
 import { readFileSync } from 'node:fs';
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
-import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon';
+import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon-node';
 import { getDebug } from '../logger';
 import type { Rect } from '../types';
 import { ifInNode } from '../utils';
@@ -61,36 +61,44 @@ export async function resizeAndConvertImgBuffer(
   imgDebug(`resizeImg start, target size: ${newSize.width}x${newSize.height}`);
 
   if (ifInNode) {
-    const Sharp = await getSharp();
-    const metadata = await Sharp(inputData).metadata();
-    const { width: originalWidth, height: originalHeight } = metadata;
+    // Node.js environment: use Sharp
+    try {
+      const Sharp = await getSharp();
+      const metadata = await Sharp(inputData).metadata();
+      const { width: originalWidth, height: originalHeight } = metadata;
+
+      if (!originalWidth || !originalHeight) {
+        throw Error('Undefined width or height from the input image.');
+      }
 
-    if (!originalWidth || !originalHeight) {
-      throw Error('Undefined width or height from the input image.');
-    }
+      if (
+        newSize.width === originalWidth &&
+        newSize.height === originalHeight
+      ) {
+        return {
+          buffer: inputData,
+          format: inputFormat,
+        };
+      }
+
+      const resizedBuffer = await Sharp(inputData)
+        .resize(newSize.width, newSize.height)
+        .jpeg({ quality: 90 })
+        .toBuffer();
+
+      const resizeEndTime = Date.now();
+      imgDebug(
+        `resizeImg done (Sharp), target size: ${newSize.width}x${newSize.height}, cost: ${resizeEndTime - resizeStartTime}ms`,
+      );
 
-    if (newSize.width === originalWidth && newSize.height === originalHeight) {
       return {
-        buffer: inputData,
-        format: inputFormat,
+        buffer: resizedBuffer,
+        // by Sharp.jpeg()
+        format: 'jpeg',
       };
+    } catch (error) {
+      imgDebug('Sharp failed, falling back to Photon:', error);
     }
-
-    const resizedBuffer = await Sharp(inputData)
-      .resize(newSize.width, newSize.height)
-      .jpeg({ quality: 90 })
-      .toBuffer();
-
-    const resizeEndTime = Date.now();
-    imgDebug(
-      `resizeImg done (Sharp), target size: ${newSize.width}x${newSize.height}, cost: ${resizeEndTime - resizeStartTime}ms`,
-    );
-
-    return {
-      buffer: resizedBuffer,
-      // by Sharp.jpeg()
-      format: 'jpeg',
-    };
   }
 
   // browser environment: use Photon (or Canvas fallback)
@@ -148,6 +156,31 @@ export async function resizeAndConvertImgBuffer(
 
 export const normalizeBase64Body = (body: string) => body.replace(/\s/g, '');
 
+/** Convert an image buffer to JPEG without changing its dimensions. */
+export async function convertImgBufferToJpeg(
+  inputData: Buffer,
+  quality = 90,
+): Promise<Buffer> {
+  if (ifInNode) {
+    try {
+      const Sharp = await getSharp();
+      return await Sharp(inputData).jpeg({ quality }).toBuffer();
+    } catch (error) {
+      imgDebug('Sharp failed, falling back to Photon:', error);
+    }
+  }
+
+  const mimeType = detectImageMimeTypeFromBuffer(inputData) ?? 'image/png';
+  const photonImage = await photonFromBase64(
+    `data:${mimeType};base64,${inputData.toString('base64')}`,
+  );
+  try {
+    return Buffer.from(photonImage.get_bytes_jpeg(quality));
+  } finally {
+    photonImage.free();
+  }
+}
+
 const base64ImageDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i;
 const supportedScreenshotDataUriPattern =
   /^data:image\/(png|jpe?g);base64,([\s\S]*)$/i;
@@ -377,38 +410,6 @@ export async function paddingToMatchBlockByBase64(
   height: number;
   imageBase64: string;
 }> {
-  if (ifInNode) {
-    const { body } = parseBase64(imageBase64);
-    const inputBuffer = Buffer.from(body, 'base64');
-    const Sharp = await getSharp();
-    const metadata = await Sharp(inputBuffer).metadata();
-    const width = metadata.width;
-    const height = metadata.height;
-    if (!width || !height) {
-      throw new Error('Failed to get image dimensions');
-    }
-
-    const targetWidth = Math.ceil(width / blockSize) * blockSize;
-    const targetHeight = Math.ceil(height / blockSize) * blockSize;
-    if (targetWidth === width && targetHeight === height) {
-      return { width, height, imageBase64 };
-    }
-
-    const output = await Sharp(inputBuffer)
-      .extend({
-        right: targetWidth - width,
-        bottom: targetHeight - height,
-        background: { r: 255, g: 255, b: 255, alpha: 1 },
-      })
-      .jpeg({ quality: 90 })
-      .toBuffer();
-    return {
-      width: targetWidth,
-      height: targetHeight,
-      imageBase64: createImgBase64ByFormat('jpeg', output.toString('base64')),
-    };
-  }
-
   const photonImage = await photonFromBase64(imageBase64);
   try {
     const paddedResult = await paddingToMatchBlock(photonImage, blockSize);
@@ -434,29 +435,6 @@ export async function cropByRect(
   height: number;
   imageBase64: string;
 }> {
-  if (ifInNode) {
-    const { body } = parseBase64(imageBase64);
-    const Sharp = await getSharp();
-    const left = Math.trunc(rect.left);
-    const top = Math.trunc(rect.top);
-    const width = Math.trunc(rect.left + rect.width) - left;
-    const height = Math.trunc(rect.top + rect.height) - top;
-    const output = await Sharp(Buffer.from(body, 'base64'))
-      .extract({
-        left,
-        top,
-        width,
-        height,
-      })
-      .jpeg({ quality: 90 })
-      .toBuffer();
-    return {
-      width,
-      height,
-      imageBase64: createImgBase64ByFormat('jpeg', output.toString('base64')),
-    };
-  }
-
   const { crop } = await getPhoton();
   const photonImage = await photonFromBase64(imageBase64);
   const { left, top, width, height } = rect;
@@ -615,43 +593,48 @@ export async function scaleImage(
   imgDebug(`scaleImage start, scale factor: ${scale}`);
 
   if (ifInNode) {
-    const Sharp = await getSharp();
-    const metadata = await Sharp(buffer).metadata();
-    const originalWidth = metadata.width || 0;
-    const originalHeight = metadata.height || 0;
-
-    if (originalWidth === 0 || originalHeight === 0) {
-      throw new Error('Failed to get image dimensions');
-    }
+    // Node.js environment: use Sharp
+    try {
+      const Sharp = await getSharp();
+      const metadata = await Sharp(buffer).metadata();
+      const originalWidth = metadata.width || 0;
+      const originalHeight = metadata.height || 0;
+
+      if (originalWidth === 0 || originalHeight === 0) {
+        throw new Error('Failed to get image dimensions');
+      }
 
-    const newWidth = Math.round(originalWidth * scale);
-    const newHeight = Math.round(originalHeight * scale);
-
-    const resizedBuffer = await Sharp(buffer)
-      .resize(newWidth, newHeight, {
-        kernel: 'lanczos3',
-        fit: 'fill',
-      })
-      .jpeg({
-        quality: 90,
-      })
-      .toBuffer();
-
-    const scaleEndTime = Date.now();
-    imgDebug(
-      `scaleImage done (Sharp): ${originalWidth}x${originalHeight} -> ${newWidth}x${newHeight} (scale=${scale}), cost: ${scaleEndTime - scaleStartTime}ms`,
-    );
+      const newWidth = Math.round(originalWidth * scale);
+      const newHeight = Math.round(originalHeight * scale);
+
+      const resizedBuffer = await Sharp(buffer)
+        .resize(newWidth, newHeight, {
+          kernel: 'lanczos3',
+          fit: 'fill',
+        })
+        .jpeg({
+          quality: 90,
+        })
+        .toBuffer();
+
+      const scaleEndTime = Date.now();
+      imgDebug(
+        `scaleImage done (Sharp): ${originalWidth}x${originalHeight} -> ${newWidth}x${newHeight} (scale=${scale}), cost: ${scaleEndTime - scaleStartTime}ms`,
+      );
 
-    const base64 = `data:image/jpeg;base64,${resizedBuffer.toString('base64')}`;
+      const base64 = `data:image/jpeg;base64,${resizedBuffer.toString('base64')}`;
 
-    return {
-      width: newWidth,
-      height: newHeight,
-      imageBase64: base64,
-    };
+      return {
+        width: newWidth,
+        height: newHeight,
+        imageBase64: base64,
+      };
+    } catch (error) {
+      imgDebug('Sharp failed, falling back to Photon:', error);
+    }
   }
 
-  // Browser environment: use Photon (or Canvas fallback)
+  // Browser environment or Sharp failed: use Photon (or Canvas fallback)
   const { PhotonImage, SamplingFilter, resize } = await getPhoton();
   const inputBytes = new Uint8Array(buffer);
   // Support both sync (Photon) and async (Canvas fallback) versions

package/src/logger.ts

@@ -8,53 +8,28 @@ import { ifInNode } from './utils';
 const topicPrefix = 'midscene';
 // Map to store file streams
 const logStreams = new Map<string, fs.WriteStream>();
-// A WriteStream queues every write made after it reports backpressure. The
-// main process can keep running while macOS has deprioritized a backgrounded
-// app, so retaining diagnostic logs here can otherwise grow without bound.
-// Drop best-effort logs until the stream drains instead.
-const backpressuredLogStreams = new Set<string>();
-const unavailableLogStreams = new Set<string>();
 // Map to store debug instances
 const debugInstances = new Map<string, DebugFunction>();
 
 // Function to get or create a log stream
-function getLogStream(topic: string): fs.WriteStream | null {
+function getLogStream(topic: string): fs.WriteStream {
   const topicFileName = topic.replace(/:/g, '-');
-  if (unavailableLogStreams.has(topicFileName)) {
-    return null;
-  }
   if (!logStreams.has(topicFileName)) {
     const logFile = path.join(
       getMidsceneRunSubDir('log'),
       `${topicFileName}.log`,
     );
     const stream = fs.createWriteStream(logFile, { flags: 'a' });
-    // A stream error without a listener terminates the Electron main process.
-    // Logging must remain best-effort, so disable this topic after a file error
-    // rather than repeatedly queuing writes to a broken stream.
-    stream.on('error', () => {
-      unavailableLogStreams.add(topicFileName);
-      backpressuredLogStreams.delete(topicFileName);
-      if (logStreams.get(topicFileName) === stream) {
-        logStreams.delete(topicFileName);
-      }
-    });
     logStreams.set(topicFileName, stream);
   }
-  return logStreams.get(topicFileName) ?? null;
+  return logStreams.get(topicFileName)!;
 }
 
 // Function to write log to file
 function writeLogToFile(topic: string, message: string): void {
   if (!ifInNode) return;
 
-  const topicFileName = topic.replace(/:/g, '-');
-  if (backpressuredLogStreams.has(topicFileName)) {
-    return;
-  }
-
   const stream = getLogStream(topic);
-  if (!stream) return;
   // Generate ISO format timestamp with local timezone
   const now = new Date();
   // Use sv-SE locale to get ISO-like format (YYYY-MM-DD HH:mm:ss)
@@ -72,17 +47,7 @@ function writeLogToFile(topic: string, message: string): void {
     .padStart(2, '0');
   const timezoneString = `${sign}${hours}:${minutes}`;
   const localISOTime = `${isoDate}T${isoTime}.${milliseconds}${timezoneString}`;
-  try {
-    if (!stream.write(`[${localISOTime}] ${message}\n`)) {
-      backpressuredLogStreams.add(topicFileName);
-      stream.once('drain', () => {
-        backpressuredLogStreams.delete(topicFileName);
-      });
-    }
-  } catch {
-    unavailableLogStreams.add(topicFileName);
-    backpressuredLogStreams.delete(topicFileName);
-  }
+  stream.write(`[${localISOTime}] ${message}\n`);
 }
 
 export type DebugFunction = (...args: unknown[]) => void;

package/src/recorder.ts

@@ -34,19 +34,6 @@ export interface MidsceneRecorderPageInfo {
   height: number;
 }
 
-/**
- * A screenshot stored outside the recording event payload.
- *
- * Recorder events are persisted in the Studio renderer. Keeping full data
- * URLs there makes long recordings retain every screenshot in the renderer
- * heap, so screenshot bytes live in the Playground run directory instead.
- */
-export interface MidsceneRecorderScreenshotAssetRef {
-  id: string;
-  mimeType: string;
-  bytes: number;
-}
-
 export interface MidsceneRecorderEvent {
   type: MidsceneRecorderEventType;
   source?: MidsceneRecorderSourceKind;
@@ -59,8 +46,6 @@ export interface MidsceneRecorderEvent {
   pageInfo: MidsceneRecorderPageInfo;
   screenshotBefore?: string;
   screenshotAfter?: string;
-  /** The single screenshot retained for AI description and Markdown export. */
-  screenshotAsset?: MidsceneRecorderScreenshotAssetRef;
   semantic?: MidsceneRecorderSemantic;
   elementDescription?: string;
   descriptionLoading?: boolean;