@midscene/shared 1.9.7-beta-20260616025249.0 → 1.9.7

package/dist/es/env/parse-model-config.mjs

@@ -5,7 +5,7 @@ import { assert } from "../utils.mjs";
 import { maskConfig, parseJson } from "./helper.mjs";
 import { initDebugConfig } from "./init-debug.mjs";
 const MODEL_CONFIG_DOC_URL = 'https://midscenejs.com/model-common-config.html';
-const getCurrentVersion = ()=>"1.9.7-beta-20260616025249.0";
+const getCurrentVersion = ()=>"1.9.7";
 const getInvalidModelFamilyMessage = (modelFamily)=>`Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}. Current version v${getCurrentVersion()} accepts the following model families: ${MODEL_FAMILY_VALUES.join(', ')}. You can also visit ${MODEL_CONFIG_DOC_URL} for the latest configuration information.`;
 const KEYS_MAP = {
     insight: INSIGHT_MODEL_CONFIG_KEYS,

package/dist/es/img/index.mjs

@@ -1,4 +1,4 @@
 import { imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, validateScreenshotBuffer } from "./info.mjs";
-import { createImgBase64ByFormat, cropByRect, httpImg2Base64, localImg2Base64, paddingToMatchBlockByBase64, parseBase64, preProcessImageUrl, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, zoomForGPT4o } from "./transform.mjs";
+import { createImgBase64ByFormat, cropByRect, httpImg2Base64, inferBase64ImageFormat, localImg2Base64, normalizeBase64Image, paddingToMatchBlockByBase64, parseBase64, preProcessImageUrl, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, zoomForGPT4o } from "./transform.mjs";
 import { annotateRects, compositeElementInfoImg, compositePointMarkerImg, processImageElementInfo } from "./box-select.mjs";
-export { annotateRects, compositeElementInfoImg, compositePointMarkerImg, createImgBase64ByFormat, cropByRect, httpImg2Base64, imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, localImg2Base64, paddingToMatchBlockByBase64, parseBase64, preProcessImageUrl, processImageElementInfo, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, validateScreenshotBuffer, zoomForGPT4o };
+export { annotateRects, compositeElementInfoImg, compositePointMarkerImg, createImgBase64ByFormat, cropByRect, httpImg2Base64, imageInfoOfBase64, inferBase64ImageFormat, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, localImg2Base64, normalizeBase64Image, paddingToMatchBlockByBase64, parseBase64, preProcessImageUrl, processImageElementInfo, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, validateScreenshotBuffer, zoomForGPT4o };

package/dist/es/img/transform.mjs

@@ -70,7 +70,26 @@ async function resizeAndConvertImgBuffer(inputFormat, inputData, newSize) {
     };
 }
 const normalizeBase64Body = (body)=>body.replace(/\s/g, '');
+const base64ImageDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i;
+const inferBase64ImageFormat = (base64Body)=>{
+    if (base64Body.startsWith('iVBORw0KGgo')) return 'png';
+    return 'jpeg';
+};
+function detectImageMimeTypeFromBuffer(buffer) {
+    if (buffer.length >= 8 && 0x89 === buffer[0] && 0x50 === buffer[1] && 0x4e === buffer[2] && 0x47 === buffer[3] && 0x0d === buffer[4] && 0x0a === buffer[5] && 0x1a === buffer[6] && 0x0a === buffer[7]) return 'image/png';
+    if (buffer.length >= 3 && 0xff === buffer[0] && 0xd8 === buffer[1] && 0xff === buffer[2]) return 'image/jpeg';
+    if (buffer.length >= 6 && 'GIF' === buffer.subarray(0, 3).toString('ascii')) return 'image/gif';
+    if (buffer.length >= 12 && 'RIFF' === buffer.subarray(0, 4).toString('ascii') && 'WEBP' === buffer.subarray(8, 12).toString('ascii')) return 'image/webp';
+    if (buffer.length >= 2 && 0x42 === buffer[0] && 0x4d === buffer[1]) return 'image/bmp';
+}
 const createImgBase64ByFormat = (format, body)=>`data:image/${format};base64,${normalizeBase64Body(body)}`;
+const normalizeBase64Image = (base64)=>{
+    const trimmedBase64 = base64.trim();
+    if (base64ImageDataUrlPattern.test(trimmedBase64)) return trimmedBase64;
+    const base64Body = normalizeBase64Body(trimmedBase64);
+    node_assert(base64Body, 'base64 image must include image data');
+    return createImgBase64ByFormat(inferBase64ImageFormat(base64Body), base64Body);
+};
 async function resizeImgBase64(inputBase64, newSize) {
     const { body, mimeType } = parseBase64(inputBase64);
     const imageBuffer = Buffer.from(body, 'base64');
@@ -198,7 +217,15 @@ const parseBase64 = (fullBase64String)=>{
     try {
         const separator = ';base64,';
         const index = fullBase64String.indexOf(separator);
-        if (-1 === index) throw new Error('Invalid base64 string');
+        if (-1 === index) {
+            const body = normalizeBase64Body(fullBase64String);
+            const mimeType = detectImageMimeTypeFromBuffer(Buffer.from(body, 'base64'));
+            if (!mimeType) throw new Error('Invalid base64 string');
+            return {
+                mimeType,
+                body
+            };
+        }
         return {
             mimeType: fullBase64String.slice(5, index),
             body: normalizeBase64Body(fullBase64String.slice(index + separator.length))
@@ -266,4 +293,4 @@ async function scaleImage(imageBase64, scale) {
         imageBase64: base64
     };
 }
-export { createImgBase64ByFormat, cropByRect, httpImg2Base64, localImg2Base64, normalizeBase64Body, paddingToMatchBlock, paddingToMatchBlockByBase64, parseBase64, photonFromBase64, photonToBase64, preProcessImageUrl, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, zoomForGPT4o };
+export { createImgBase64ByFormat, cropByRect, httpImg2Base64, inferBase64ImageFormat, localImg2Base64, normalizeBase64Body, normalizeBase64Image, paddingToMatchBlock, paddingToMatchBlockByBase64, parseBase64, photonFromBase64, photonToBase64, preProcessImageUrl, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, zoomForGPT4o };

package/dist/es/recorder.mjs

@@ -2,10 +2,87 @@ const DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS = 20;
 function isMidsceneRecorderPendingDescription(value) {
     return value?.trim() === 'AI is analyzing element...';
 }
+function getMidsceneRecorderSemantic(event) {
+    return event.semantic;
+}
+function getRecorderPointerActionVerb(actionType) {
+    switch(actionType){
+        case 'Tap':
+            return 'Tap';
+        case 'DoubleClick':
+            return 'Double click';
+        case 'LongPress':
+            return 'Long press';
+        case 'RightClick':
+            return 'Right click';
+        default:
+            return 'Click';
+    }
+}
+function getRecorderDragActionVerb(actionType) {
+    switch(actionType){
+        case 'Swipe':
+            return 'Swipe';
+        case 'DragAndDrop':
+            return 'Drag';
+        default:
+            return 'Drag';
+    }
+}
+function buildMidsceneRecorderReplayInstruction(event, elementDescription) {
+    switch(event.type){
+        case 'navigation':
+            if ('Stop' === event.actionType) return 'Stop loading the current page.';
+            if ('GoBack' === event.actionType) return 'Go back in the browser.';
+            if ('GoForward' === event.actionType) return 'Go forward in the browser.';
+            if ('Reload' === event.actionType) return 'Reload the current page.';
+            if ('NavigationChanged' === event.actionType && event.url) return `Wait for navigation to complete at \`${event.url}\`.`;
+            return event.url ? `Navigate to \`${event.url}\`.` : `Navigate using ${elementDescription}.`;
+        case 'scroll':
+            return event.scrollDestinationDescription ? `Scroll the page/region with description "${elementDescription}" by value "${event.value || 'down'}" until "${event.scrollDestinationDescription}" is visible.` : `Scroll the page/region with description "${elementDescription}" by value "${event.value || 'down'}".`;
+        case 'drag':
+            {
+                const verb = getRecorderDragActionVerb(event.actionType);
+                return `${verb} through the area described as "${elementDescription}".`;
+            }
+        case 'input':
+            return `Input "${event.value || ''}" into the element described as "${elementDescription}".`;
+        case 'keydown':
+            return `Press "${event.value || 'the recorded key'}" on the element described as "${elementDescription}".`;
+        default:
+            {
+                const verb = getRecorderPointerActionVerb(event.actionType);
+                if ('Long press' === verb) return `${verb} the element described as "${elementDescription}".`;
+                return `${verb} on the element described as "${elementDescription}".`;
+            }
+    }
+}
+function buildMidsceneRecorderActionSummary(event, elementDescription) {
+    switch(event.type){
+        case 'navigation':
+            if ('Stop' === event.actionType) return 'Stop page loading';
+            if ('GoBack' === event.actionType) return 'Go back';
+            if ('GoForward' === event.actionType) return 'Go forward';
+            if ('Reload' === event.actionType) return 'Reload page';
+            if ('NavigationChanged' === event.actionType && event.url) return `Wait for navigation to complete at ${event.url}`;
+            return event.url ? `Navigate to ${event.url}` : 'Navigate';
+        case 'scroll':
+            return event.scrollDestinationDescription ? `Scroll ${elementDescription} toward ${event.scrollDestinationDescription}` : `Scroll ${elementDescription}`;
+        case 'drag':
+            return `${getRecorderDragActionVerb(event.actionType)} ${elementDescription}`;
+        case 'input':
+            return `Input into ${elementDescription}`;
+        case 'keydown':
+            return `Press ${event.value || 'key'} on ${elementDescription}`;
+        default:
+            return `${getRecorderPointerActionVerb(event.actionType)} ${elementDescription}`;
+    }
+}
 function getMidsceneRecorderEventDescription(event) {
-    if (event.actionSummary && !isMidsceneRecorderPendingDescription(event.actionSummary)) return event.actionSummary;
-    if (event.elementDescription && !isMidsceneRecorderPendingDescription(event.elementDescription)) return event.elementDescription;
-    if (event.replayInstruction && !isMidsceneRecorderPendingDescription(event.replayInstruction)) return event.replayInstruction;
+    const semantic = getMidsceneRecorderSemantic(event);
+    if (semantic?.actionSummary && !isMidsceneRecorderPendingDescription(semantic.actionSummary)) return semantic.actionSummary;
+    if (semantic?.elementDescription && !isMidsceneRecorderPendingDescription(semantic.elementDescription)) return semantic.elementDescription;
+    if (semantic?.replayInstruction && !isMidsceneRecorderPendingDescription(semantic.replayInstruction)) return semantic.replayInstruction;
     if ('navigation' === event.type && event.url) return `Navigate to ${event.url}`;
     if (event.value) return event.actionType ? `${event.actionType} ${event.value}` : event.value;
     if (event.elementRect?.x !== void 0 && event.elementRect?.y !== void 0) {
@@ -54,10 +131,12 @@ function getRecorderEventScreenshot(event) {
     return event.screenshotWithBox || event.screenshotAfter || event.screenshotBefore;
 }
 function hasCoordinateFallback(event) {
-    return !event.elementDescription && event.elementRect?.x !== void 0 && event.elementRect?.y !== void 0;
+    const semantic = getMidsceneRecorderSemantic(event);
+    return !semantic?.elementDescription && event.elementRect?.x !== void 0 && event.elementRect?.y !== void 0;
 }
 function shouldIncludeMarkdownScreenshot(event, eventIndex, lastEventIndex) {
-    return 0 === eventIndex || eventIndex === lastEventIndex || 'navigation' === event.type || 'scroll' === event.type || 'input' === event.type || Boolean(event.screenshotWithBox) || !event.elementDescription || hasCoordinateFallback(event);
+    const semantic = getMidsceneRecorderSemantic(event);
+    return 0 === eventIndex || eventIndex === lastEventIndex || 'navigation' === event.type || 'scroll' === event.type || 'input' === event.type || Boolean(event.screenshotWithBox) || !semantic?.elementDescription || hasCoordinateFallback(event);
 }
 function getRecorderScreenshotCandidatePriority(candidate, firstEventIndex, lastEventIndex) {
     const event = candidate.event;
@@ -66,9 +145,10 @@ function getRecorderScreenshotCandidatePriority(candidate, firstEventIndex, last
     if (candidate.eventIndex === lastEventIndex) priority += 95;
     if ('navigation' === event.type) priority += 80;
     if (event.screenshotWithBox) priority += 70;
-    if ('fallback' === event.descriptionSource || 'low' === event.semanticConfidence || event.descriptionError) priority += 60;
+    const semantic = getMidsceneRecorderSemantic(event);
+    if (semantic?.source === 'heuristic' || semantic?.confidence === 'low' || semantic?.error) priority += 60;
     if ('input' === event.type || 'scroll' === event.type) priority += 40;
-    if (!event.elementDescription || hasCoordinateFallback(event)) priority += 30;
+    if (!semantic?.elementDescription || hasCoordinateFallback(event)) priority += 30;
     return priority;
 }
 function selectEvenlyDistributedCandidates(candidates, count) {
@@ -166,4 +246,4 @@ function stringifyMidsceneRecorderTargetBlock(target) {
     for (const [key, value] of values)lines.push(`  ${key}: ${scalarToYaml(value)}`);
     return lines.join('\n');
 }
-export { DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS, createMidsceneRecorderMarkdownScreenshotAssets, getMidsceneRecorderEventDescription, getMidsceneRecorderScreenshotsForLLM, sanitizeMidsceneRecorderFileName, stringifyMidsceneRecorderTargetBlock };
+export { DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS, buildMidsceneRecorderActionSummary, buildMidsceneRecorderReplayInstruction, createMidsceneRecorderMarkdownScreenshotAssets, getMidsceneRecorderEventDescription, getMidsceneRecorderScreenshotsForLLM, getMidsceneRecorderSemantic, sanitizeMidsceneRecorderFileName, stringifyMidsceneRecorderTargetBlock };

package/dist/lib/env/parse-model-config.js

@@ -37,7 +37,7 @@ const external_utils_js_namespaceObject = require("../utils.js");
 const external_helper_js_namespaceObject = require("./helper.js");
 const external_init_debug_js_namespaceObject = require("./init-debug.js");
 const MODEL_CONFIG_DOC_URL = 'https://midscenejs.com/model-common-config.html';
-const getCurrentVersion = ()=>"1.9.7-beta-20260616025249.0";
+const getCurrentVersion = ()=>"1.9.7";
 const getInvalidModelFamilyMessage = (modelFamily)=>`Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}. Current version v${getCurrentVersion()} accepts the following model families: ${external_types_js_namespaceObject.MODEL_FAMILY_VALUES.join(', ')}. You can also visit ${MODEL_CONFIG_DOC_URL} for the latest configuration information.`;
 const KEYS_MAP = {
     insight: external_constants_js_namespaceObject.INSIGHT_MODEL_CONFIG_KEYS,

package/dist/lib/img/index.js

@@ -25,6 +25,7 @@ var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
     paddingToMatchBlockByBase64: ()=>external_transform_js_namespaceObject.paddingToMatchBlockByBase64,
+    inferBase64ImageFormat: ()=>external_transform_js_namespaceObject.inferBase64ImageFormat,
     localImg2Base64: ()=>external_transform_js_namespaceObject.localImg2Base64,
     parseBase64: ()=>external_transform_js_namespaceObject.parseBase64,
     resizeAndConvertImgBuffer: ()=>external_transform_js_namespaceObject.resizeAndConvertImgBuffer,
@@ -36,10 +37,11 @@ __webpack_require__.d(__webpack_exports__, {
     annotateRects: ()=>external_box_select_js_namespaceObject.annotateRects,
     compositeElementInfoImg: ()=>external_box_select_js_namespaceObject.compositeElementInfoImg,
     isValidPNGImageBuffer: ()=>external_info_js_namespaceObject.isValidPNGImageBuffer,
-    cropByRect: ()=>external_transform_js_namespaceObject.cropByRect,
+    normalizeBase64Image: ()=>external_transform_js_namespaceObject.normalizeBase64Image,
     compositePointMarkerImg: ()=>external_box_select_js_namespaceObject.compositePointMarkerImg,
-    processImageElementInfo: ()=>external_box_select_js_namespaceObject.processImageElementInfo,
+    cropByRect: ()=>external_transform_js_namespaceObject.cropByRect,
     isValidJPEGImageBuffer: ()=>external_info_js_namespaceObject.isValidJPEGImageBuffer,
+    processImageElementInfo: ()=>external_box_select_js_namespaceObject.processImageElementInfo,
     scaleImage: ()=>external_transform_js_namespaceObject.scaleImage,
     zoomForGPT4o: ()=>external_transform_js_namespaceObject.zoomForGPT4o,
     createImgBase64ByFormat: ()=>external_transform_js_namespaceObject.createImgBase64ByFormat,
@@ -56,10 +58,12 @@ exports.createImgBase64ByFormat = __webpack_exports__.createImgBase64ByFormat;
 exports.cropByRect = __webpack_exports__.cropByRect;
 exports.httpImg2Base64 = __webpack_exports__.httpImg2Base64;
 exports.imageInfoOfBase64 = __webpack_exports__.imageInfoOfBase64;
+exports.inferBase64ImageFormat = __webpack_exports__.inferBase64ImageFormat;
 exports.isValidImageBuffer = __webpack_exports__.isValidImageBuffer;
 exports.isValidJPEGImageBuffer = __webpack_exports__.isValidJPEGImageBuffer;
 exports.isValidPNGImageBuffer = __webpack_exports__.isValidPNGImageBuffer;
 exports.localImg2Base64 = __webpack_exports__.localImg2Base64;
+exports.normalizeBase64Image = __webpack_exports__.normalizeBase64Image;
 exports.paddingToMatchBlockByBase64 = __webpack_exports__.paddingToMatchBlockByBase64;
 exports.parseBase64 = __webpack_exports__.parseBase64;
 exports.preProcessImageUrl = __webpack_exports__.preProcessImageUrl;
@@ -78,10 +82,12 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "cropByRect",
     "httpImg2Base64",
     "imageInfoOfBase64",
+    "inferBase64ImageFormat",
     "isValidImageBuffer",
     "isValidJPEGImageBuffer",
     "isValidPNGImageBuffer",
     "localImg2Base64",
+    "normalizeBase64Image",
     "paddingToMatchBlockByBase64",
     "parseBase64",
     "preProcessImageUrl",

package/dist/lib/img/transform.js

@@ -34,6 +34,7 @@ var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
     paddingToMatchBlockByBase64: ()=>paddingToMatchBlockByBase64,
+    inferBase64ImageFormat: ()=>inferBase64ImageFormat,
     localImg2Base64: ()=>localImg2Base64,
     photonFromBase64: ()=>photonFromBase64,
     photonToBase64: ()=>photonToBase64,
@@ -44,6 +45,7 @@ __webpack_require__.d(__webpack_exports__, {
     preProcessImageUrl: ()=>preProcessImageUrl,
     cropByRect: ()=>cropByRect,
     scaleImage: ()=>scaleImage,
+    normalizeBase64Image: ()=>normalizeBase64Image,
     normalizeBase64Body: ()=>normalizeBase64Body,
     zoomForGPT4o: ()=>zoomForGPT4o,
     createImgBase64ByFormat: ()=>createImgBase64ByFormat,
@@ -126,7 +128,26 @@ async function resizeAndConvertImgBuffer(inputFormat, inputData, newSize) {
     };
 }
 const normalizeBase64Body = (body)=>body.replace(/\s/g, '');
+const base64ImageDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i;
+const inferBase64ImageFormat = (base64Body)=>{
+    if (base64Body.startsWith('iVBORw0KGgo')) return 'png';
+    return 'jpeg';
+};
+function detectImageMimeTypeFromBuffer(buffer) {
+    if (buffer.length >= 8 && 0x89 === buffer[0] && 0x50 === buffer[1] && 0x4e === buffer[2] && 0x47 === buffer[3] && 0x0d === buffer[4] && 0x0a === buffer[5] && 0x1a === buffer[6] && 0x0a === buffer[7]) return 'image/png';
+    if (buffer.length >= 3 && 0xff === buffer[0] && 0xd8 === buffer[1] && 0xff === buffer[2]) return 'image/jpeg';
+    if (buffer.length >= 6 && 'GIF' === buffer.subarray(0, 3).toString('ascii')) return 'image/gif';
+    if (buffer.length >= 12 && 'RIFF' === buffer.subarray(0, 4).toString('ascii') && 'WEBP' === buffer.subarray(8, 12).toString('ascii')) return 'image/webp';
+    if (buffer.length >= 2 && 0x42 === buffer[0] && 0x4d === buffer[1]) return 'image/bmp';
+}
 const createImgBase64ByFormat = (format, body)=>`data:image/${format};base64,${normalizeBase64Body(body)}`;
+const normalizeBase64Image = (base64)=>{
+    const trimmedBase64 = base64.trim();
+    if (base64ImageDataUrlPattern.test(trimmedBase64)) return trimmedBase64;
+    const base64Body = normalizeBase64Body(trimmedBase64);
+    external_node_assert_default()(base64Body, 'base64 image must include image data');
+    return createImgBase64ByFormat(inferBase64ImageFormat(base64Body), base64Body);
+};
 async function resizeImgBase64(inputBase64, newSize) {
     const { body, mimeType } = parseBase64(inputBase64);
     const imageBuffer = external_node_buffer_namespaceObject.Buffer.from(body, 'base64');
@@ -254,7 +275,15 @@ const parseBase64 = (fullBase64String)=>{
     try {
         const separator = ';base64,';
         const index = fullBase64String.indexOf(separator);
-        if (-1 === index) throw new Error('Invalid base64 string');
+        if (-1 === index) {
+            const body = normalizeBase64Body(fullBase64String);
+            const mimeType = detectImageMimeTypeFromBuffer(external_node_buffer_namespaceObject.Buffer.from(body, 'base64'));
+            if (!mimeType) throw new Error('Invalid base64 string');
+            return {
+                mimeType,
+                body
+            };
+        }
         return {
             mimeType: fullBase64String.slice(5, index),
             body: normalizeBase64Body(fullBase64String.slice(index + separator.length))
@@ -325,8 +354,10 @@ async function scaleImage(imageBase64, scale) {
 exports.createImgBase64ByFormat = __webpack_exports__.createImgBase64ByFormat;
 exports.cropByRect = __webpack_exports__.cropByRect;
 exports.httpImg2Base64 = __webpack_exports__.httpImg2Base64;
+exports.inferBase64ImageFormat = __webpack_exports__.inferBase64ImageFormat;
 exports.localImg2Base64 = __webpack_exports__.localImg2Base64;
 exports.normalizeBase64Body = __webpack_exports__.normalizeBase64Body;
+exports.normalizeBase64Image = __webpack_exports__.normalizeBase64Image;
 exports.paddingToMatchBlock = __webpack_exports__.paddingToMatchBlock;
 exports.paddingToMatchBlockByBase64 = __webpack_exports__.paddingToMatchBlockByBase64;
 exports.parseBase64 = __webpack_exports__.parseBase64;
@@ -342,8 +373,10 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "createImgBase64ByFormat",
     "cropByRect",
     "httpImg2Base64",
+    "inferBase64ImageFormat",
     "localImg2Base64",
     "normalizeBase64Body",
+    "normalizeBase64Image",
     "paddingToMatchBlock",
     "paddingToMatchBlockByBase64",
     "parseBase64",

package/dist/lib/recorder.js

@@ -25,9 +25,12 @@ var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
     DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS: ()=>DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS,
+    buildMidsceneRecorderActionSummary: ()=>buildMidsceneRecorderActionSummary,
+    buildMidsceneRecorderReplayInstruction: ()=>buildMidsceneRecorderReplayInstruction,
     createMidsceneRecorderMarkdownScreenshotAssets: ()=>createMidsceneRecorderMarkdownScreenshotAssets,
     getMidsceneRecorderEventDescription: ()=>getMidsceneRecorderEventDescription,
     getMidsceneRecorderScreenshotsForLLM: ()=>getMidsceneRecorderScreenshotsForLLM,
+    getMidsceneRecorderSemantic: ()=>getMidsceneRecorderSemantic,
     sanitizeMidsceneRecorderFileName: ()=>sanitizeMidsceneRecorderFileName,
     stringifyMidsceneRecorderTargetBlock: ()=>stringifyMidsceneRecorderTargetBlock
 });
@@ -35,10 +38,87 @@ const DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS = 20;
 function isMidsceneRecorderPendingDescription(value) {
     return value?.trim() === 'AI is analyzing element...';
 }
+function getMidsceneRecorderSemantic(event) {
+    return event.semantic;
+}
+function getRecorderPointerActionVerb(actionType) {
+    switch(actionType){
+        case 'Tap':
+            return 'Tap';
+        case 'DoubleClick':
+            return 'Double click';
+        case 'LongPress':
+            return 'Long press';
+        case 'RightClick':
+            return 'Right click';
+        default:
+            return 'Click';
+    }
+}
+function getRecorderDragActionVerb(actionType) {
+    switch(actionType){
+        case 'Swipe':
+            return 'Swipe';
+        case 'DragAndDrop':
+            return 'Drag';
+        default:
+            return 'Drag';
+    }
+}
+function buildMidsceneRecorderReplayInstruction(event, elementDescription) {
+    switch(event.type){
+        case 'navigation':
+            if ('Stop' === event.actionType) return 'Stop loading the current page.';
+            if ('GoBack' === event.actionType) return 'Go back in the browser.';
+            if ('GoForward' === event.actionType) return 'Go forward in the browser.';
+            if ('Reload' === event.actionType) return 'Reload the current page.';
+            if ('NavigationChanged' === event.actionType && event.url) return `Wait for navigation to complete at \`${event.url}\`.`;
+            return event.url ? `Navigate to \`${event.url}\`.` : `Navigate using ${elementDescription}.`;
+        case 'scroll':
+            return event.scrollDestinationDescription ? `Scroll the page/region with description "${elementDescription}" by value "${event.value || 'down'}" until "${event.scrollDestinationDescription}" is visible.` : `Scroll the page/region with description "${elementDescription}" by value "${event.value || 'down'}".`;
+        case 'drag':
+            {
+                const verb = getRecorderDragActionVerb(event.actionType);
+                return `${verb} through the area described as "${elementDescription}".`;
+            }
+        case 'input':
+            return `Input "${event.value || ''}" into the element described as "${elementDescription}".`;
+        case 'keydown':
+            return `Press "${event.value || 'the recorded key'}" on the element described as "${elementDescription}".`;
+        default:
+            {
+                const verb = getRecorderPointerActionVerb(event.actionType);
+                if ('Long press' === verb) return `${verb} the element described as "${elementDescription}".`;
+                return `${verb} on the element described as "${elementDescription}".`;
+            }
+    }
+}
+function buildMidsceneRecorderActionSummary(event, elementDescription) {
+    switch(event.type){
+        case 'navigation':
+            if ('Stop' === event.actionType) return 'Stop page loading';
+            if ('GoBack' === event.actionType) return 'Go back';
+            if ('GoForward' === event.actionType) return 'Go forward';
+            if ('Reload' === event.actionType) return 'Reload page';
+            if ('NavigationChanged' === event.actionType && event.url) return `Wait for navigation to complete at ${event.url}`;
+            return event.url ? `Navigate to ${event.url}` : 'Navigate';
+        case 'scroll':
+            return event.scrollDestinationDescription ? `Scroll ${elementDescription} toward ${event.scrollDestinationDescription}` : `Scroll ${elementDescription}`;
+        case 'drag':
+            return `${getRecorderDragActionVerb(event.actionType)} ${elementDescription}`;
+        case 'input':
+            return `Input into ${elementDescription}`;
+        case 'keydown':
+            return `Press ${event.value || 'key'} on ${elementDescription}`;
+        default:
+            return `${getRecorderPointerActionVerb(event.actionType)} ${elementDescription}`;
+    }
+}
 function getMidsceneRecorderEventDescription(event) {
-    if (event.actionSummary && !isMidsceneRecorderPendingDescription(event.actionSummary)) return event.actionSummary;
-    if (event.elementDescription && !isMidsceneRecorderPendingDescription(event.elementDescription)) return event.elementDescription;
-    if (event.replayInstruction && !isMidsceneRecorderPendingDescription(event.replayInstruction)) return event.replayInstruction;
+    const semantic = getMidsceneRecorderSemantic(event);
+    if (semantic?.actionSummary && !isMidsceneRecorderPendingDescription(semantic.actionSummary)) return semantic.actionSummary;
+    if (semantic?.elementDescription && !isMidsceneRecorderPendingDescription(semantic.elementDescription)) return semantic.elementDescription;
+    if (semantic?.replayInstruction && !isMidsceneRecorderPendingDescription(semantic.replayInstruction)) return semantic.replayInstruction;
     if ('navigation' === event.type && event.url) return `Navigate to ${event.url}`;
     if (event.value) return event.actionType ? `${event.actionType} ${event.value}` : event.value;
     if (event.elementRect?.x !== void 0 && event.elementRect?.y !== void 0) {
@@ -87,10 +167,12 @@ function getRecorderEventScreenshot(event) {
     return event.screenshotWithBox || event.screenshotAfter || event.screenshotBefore;
 }
 function hasCoordinateFallback(event) {
-    return !event.elementDescription && event.elementRect?.x !== void 0 && event.elementRect?.y !== void 0;
+    const semantic = getMidsceneRecorderSemantic(event);
+    return !semantic?.elementDescription && event.elementRect?.x !== void 0 && event.elementRect?.y !== void 0;
 }
 function shouldIncludeMarkdownScreenshot(event, eventIndex, lastEventIndex) {
-    return 0 === eventIndex || eventIndex === lastEventIndex || 'navigation' === event.type || 'scroll' === event.type || 'input' === event.type || Boolean(event.screenshotWithBox) || !event.elementDescription || hasCoordinateFallback(event);
+    const semantic = getMidsceneRecorderSemantic(event);
+    return 0 === eventIndex || eventIndex === lastEventIndex || 'navigation' === event.type || 'scroll' === event.type || 'input' === event.type || Boolean(event.screenshotWithBox) || !semantic?.elementDescription || hasCoordinateFallback(event);
 }
 function getRecorderScreenshotCandidatePriority(candidate, firstEventIndex, lastEventIndex) {
     const event = candidate.event;
@@ -99,9 +181,10 @@ function getRecorderScreenshotCandidatePriority(candidate, firstEventIndex, last
     if (candidate.eventIndex === lastEventIndex) priority += 95;
     if ('navigation' === event.type) priority += 80;
     if (event.screenshotWithBox) priority += 70;
-    if ('fallback' === event.descriptionSource || 'low' === event.semanticConfidence || event.descriptionError) priority += 60;
+    const semantic = getMidsceneRecorderSemantic(event);
+    if (semantic?.source === 'heuristic' || semantic?.confidence === 'low' || semantic?.error) priority += 60;
     if ('input' === event.type || 'scroll' === event.type) priority += 40;
-    if (!event.elementDescription || hasCoordinateFallback(event)) priority += 30;
+    if (!semantic?.elementDescription || hasCoordinateFallback(event)) priority += 30;
     return priority;
 }
 function selectEvenlyDistributedCandidates(candidates, count) {
@@ -200,16 +283,22 @@ function stringifyMidsceneRecorderTargetBlock(target) {
     return lines.join('\n');
 }
 exports.DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS = __webpack_exports__.DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS;
+exports.buildMidsceneRecorderActionSummary = __webpack_exports__.buildMidsceneRecorderActionSummary;
+exports.buildMidsceneRecorderReplayInstruction = __webpack_exports__.buildMidsceneRecorderReplayInstruction;
 exports.createMidsceneRecorderMarkdownScreenshotAssets = __webpack_exports__.createMidsceneRecorderMarkdownScreenshotAssets;
 exports.getMidsceneRecorderEventDescription = __webpack_exports__.getMidsceneRecorderEventDescription;
 exports.getMidsceneRecorderScreenshotsForLLM = __webpack_exports__.getMidsceneRecorderScreenshotsForLLM;
+exports.getMidsceneRecorderSemantic = __webpack_exports__.getMidsceneRecorderSemantic;
 exports.sanitizeMidsceneRecorderFileName = __webpack_exports__.sanitizeMidsceneRecorderFileName;
 exports.stringifyMidsceneRecorderTargetBlock = __webpack_exports__.stringifyMidsceneRecorderTargetBlock;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
     "DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS",
+    "buildMidsceneRecorderActionSummary",
+    "buildMidsceneRecorderReplayInstruction",
     "createMidsceneRecorderMarkdownScreenshotAssets",
     "getMidsceneRecorderEventDescription",
     "getMidsceneRecorderScreenshotsForLLM",
+    "getMidsceneRecorderSemantic",
     "sanitizeMidsceneRecorderFileName",
     "stringifyMidsceneRecorderTargetBlock"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];

package/dist/types/img/index.d.ts

@@ -1,3 +1,3 @@
 export { imageInfoOfBase64, isValidPNGImageBuffer, isValidJPEGImageBuffer, isValidImageBuffer, validateScreenshotBuffer, type ValidateScreenshotBufferOptions, } from './info';
-export { resizeAndConvertImgBuffer, resizeImgBase64, zoomForGPT4o, saveBase64Image, paddingToMatchBlockByBase64, cropByRect, scaleImage, localImg2Base64, httpImg2Base64, preProcessImageUrl, parseBase64, createImgBase64ByFormat, } from './transform';
+export { resizeAndConvertImgBuffer, resizeImgBase64, zoomForGPT4o, saveBase64Image, paddingToMatchBlockByBase64, cropByRect, scaleImage, localImg2Base64, httpImg2Base64, preProcessImageUrl, parseBase64, createImgBase64ByFormat, inferBase64ImageFormat, normalizeBase64Image, } from './transform';
 export { processImageElementInfo, compositeElementInfoImg, compositePointMarkerImg, annotateRects, } from './box-select';

package/dist/types/img/transform.d.ts

@@ -27,7 +27,9 @@ export declare function resizeAndConvertImgBuffer(inputFormat: string, inputData
     format: string;
 }>;
 export declare const normalizeBase64Body: (body: string) => string;
+export declare const inferBase64ImageFormat: (base64Body: string) => "jpeg" | "png";
 export declare const createImgBase64ByFormat: (format: string, body: string) => string;
+export declare const normalizeBase64Image: (base64: string) => string;
 export declare function resizeImgBase64(inputBase64: string, newSize: {
     width: number;
     height: number;

package/dist/types/recorder.d.ts

@@ -25,16 +25,43 @@ export interface MidsceneRecorderEvent {
     pageInfo: MidsceneRecorderPageInfo;
     screenshotBefore?: string;
     screenshotAfter?: string;
+    semantic?: MidsceneRecorderSemantic;
     elementDescription?: string;
-    replayInstruction?: string;
-    actionSummary?: string;
-    semanticConfidence?: 'high' | 'medium' | 'low';
     descriptionLoading?: boolean;
-    descriptionSource?: 'ai' | 'fallback';
-    descriptionError?: string;
     screenshotWithBox?: string;
     timestamp: number;
     hashId: string;
+    mergedHashIds?: string[];
+}
+export type MidsceneRecorderSemanticSource = 'aiDescribe' | 'recorderAI' | 'heuristic';
+export type MidsceneRecorderSemanticStatus = 'pending' | 'ready' | 'failed';
+export type MidsceneRecorderSemanticConfidence = 'high' | 'medium' | 'low';
+export interface MidsceneRecorderSemanticAiDescribe {
+    verifyPrompt: boolean;
+    verifyPassed?: boolean;
+    deepLocate?: boolean;
+    centerDistance?: number;
+    expectedCenter?: [number, number];
+    actualCenter?: [number, number];
+    annotatedScreenshotPath?: string;
+}
+export interface MidsceneRecorderSemantic {
+    source: MidsceneRecorderSemanticSource;
+    status: MidsceneRecorderSemanticStatus;
+    elementDescription?: string;
+    replayInstruction?: string;
+    actionSummary?: string;
+    confidence?: MidsceneRecorderSemanticConfidence;
+    error?: string;
+    aiDescribe?: MidsceneRecorderSemanticAiDescribe;
+    fallbackFrom?: MidsceneRecorderSemantic;
+}
+export interface MidsceneRecorderSemanticAction {
+    type: MidsceneRecorderEventType;
+    actionType?: string;
+    value?: string;
+    url?: string;
+    scrollDestinationDescription?: string;
 }
 export interface MidsceneRecorderTarget {
     platformId: MidsceneRecorderPlatformId;
@@ -62,6 +89,9 @@ export interface MidsceneRecorderMarkdownScreenshotOptions {
     maxScreenshots?: number;
 }
 export declare const DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS = 20;
+export declare function getMidsceneRecorderSemantic(event: Pick<MidsceneRecorderEvent, 'semantic'>): MidsceneRecorderSemantic | undefined;
+export declare function buildMidsceneRecorderReplayInstruction(event: MidsceneRecorderSemanticAction, elementDescription: string): string;
+export declare function buildMidsceneRecorderActionSummary(event: MidsceneRecorderSemanticAction, elementDescription: string): string;
 export declare function getMidsceneRecorderEventDescription(event: MidsceneRecorderEvent): string;
 export declare function getMidsceneRecorderScreenshotsForLLM(events: MidsceneRecorderEvent[], maxScreenshots?: number): string[];
 export declare function sanitizeMidsceneRecorderFileName(value: string): string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/shared",
-  "version": "1.9.7-beta-20260616025249.0",
+  "version": "1.9.7",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",

package/src/img/index.ts

@@ -19,6 +19,8 @@ export {
   preProcessImageUrl,
   parseBase64,
   createImgBase64ByFormat,
+  inferBase64ImageFormat,
+  normalizeBase64Image,
 } from './transform';
 export {
   processImageElementInfo,

package/src/img/transform.ts

@@ -156,10 +156,71 @@ export async function resizeAndConvertImgBuffer(
 
 export const normalizeBase64Body = (body: string) => body.replace(/\s/g, '');
 
+const base64ImageDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i;
+
+export const inferBase64ImageFormat = (base64Body: string) => {
+  if (base64Body.startsWith('iVBORw0KGgo')) {
+    return 'png';
+  }
+  return 'jpeg';
+};
+
+function detectImageMimeTypeFromBuffer(buffer: Buffer): string | undefined {
+  if (
+    buffer.length >= 8 &&
+    buffer[0] === 0x89 &&
+    buffer[1] === 0x50 &&
+    buffer[2] === 0x4e &&
+    buffer[3] === 0x47 &&
+    buffer[4] === 0x0d &&
+    buffer[5] === 0x0a &&
+    buffer[6] === 0x1a &&
+    buffer[7] === 0x0a
+  ) {
+    return 'image/png';
+  }
+  if (
+    buffer.length >= 3 &&
+    buffer[0] === 0xff &&
+    buffer[1] === 0xd8 &&
+    buffer[2] === 0xff
+  ) {
+    return 'image/jpeg';
+  }
+  if (buffer.length >= 6 && buffer.subarray(0, 3).toString('ascii') === 'GIF') {
+    return 'image/gif';
+  }
+  if (
+    buffer.length >= 12 &&
+    buffer.subarray(0, 4).toString('ascii') === 'RIFF' &&
+    buffer.subarray(8, 12).toString('ascii') === 'WEBP'
+  ) {
+    return 'image/webp';
+  }
+  if (buffer.length >= 2 && buffer[0] === 0x42 && buffer[1] === 0x4d) {
+    return 'image/bmp';
+  }
+  return undefined;
+}
+
 export const createImgBase64ByFormat = (format: string, body: string) => {
   return `data:image/${format};base64,${normalizeBase64Body(body)}`;
 };
 
+export const normalizeBase64Image = (base64: string) => {
+  const trimmedBase64 = base64.trim();
+  if (base64ImageDataUrlPattern.test(trimmedBase64)) {
+    return trimmedBase64;
+  }
+
+  const base64Body = normalizeBase64Body(trimmedBase64);
+  assert(base64Body, 'base64 image must include image data');
+  return createImgBase64ByFormat(
+    inferBase64ImageFormat(base64Body),
+    base64Body,
+  );
+};
+
 export async function resizeImgBase64(
   inputBase64: string,
   newSize: {
@@ -411,7 +472,14 @@ export const parseBase64 = (
     const separator = ';base64,';
     const index = fullBase64String.indexOf(separator);
     if (index === -1) {
-      throw new Error('Invalid base64 string');
+      const body = normalizeBase64Body(fullBase64String);
+      const mimeType = detectImageMimeTypeFromBuffer(
+        Buffer.from(body, 'base64'),
+      );
+      if (!mimeType) {
+        throw new Error('Invalid base64 string');
+      }
+      return { mimeType, body };
     }
     return {
       // 5 means 'data:'

package/src/recorder.ts

@@ -46,16 +46,52 @@ export interface MidsceneRecorderEvent {
   pageInfo: MidsceneRecorderPageInfo;
   screenshotBefore?: string;
   screenshotAfter?: string;
+  semantic?: MidsceneRecorderSemantic;
   elementDescription?: string;
-  replayInstruction?: string;
-  actionSummary?: string;
-  semanticConfidence?: 'high' | 'medium' | 'low';
   descriptionLoading?: boolean;
-  descriptionSource?: 'ai' | 'fallback';
-  descriptionError?: string;
   screenshotWithBox?: string;
   timestamp: number;
   hashId: string;
+  mergedHashIds?: string[];
+}
+
+export type MidsceneRecorderSemanticSource =
+  | 'aiDescribe'
+  | 'recorderAI'
+  | 'heuristic';
+
+export type MidsceneRecorderSemanticStatus = 'pending' | 'ready' | 'failed';
+
+export type MidsceneRecorderSemanticConfidence = 'high' | 'medium' | 'low';
+
+export interface MidsceneRecorderSemanticAiDescribe {
+  verifyPrompt: boolean;
+  verifyPassed?: boolean;
+  deepLocate?: boolean;
+  centerDistance?: number;
+  expectedCenter?: [number, number];
+  actualCenter?: [number, number];
+  annotatedScreenshotPath?: string;
+}
+
+export interface MidsceneRecorderSemantic {
+  source: MidsceneRecorderSemanticSource;
+  status: MidsceneRecorderSemanticStatus;
+  elementDescription?: string;
+  replayInstruction?: string;
+  actionSummary?: string;
+  confidence?: MidsceneRecorderSemanticConfidence;
+  error?: string;
+  aiDescribe?: MidsceneRecorderSemanticAiDescribe;
+  fallbackFrom?: MidsceneRecorderSemantic;
+}
+
+export interface MidsceneRecorderSemanticAction {
+  type: MidsceneRecorderEventType;
+  actionType?: string;
+  value?: string;
+  url?: string;
+  scrollDestinationDescription?: string;
 }
 
 export interface MidsceneRecorderTarget {
@@ -93,26 +129,142 @@ function isMidsceneRecorderPendingDescription(value?: string) {
   return value?.trim() === 'AI is analyzing element...';
 }
 
+export function getMidsceneRecorderSemantic(
+  event: Pick<MidsceneRecorderEvent, 'semantic'>,
+) {
+  return event.semantic;
+}
+
+function getRecorderPointerActionVerb(actionType?: string) {
+  switch (actionType) {
+    case 'Tap':
+      return 'Tap';
+    case 'DoubleClick':
+      return 'Double click';
+    case 'LongPress':
+      return 'Long press';
+    case 'RightClick':
+      return 'Right click';
+    default:
+      return 'Click';
+  }
+}
+
+function getRecorderDragActionVerb(actionType?: string) {
+  switch (actionType) {
+    case 'Swipe':
+      return 'Swipe';
+    case 'DragAndDrop':
+      return 'Drag';
+    default:
+      return 'Drag';
+  }
+}
+
+export function buildMidsceneRecorderReplayInstruction(
+  event: MidsceneRecorderSemanticAction,
+  elementDescription: string,
+) {
+  switch (event.type) {
+    case 'navigation':
+      if (event.actionType === 'Stop') {
+        return 'Stop loading the current page.';
+      }
+      if (event.actionType === 'GoBack') {
+        return 'Go back in the browser.';
+      }
+      if (event.actionType === 'GoForward') {
+        return 'Go forward in the browser.';
+      }
+      if (event.actionType === 'Reload') {
+        return 'Reload the current page.';
+      }
+      if (event.actionType === 'NavigationChanged' && event.url) {
+        return `Wait for navigation to complete at \`${event.url}\`.`;
+      }
+      return event.url
+        ? `Navigate to \`${event.url}\`.`
+        : `Navigate using ${elementDescription}.`;
+    case 'scroll':
+      return event.scrollDestinationDescription
+        ? `Scroll the page/region with description "${elementDescription}" by value "${event.value || 'down'}" until "${event.scrollDestinationDescription}" is visible.`
+        : `Scroll the page/region with description "${elementDescription}" by value "${event.value || 'down'}".`;
+    case 'drag': {
+      const verb = getRecorderDragActionVerb(event.actionType);
+      return `${verb} through the area described as "${elementDescription}".`;
+    }
+    case 'input':
+      return `Input "${event.value || ''}" into the element described as "${elementDescription}".`;
+    case 'keydown':
+      return `Press "${event.value || 'the recorded key'}" on the element described as "${elementDescription}".`;
+    default: {
+      const verb = getRecorderPointerActionVerb(event.actionType);
+      if (verb === 'Long press') {
+        return `${verb} the element described as "${elementDescription}".`;
+      }
+      return `${verb} on the element described as "${elementDescription}".`;
+    }
+  }
+}
+
+export function buildMidsceneRecorderActionSummary(
+  event: MidsceneRecorderSemanticAction,
+  elementDescription: string,
+) {
+  switch (event.type) {
+    case 'navigation':
+      if (event.actionType === 'Stop') {
+        return 'Stop page loading';
+      }
+      if (event.actionType === 'GoBack') {
+        return 'Go back';
+      }
+      if (event.actionType === 'GoForward') {
+        return 'Go forward';
+      }
+      if (event.actionType === 'Reload') {
+        return 'Reload page';
+      }
+      if (event.actionType === 'NavigationChanged' && event.url) {
+        return `Wait for navigation to complete at ${event.url}`;
+      }
+      return event.url ? `Navigate to ${event.url}` : 'Navigate';
+    case 'scroll':
+      return event.scrollDestinationDescription
+        ? `Scroll ${elementDescription} toward ${event.scrollDestinationDescription}`
+        : `Scroll ${elementDescription}`;
+    case 'drag':
+      return `${getRecorderDragActionVerb(event.actionType)} ${elementDescription}`;
+    case 'input':
+      return `Input into ${elementDescription}`;
+    case 'keydown':
+      return `Press ${event.value || 'key'} on ${elementDescription}`;
+    default:
+      return `${getRecorderPointerActionVerb(event.actionType)} ${elementDescription}`;
+  }
+}
+
 export function getMidsceneRecorderEventDescription(
   event: MidsceneRecorderEvent,
 ) {
+  const semantic = getMidsceneRecorderSemantic(event);
   if (
-    event.actionSummary &&
-    !isMidsceneRecorderPendingDescription(event.actionSummary)
+    semantic?.actionSummary &&
+    !isMidsceneRecorderPendingDescription(semantic.actionSummary)
   ) {
-    return event.actionSummary;
+    return semantic.actionSummary;
   }
   if (
-    event.elementDescription &&
-    !isMidsceneRecorderPendingDescription(event.elementDescription)
+    semantic?.elementDescription &&
+    !isMidsceneRecorderPendingDescription(semantic.elementDescription)
   ) {
-    return event.elementDescription;
+    return semantic.elementDescription;
   }
   if (
-    event.replayInstruction &&
-    !isMidsceneRecorderPendingDescription(event.replayInstruction)
+    semantic?.replayInstruction &&
+    !isMidsceneRecorderPendingDescription(semantic.replayInstruction)
   ) {
-    return event.replayInstruction;
+    return semantic.replayInstruction;
   }
   if (event.type === 'navigation' && event.url) {
     return `Navigate to ${event.url}`;
@@ -212,8 +364,9 @@ function getRecorderEventScreenshot(event: MidsceneRecorderEvent) {
 }
 
 function hasCoordinateFallback(event: MidsceneRecorderEvent) {
+  const semantic = getMidsceneRecorderSemantic(event);
   return (
-    !event.elementDescription &&
+    !semantic?.elementDescription &&
     event.elementRect?.x !== undefined &&
     event.elementRect?.y !== undefined
   );
@@ -224,6 +377,7 @@ function shouldIncludeMarkdownScreenshot(
   eventIndex: number,
   lastEventIndex: number,
 ) {
+  const semantic = getMidsceneRecorderSemantic(event);
   return (
     eventIndex === 0 ||
     eventIndex === lastEventIndex ||
@@ -231,7 +385,7 @@ function shouldIncludeMarkdownScreenshot(
     event.type === 'scroll' ||
     event.type === 'input' ||
     Boolean(event.screenshotWithBox) ||
-    !event.elementDescription ||
+    !semantic?.elementDescription ||
     hasCoordinateFallback(event)
   );
 }
@@ -262,17 +416,18 @@ function getRecorderScreenshotCandidatePriority(
   if (event.screenshotWithBox) {
     priority += 70;
   }
+  const semantic = getMidsceneRecorderSemantic(event);
   if (
-    event.descriptionSource === 'fallback' ||
-    event.semanticConfidence === 'low' ||
-    event.descriptionError
+    semantic?.source === 'heuristic' ||
+    semantic?.confidence === 'low' ||
+    semantic?.error
   ) {
     priority += 60;
   }
   if (event.type === 'input' || event.type === 'scroll') {
     priority += 40;
   }
-  if (!event.elementDescription || hasCoordinateFallback(event)) {
+  if (!semantic?.elementDescription || hasCoordinateFallback(event)) {
     priority += 30;
   }