@midscene/shared 1.10.5 → 1.10.6-beta-20260717061640.0

package/dist/lib/img/box-select.js

@@ -41,8 +41,11 @@ __webpack_require__.d(__webpack_exports__, {
 const external_node_assert_namespaceObject = require("node:assert");
 var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
 const index_js_namespaceObject = require("../constants/index.js");
+const external_utils_js_namespaceObject = require("../utils.js");
 const external_get_photon_js_namespaceObject = require("./get-photon.js");
 var external_get_photon_js_default = /*#__PURE__*/ __webpack_require__.n(external_get_photon_js_namespaceObject);
+const external_get_sharp_js_namespaceObject = require("./get-sharp.js");
+var external_get_sharp_js_default = /*#__PURE__*/ __webpack_require__.n(external_get_sharp_js_namespaceObject);
 const external_transform_js_namespaceObject = require("./transform.js");
 const DIGIT_FONT = {
     0: [
@@ -956,8 +959,45 @@ const createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding =
     }
     return overlayPixels;
 };
+async function decodeImageWithSharp(inputImgBase64, size) {
+    const { body } = (0, external_transform_js_namespaceObject.parseBase64)(inputImgBase64);
+    const Sharp = await external_get_sharp_js_default()();
+    let pipeline = Sharp(Buffer.from(body, 'base64'));
+    if (size) pipeline = pipeline.resize(size.width, size.height, {
+        fit: 'fill',
+        kernel: 'nearest'
+    });
+    const { data, info } = await pipeline.ensureAlpha().raw().toBuffer({
+        resolveWithObject: true
+    });
+    if (!info.width || !info.height || 4 !== info.channels) throw new Error('Image processing failed to produce RGBA pixels');
+    return {
+        pixels: new Uint8Array(data),
+        width: info.width,
+        height: info.height
+    };
+}
+async function encodeRgbaWithSharp(pixels, width, height) {
+    const Sharp = await external_get_sharp_js_default()();
+    const output = await Sharp(Buffer.from(pixels), {
+        raw: {
+            width,
+            height,
+            channels: 4
+        }
+    }).jpeg({
+        quality: 90,
+        chromaSubsampling: '4:4:4'
+    }).toBuffer();
+    return (0, external_transform_js_namespaceObject.createImgBase64ByFormat)('jpeg', output.toString('base64'));
+}
 const compositeElementInfoImg = async (options)=>{
     external_node_assert_default()(options.inputImgBase64, 'inputImgBase64 is required');
+    if (external_utils_js_namespaceObject.ifInNode) {
+        const { pixels, width, height } = await decodeImageWithSharp(options.inputImgBase64, options.size);
+        const overlayPixels = await createSvgOverlay(options.elementsPositionInfo, width, height, options.annotationPadding, options.borderThickness, options.prompt, options.centerPoint);
+        return encodeRgbaWithSharp(blendPixels(pixels, overlayPixels, width, height), width, height);
+    }
     const { PhotonImage, SamplingFilter, resize } = await external_get_photon_js_default()();
     let width = 0;
     let height = 0;
@@ -997,6 +1037,12 @@ const compositeElementInfoImg = async (options)=>{
 };
 const compositePointMarkerImg = async (options)=>{
     external_node_assert_default()(options.inputImgBase64, 'inputImgBase64 is required');
+    if (external_utils_js_namespaceObject.ifInNode) {
+        const { pixels, width, height } = await decodeImageWithSharp(options.inputImgBase64, options.size);
+        const overlayPixels = new Uint8Array(width * height * 4);
+        drawPointMarker(overlayPixels, width, height, options.point, options.radius ?? 14, options.indexId ?? 1);
+        return encodeRgbaWithSharp(blendPixels(pixels, overlayPixels, width, height), width, height);
+    }
     const { PhotonImage, SamplingFilter, resize } = await external_get_photon_js_default()();
     let width = 0;
     let height = 0;

package/dist/lib/img/get-photon.js

@@ -35,7 +35,7 @@ let isInitialized = false;
 let usingCanvasFallback = false;
 async function getPhoton() {
     if (photonModule && isInitialized) return photonModule;
-    const env = external_utils_js_namespaceObject.ifInBrowser ? 'browser' : external_utils_js_namespaceObject.ifInWorker ? 'worker' : external_utils_js_namespaceObject.ifInNode ? 'node' : 'unknown';
+    const env = external_utils_js_namespaceObject.ifInBrowser ? 'browser' : external_utils_js_namespaceObject.ifInWorker ? 'worker' : 'unknown';
     debug(`Loading photon module in ${env} environment`);
     try {
         if (external_utils_js_namespaceObject.ifInBrowser || external_utils_js_namespaceObject.ifInWorker) {
@@ -43,10 +43,7 @@ async function getPhoton() {
             if ('function' == typeof photon.default) await photon.default();
             debug('Photon loaded: @silvia-odwyer/photon (browser/worker)');
             photonModule = photon;
-        } else if (external_utils_js_namespaceObject.ifInNode) {
-            photonModule = await import("@silvia-odwyer/photon-node");
-            debug('Photon loaded: @silvia-odwyer/photon-node (node)');
-        }
+        } else throw new Error('Photon is only available in browser environments');
         if (!photonModule?.PhotonImage) throw new Error('PhotonImage is not available');
         if (!photonModule.PhotonImage.new_from_byteslice && !photonModule.PhotonImage.new_from_base64) throw new Error('PhotonImage.new_from_byteslice or new_from_base64 is not available');
         isInitialized = true;

package/dist/lib/img/info.js

@@ -42,15 +42,34 @@ __webpack_require__.d(__webpack_exports__, {
 const external_node_assert_namespaceObject = require("node:assert");
 var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
 const external_node_buffer_namespaceObject = require("node:buffer");
+const external_utils_js_namespaceObject = require("../utils.js");
 const external_get_photon_js_namespaceObject = require("./get-photon.js");
 var external_get_photon_js_default = /*#__PURE__*/ __webpack_require__.n(external_get_photon_js_namespaceObject);
+const external_get_sharp_js_namespaceObject = require("./get-sharp.js");
+var external_get_sharp_js_default = /*#__PURE__*/ __webpack_require__.n(external_get_sharp_js_namespaceObject);
 async function imageInfoOfBase64(imageBase64) {
-    const { PhotonImage } = await external_get_photon_js_default()();
     const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '').replace(/\s/g, '');
     external_node_assert_default()(base64Data, 'Invalid image: empty base64 data');
     external_node_assert_default()(/^[A-Za-z0-9+/]+={0,2}$/.test(base64Data) && base64Data.length % 4 !== 1, 'Invalid image: malformed base64 data');
     const imageBuffer = external_node_buffer_namespaceObject.Buffer.from(base64Data, 'base64');
     external_node_assert_default()(isValidImageBuffer(imageBuffer), 'Invalid image: unsupported format');
+    if (external_utils_js_namespaceObject.ifInNode) {
+        let metadata;
+        try {
+            const Sharp = await external_get_sharp_js_default()();
+            metadata = await Sharp(imageBuffer).metadata();
+        } catch (error) {
+            throw new Error(`Invalid image: failed to decode base64 data (${error instanceof Error ? error.message : String(error)})`, {
+                cause: error
+            });
+        }
+        external_node_assert_default()(metadata.width && metadata.height, 'Invalid image: cannot get width or height');
+        return {
+            width: metadata.width,
+            height: metadata.height
+        };
+    }
+    const { PhotonImage } = await external_get_photon_js_default()();
     let result;
     try {
         result = PhotonImage.new_from_base64(base64Data);

package/dist/lib/img/transform.js

@@ -79,7 +79,7 @@ async function resizeAndConvertImgBuffer(inputFormat, inputData, newSize) {
     external_node_assert_default()(newSize && newSize.width > 0 && newSize.height > 0, 'newSize must be positive');
     const resizeStartTime = Date.now();
     imgDebug(`resizeImg start, target size: ${newSize.width}x${newSize.height}`);
-    if (external_utils_js_namespaceObject.ifInNode) try {
+    if (external_utils_js_namespaceObject.ifInNode) {
         const Sharp = await external_get_sharp_js_default()();
         const metadata = await Sharp(inputData).metadata();
         const { width: originalWidth, height: originalHeight } = metadata;
@@ -97,8 +97,6 @@ async function resizeAndConvertImgBuffer(inputFormat, inputData, newSize) {
             buffer: resizedBuffer,
             format: 'jpeg'
         };
-    } catch (error) {
-        imgDebug('Sharp failed, falling back to Photon:', error);
     }
     const { PhotonImage, SamplingFilter, resize } = await external_get_photon_js_default()();
     const inputBytes = new Uint8Array(inputData);
@@ -246,6 +244,39 @@ async function paddingToMatchBlock(image, blockSize = 28) {
     };
 }
 async function paddingToMatchBlockByBase64(imageBase64, blockSize = 28) {
+    if (external_utils_js_namespaceObject.ifInNode) {
+        const { body } = parseBase64(imageBase64);
+        const inputBuffer = external_node_buffer_namespaceObject.Buffer.from(body, 'base64');
+        const Sharp = await external_get_sharp_js_default()();
+        const metadata = await Sharp(inputBuffer).metadata();
+        const width = metadata.width;
+        const height = metadata.height;
+        if (!width || !height) throw new Error('Failed to get image dimensions');
+        const targetWidth = Math.ceil(width / blockSize) * blockSize;
+        const targetHeight = Math.ceil(height / blockSize) * blockSize;
+        if (targetWidth === width && targetHeight === height) return {
+            width,
+            height,
+            imageBase64
+        };
+        const output = await Sharp(inputBuffer).extend({
+            right: targetWidth - width,
+            bottom: targetHeight - height,
+            background: {
+                r: 255,
+                g: 255,
+                b: 255,
+                alpha: 1
+            }
+        }).jpeg({
+            quality: 90
+        }).toBuffer();
+        return {
+            width: targetWidth,
+            height: targetHeight,
+            imageBase64: createImgBase64ByFormat('jpeg', output.toString('base64'))
+        };
+    }
     const photonImage = await photonFromBase64(imageBase64);
     try {
         const paddedResult = await paddingToMatchBlock(photonImage, blockSize);
@@ -261,6 +292,27 @@ async function paddingToMatchBlockByBase64(imageBase64, blockSize = 28) {
     }
 }
 async function cropByRect(imageBase64, rect) {
+    if (external_utils_js_namespaceObject.ifInNode) {
+        const { body } = parseBase64(imageBase64);
+        const Sharp = await external_get_sharp_js_default()();
+        const left = Math.trunc(rect.left);
+        const top = Math.trunc(rect.top);
+        const width = Math.trunc(rect.left + rect.width) - left;
+        const height = Math.trunc(rect.top + rect.height) - top;
+        const output = await Sharp(external_node_buffer_namespaceObject.Buffer.from(body, 'base64')).extract({
+            left,
+            top,
+            width,
+            height
+        }).jpeg({
+            quality: 90
+        }).toBuffer();
+        return {
+            width,
+            height,
+            imageBase64: createImgBase64ByFormat('jpeg', output.toString('base64'))
+        };
+    }
     const { crop } = await external_get_photon_js_default()();
     const photonImage = await photonFromBase64(imageBase64);
     const { left, top, width, height } = rect;
@@ -336,7 +388,7 @@ async function scaleImage(imageBase64, scale) {
     const buffer = external_node_buffer_namespaceObject.Buffer.from(body, 'base64');
     const scaleStartTime = Date.now();
     imgDebug(`scaleImage start, scale factor: ${scale}`);
-    if (external_utils_js_namespaceObject.ifInNode) try {
+    if (external_utils_js_namespaceObject.ifInNode) {
         const Sharp = await external_get_sharp_js_default()();
         const metadata = await Sharp(buffer).metadata();
         const originalWidth = metadata.width || 0;
@@ -358,8 +410,6 @@ async function scaleImage(imageBase64, scale) {
             height: newHeight,
             imageBase64: base64
         };
-    } catch (error) {
-        imgDebug('Sharp failed, falling back to Photon:', error);
     }
     const { PhotonImage, SamplingFilter, resize } = await external_get_photon_js_default()();
     const inputBytes = new Uint8Array(buffer);

package/dist/lib/logger.js

@@ -48,21 +48,32 @@ const external_common_js_namespaceObject = require("./common.js");
 const external_utils_js_namespaceObject = require("./utils.js");
 const topicPrefix = 'midscene';
 const logStreams = new Map();
+const backpressuredLogStreams = new Set();
+const unavailableLogStreams = new Set();
 const debugInstances = new Map();
 function getLogStream(topic) {
     const topicFileName = topic.replace(/:/g, '-');
+    if (unavailableLogStreams.has(topicFileName)) return null;
     if (!logStreams.has(topicFileName)) {
         const logFile = external_node_path_default().join((0, external_common_js_namespaceObject.getMidsceneRunSubDir)('log'), `${topicFileName}.log`);
         const stream = external_node_fs_default().createWriteStream(logFile, {
             flags: 'a'
         });
+        stream.on('error', ()=>{
+            unavailableLogStreams.add(topicFileName);
+            backpressuredLogStreams.delete(topicFileName);
+            if (logStreams.get(topicFileName) === stream) logStreams.delete(topicFileName);
+        });
         logStreams.set(topicFileName, stream);
     }
-    return logStreams.get(topicFileName);
+    return logStreams.get(topicFileName) ?? null;
 }
 function writeLogToFile(topic, message) {
     if (!external_utils_js_namespaceObject.ifInNode) return;
+    const topicFileName = topic.replace(/:/g, '-');
+    if (backpressuredLogStreams.has(topicFileName)) return;
     const stream = getLogStream(topic);
+    if (!stream) return;
     const now = new Date();
     const isoDate = now.toLocaleDateString('sv-SE');
     const isoTime = now.toLocaleTimeString('sv-SE');
@@ -73,7 +84,17 @@ function writeLogToFile(topic, message) {
     const minutes = (Math.abs(timezoneOffsetMinutes) % 60).toString().padStart(2, '0');
     const timezoneString = `${sign}${hours}:${minutes}`;
     const localISOTime = `${isoDate}T${isoTime}.${milliseconds}${timezoneString}`;
-    stream.write(`[${localISOTime}] ${message}\n`);
+    try {
+        if (!stream.write(`[${localISOTime}] ${message}\n`)) {
+            backpressuredLogStreams.add(topicFileName);
+            stream.once('drain', ()=>{
+                backpressuredLogStreams.delete(topicFileName);
+            });
+        }
+    } catch  {
+        unavailableLogStreams.add(topicFileName);
+        backpressuredLogStreams.delete(topicFileName);
+    }
 }
 function getDebug(topic, options) {
     const fullTopic = `${topicPrefix}:${topic}`;

package/dist/lib/zod-schema-utils.js

@@ -44,7 +44,6 @@ function isMidsceneLocatorField(field) {
     if (actualField._def?.typeName === 'ZodObject') {
         const shape = actualField._def.shape?.();
         if (shape) {
-            if ('midscene_location_field_flag' in shape) return true;
             if ('prompt' in shape && shape.prompt) return true;
         }
     }

package/dist/types/env/constants.d.ts

@@ -32,6 +32,7 @@ interface IModelConfigKeys {
     reasoningEffort: string;
     reasoningEnabled: string;
     reasoningBudget: string;
+    responseFormat: string;
 }
 export declare const INSIGHT_MODEL_CONFIG_KEYS: IModelConfigKeys;
 export declare const PLANNING_MODEL_CONFIG_KEYS: IModelConfigKeys;

package/dist/types/env/types.d.ts

@@ -24,7 +24,9 @@ export declare const MIDSCENE_MODEL_RETRY_INTERVAL = "MIDSCENE_MODEL_RETRY_INTER
 export declare const MIDSCENE_MODEL_REASONING_EFFORT = "MIDSCENE_MODEL_REASONING_EFFORT";
 export declare const MIDSCENE_MODEL_REASONING_ENABLED = "MIDSCENE_MODEL_REASONING_ENABLED";
 export declare const MIDSCENE_MODEL_REASONING_BUDGET = "MIDSCENE_MODEL_REASONING_BUDGET";
+export declare const MIDSCENE_MODEL_RESPONSE_FORMAT = "MIDSCENE_MODEL_RESPONSE_FORMAT";
 export type TModelReasoningEnabled = boolean | 'default';
+export type TModelResponseFormat = 'none' | 'auto';
 /**
  * @deprecated Use MIDSCENE_MODEL_API_KEY instead. This is kept for backward compatibility.
  */
@@ -81,6 +83,7 @@ export declare const MIDSCENE_INSIGHT_MODEL_FAMILY = "MIDSCENE_INSIGHT_MODEL_FAM
 export declare const MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT = "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT";
 export declare const MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED = "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED";
 export declare const MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET = "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET";
+export declare const MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT = "MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT";
 export declare const MIDSCENE_PLANNING_MODEL_NAME = "MIDSCENE_PLANNING_MODEL_NAME";
 export declare const MIDSCENE_PLANNING_MODEL_SOCKS_PROXY = "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY";
 export declare const MIDSCENE_PLANNING_MODEL_HTTP_PROXY = "MIDSCENE_PLANNING_MODEL_HTTP_PROXY";
@@ -96,6 +99,7 @@ export declare const MIDSCENE_PLANNING_MODEL_FAMILY = "MIDSCENE_PLANNING_MODEL_F
 export declare const MIDSCENE_PLANNING_MODEL_REASONING_EFFORT = "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT";
 export declare const MIDSCENE_PLANNING_MODEL_REASONING_ENABLED = "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED";
 export declare const MIDSCENE_PLANNING_MODEL_REASONING_BUDGET = "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET";
+export declare const MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT = "MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT";
 export declare const MIDSCENE_MODEL_FAMILY = "MIDSCENE_MODEL_FAMILY";
 /**
  * env keys declared but unused
@@ -120,8 +124,8 @@ export declare const GLOBAL_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_LANG
  * Can be override by both agent.modelConfig and overrideAIConfig
  * Can only be access after agent.constructor
  */
-export declare const MODEL_ENV_KEYS: readonly ["MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_FAMILY"];
-export declare const ALL_ENV_KEYS: readonly [...string[], "MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR", "MIDSCENE_CACHE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER", "MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_FAMILY"];
+export declare const MODEL_ENV_KEYS: readonly ["MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_RESPONSE_FORMAT", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT", "MIDSCENE_MODEL_FAMILY"];
+export declare const ALL_ENV_KEYS: readonly [...string[], "MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_MODEL_PROFILE", "MIDSCENE_DEBUG_MODEL_RESPONSE", "MIDSCENE_RUN_DIR", "MIDSCENE_CACHE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_LANGFUSE_DEBUG", "MIDSCENE_REPORT_QUIET", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_CHROME_PATH", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER", "MIDSCENE_MODEL_NAME", "MIDSCENE_MODEL_INIT_CONFIG_JSON", "MIDSCENE_MODEL_EXTRA_BODY_JSON", "MIDSCENE_MODEL_API_KEY", "MIDSCENE_MODEL_BASE_URL", "MIDSCENE_MODEL_SOCKS_PROXY", "MIDSCENE_MODEL_HTTP_PROXY", "MIDSCENE_MODEL_TIMEOUT", "MIDSCENE_MODEL_TEMPERATURE", "MIDSCENE_MODEL_RETRY_COUNT", "MIDSCENE_MODEL_RETRY_INTERVAL", "MIDSCENE_MODEL_REASONING_EFFORT", "MIDSCENE_MODEL_REASONING_ENABLED", "MIDSCENE_MODEL_REASONING_BUDGET", "MIDSCENE_MODEL_RESPONSE_FORMAT", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "OPENAI_API_KEY", "OPENAI_BASE_URL", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_NAME", "MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY", "MIDSCENE_INSIGHT_MODEL_HTTP_PROXY", "MIDSCENE_INSIGHT_MODEL_BASE_URL", "MIDSCENE_INSIGHT_MODEL_API_KEY", "MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON", "MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON", "MIDSCENE_INSIGHT_MODEL_TIMEOUT", "MIDSCENE_INSIGHT_MODEL_TEMPERATURE", "MIDSCENE_INSIGHT_MODEL_RETRY_COUNT", "MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL", "MIDSCENE_INSIGHT_MODEL_FAMILY", "MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT", "MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED", "MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET", "MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_MODEL_SOCKS_PROXY", "MIDSCENE_PLANNING_MODEL_HTTP_PROXY", "MIDSCENE_PLANNING_MODEL_BASE_URL", "MIDSCENE_PLANNING_MODEL_API_KEY", "MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON", "MIDSCENE_PLANNING_MODEL_TIMEOUT", "MIDSCENE_PLANNING_MODEL_TEMPERATURE", "MIDSCENE_PLANNING_MODEL_RETRY_COUNT", "MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL", "MIDSCENE_PLANNING_MODEL_FAMILY", "MIDSCENE_PLANNING_MODEL_REASONING_EFFORT", "MIDSCENE_PLANNING_MODEL_REASONING_ENABLED", "MIDSCENE_PLANNING_MODEL_REASONING_BUDGET", "MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT", "MIDSCENE_MODEL_FAMILY"];
 export type TEnvKeys = (typeof ALL_ENV_KEYS)[number];
 export type TGlobalConfig = Record<TEnvKeys, string | undefined>;
 /**
@@ -174,6 +178,7 @@ export interface IModelConfigForDefault {
     [MIDSCENE_MODEL_REASONING_EFFORT]?: string;
     [MIDSCENE_MODEL_REASONING_ENABLED]?: string;
     [MIDSCENE_MODEL_REASONING_BUDGET]?: string;
+    [MIDSCENE_MODEL_RESPONSE_FORMAT]?: TModelResponseFormat;
 }
 export interface IModelConfigForDefaultLegacy {
     [MIDSCENE_MODEL_NAME]: string;
@@ -262,6 +267,8 @@ export interface IModelConfig {
     /**
      * Number of retries when AI call fails.
      * Default is 1 (retry once after failure).
+     * Retries occur on HTTP errors or when the model response cannot be
+     * structurally parsed.
      */
     retryCount?: number;
     /**
@@ -285,6 +292,11 @@ export interface IModelConfig {
      * Passed through to model-family-specific parameters (e.g., thinking_budget for qwen).
      */
     reasoningBudget?: number;
+    /**
+     * Response format strategy. "auto" lets the model adapter enable a
+     * provider-supported structured response format for eligible intents.
+     */
+    responseFormat?: TModelResponseFormat;
     /**
      * Model family - unified model configuration
      * Maps directly to model families like 'qwen2.5-vl', 'qwen3-vl', 'doubao-vision', 'doubao-seed', etc.

package/dist/types/img/get-photon.d.ts

@@ -1,17 +1,17 @@
 export default function getPhoton(): Promise<{
-    PhotonImage: typeof import('@silvia-odwyer/photon-node').PhotonImage;
-    SamplingFilter: typeof import('@silvia-odwyer/photon-node').SamplingFilter;
-    resize: typeof import('@silvia-odwyer/photon-node').resize;
-    crop: typeof import('@silvia-odwyer/photon-node').crop;
-    open_image: typeof import('@silvia-odwyer/photon-node').open_image;
-    base64_to_image: typeof import('@silvia-odwyer/photon-node').base64_to_image;
-    padding_uniform: typeof import('@silvia-odwyer/photon-node').padding_uniform;
-    padding_left: typeof import('@silvia-odwyer/photon-node').padding_left;
-    padding_right: typeof import('@silvia-odwyer/photon-node').padding_right;
-    padding_top: typeof import('@silvia-odwyer/photon-node').padding_top;
-    padding_bottom: typeof import('@silvia-odwyer/photon-node').padding_bottom;
-    watermark: typeof import('@silvia-odwyer/photon-node').watermark;
-    Rgba: typeof import('@silvia-odwyer/photon-node').Rgba;
+    PhotonImage: typeof import('@silvia-odwyer/photon').PhotonImage;
+    SamplingFilter: typeof import('@silvia-odwyer/photon').SamplingFilter;
+    resize: typeof import('@silvia-odwyer/photon').resize;
+    crop: typeof import('@silvia-odwyer/photon').crop;
+    open_image: typeof import('@silvia-odwyer/photon').open_image;
+    base64_to_image: typeof import('@silvia-odwyer/photon').base64_to_image;
+    padding_uniform: typeof import('@silvia-odwyer/photon').padding_uniform;
+    padding_left: typeof import('@silvia-odwyer/photon').padding_left;
+    padding_right: typeof import('@silvia-odwyer/photon').padding_right;
+    padding_top: typeof import('@silvia-odwyer/photon').padding_top;
+    padding_bottom: typeof import('@silvia-odwyer/photon').padding_bottom;
+    watermark: typeof import('@silvia-odwyer/photon').watermark;
+    Rgba: typeof import('@silvia-odwyer/photon').Rgba;
 }>;
 /**
  * Check if we're using the Canvas fallback instead of Photon

package/dist/types/img/transform.d.ts

@@ -1,5 +1,5 @@
 import { Buffer } from 'node:buffer';
-import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon-node';
+import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon';
 import type { Rect } from '../types';
 /**
  * Saves a Base64-encoded image to a file

package/dist/types/recorder.d.ts

@@ -13,6 +13,18 @@ export interface MidsceneRecorderPageInfo {
     width: number;
     height: number;
 }
+/**
+ * A screenshot stored outside the recording event payload.
+ *
+ * Recorder events are persisted in the Studio renderer. Keeping full data
+ * URLs there makes long recordings retain every screenshot in the renderer
+ * heap, so screenshot bytes live in the Playground run directory instead.
+ */
+export interface MidsceneRecorderScreenshotAssetRef {
+    id: string;
+    mimeType: string;
+    bytes: number;
+}
 export interface MidsceneRecorderEvent {
     type: MidsceneRecorderEventType;
     source?: MidsceneRecorderSourceKind;
@@ -25,6 +37,8 @@ export interface MidsceneRecorderEvent {
     pageInfo: MidsceneRecorderPageInfo;
     screenshotBefore?: string;
     screenshotAfter?: string;
+    /** The single screenshot retained for AI description and Markdown export. */
+    screenshotAsset?: MidsceneRecorderScreenshotAssetRef;
     semantic?: MidsceneRecorderSemantic;
     elementDescription?: string;
     descriptionLoading?: boolean;

package/dist/types/zod-schema-utils.d.ts

@@ -6,9 +6,7 @@ import type { z } from 'zod';
 export declare function unwrapZodField(field: unknown): unknown;
 /**
  * Check if a field is a Midscene locator field
- * Checks for either:
- * 1. midscene_location_field_flag in shape (result schema)
- * 2. prompt field in shape (input schema)
+ * Locator input schemas are identified by their prompt field.
  */
 export declare function isMidsceneLocatorField(field: unknown): boolean;
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/shared",
-  "version": "1.10.5",
+  "version": "1.10.6-beta-20260717061640.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",
@@ -95,7 +95,6 @@
   ],
   "dependencies": {
     "@silvia-odwyer/photon": "0.3.3",
-    "@silvia-odwyer/photon-node": "0.3.3",
     "debug": "4.4.0",
     "js-sha256": "0.11.0",
     "sharp": "^0.34.3",

package/src/env/constants.ts

@@ -9,6 +9,7 @@ import {
   MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET,
   MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT,
   MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED,
+  MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT,
   MIDSCENE_INSIGHT_MODEL_RETRY_COUNT,
   MIDSCENE_INSIGHT_MODEL_RETRY_INTERVAL,
   MIDSCENE_INSIGHT_MODEL_SOCKS_PROXY,
@@ -24,6 +25,7 @@ import {
   MIDSCENE_MODEL_REASONING_BUDGET,
   MIDSCENE_MODEL_REASONING_EFFORT,
   MIDSCENE_MODEL_REASONING_ENABLED,
+  MIDSCENE_MODEL_RESPONSE_FORMAT,
   MIDSCENE_MODEL_RETRY_COUNT,
   MIDSCENE_MODEL_RETRY_INTERVAL,
   MIDSCENE_MODEL_SOCKS_PROXY,
@@ -42,6 +44,7 @@ import {
   MIDSCENE_PLANNING_MODEL_REASONING_BUDGET,
   MIDSCENE_PLANNING_MODEL_REASONING_EFFORT,
   MIDSCENE_PLANNING_MODEL_REASONING_ENABLED,
+  MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT,
   MIDSCENE_PLANNING_MODEL_RETRY_COUNT,
   MIDSCENE_PLANNING_MODEL_RETRY_INTERVAL,
   MIDSCENE_PLANNING_MODEL_SOCKS_PROXY,
@@ -85,6 +88,7 @@ interface IModelConfigKeys {
   reasoningEffort: string;
   reasoningEnabled: string;
   reasoningBudget: string;
+  responseFormat: string;
 }
 
 export const INSIGHT_MODEL_CONFIG_KEYS: IModelConfigKeys = {
@@ -121,6 +125,7 @@ export const INSIGHT_MODEL_CONFIG_KEYS: IModelConfigKeys = {
   reasoningEffort: MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT,
   reasoningEnabled: MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED,
   reasoningBudget: MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET,
+  responseFormat: MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT,
 } as const;
 
 export const PLANNING_MODEL_CONFIG_KEYS: IModelConfigKeys = {
@@ -157,6 +162,7 @@ export const PLANNING_MODEL_CONFIG_KEYS: IModelConfigKeys = {
   reasoningEffort: MIDSCENE_PLANNING_MODEL_REASONING_EFFORT,
   reasoningEnabled: MIDSCENE_PLANNING_MODEL_REASONING_ENABLED,
   reasoningBudget: MIDSCENE_PLANNING_MODEL_REASONING_BUDGET,
+  responseFormat: MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT,
 } as const;
 
 // modelConfig return default
@@ -194,6 +200,7 @@ export const DEFAULT_MODEL_CONFIG_KEYS: IModelConfigKeys = {
   reasoningEffort: MIDSCENE_MODEL_REASONING_EFFORT,
   reasoningEnabled: MIDSCENE_MODEL_REASONING_ENABLED,
   reasoningBudget: MIDSCENE_MODEL_REASONING_BUDGET,
+  responseFormat: MIDSCENE_MODEL_RESPONSE_FORMAT,
 } as const;
 
 // read from process.env
@@ -231,4 +238,5 @@ export const DEFAULT_MODEL_CONFIG_KEYS_LEGACY: IModelConfigKeys = {
   reasoningEffort: MIDSCENE_MODEL_REASONING_EFFORT,
   reasoningEnabled: MIDSCENE_MODEL_REASONING_ENABLED,
   reasoningBudget: MIDSCENE_MODEL_REASONING_BUDGET,
+  responseFormat: MIDSCENE_MODEL_RESPONSE_FORMAT,
 } as const;

package/src/env/parse-model-config.ts

@@ -288,6 +288,14 @@ export const parseOpenaiSdkConfig = ({
       const num = Number(val);
       return Number.isFinite(num) ? num : undefined;
     })(),
+    responseFormat: (() => {
+      const val = provider[keys.responseFormat]?.trim()?.toLowerCase();
+      if (!val || val === 'auto') return 'auto';
+      if (val === 'none') return 'none';
+      throw new Error(
+        `${keys.responseFormat} must be one of: none, auto. Got: ${val}`,
+      );
+    })(),
   };
 };
 

package/src/env/types.ts

@@ -35,8 +35,10 @@ export const MIDSCENE_MODEL_REASONING_ENABLED =
   'MIDSCENE_MODEL_REASONING_ENABLED';
 export const MIDSCENE_MODEL_REASONING_BUDGET =
   'MIDSCENE_MODEL_REASONING_BUDGET';
+export const MIDSCENE_MODEL_RESPONSE_FORMAT = 'MIDSCENE_MODEL_RESPONSE_FORMAT';
 
 export type TModelReasoningEnabled = boolean | 'default';
+export type TModelResponseFormat = 'none' | 'auto';
 
 /**
  * @deprecated Use MIDSCENE_MODEL_API_KEY instead. This is kept for backward compatibility.
@@ -117,6 +119,8 @@ export const MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED =
   'MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED';
 export const MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET =
   'MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET';
+export const MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT =
+  'MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT';
 
 // PLANNING
 export const MIDSCENE_PLANNING_MODEL_NAME = 'MIDSCENE_PLANNING_MODEL_NAME';
@@ -147,6 +151,8 @@ export const MIDSCENE_PLANNING_MODEL_REASONING_ENABLED =
   'MIDSCENE_PLANNING_MODEL_REASONING_ENABLED';
 export const MIDSCENE_PLANNING_MODEL_REASONING_BUDGET =
   'MIDSCENE_PLANNING_MODEL_REASONING_BUDGET';
+export const MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT =
+  'MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT';
 export const MIDSCENE_MODEL_FAMILY = 'MIDSCENE_MODEL_FAMILY';
 
 /**
@@ -224,6 +230,7 @@ export const MODEL_ENV_KEYS = [
   MIDSCENE_MODEL_REASONING_EFFORT,
   MIDSCENE_MODEL_REASONING_ENABLED,
   MIDSCENE_MODEL_REASONING_BUDGET,
+  MIDSCENE_MODEL_RESPONSE_FORMAT,
   MIDSCENE_USE_VLM_UI_TARS,
   MIDSCENE_USE_QWEN_VL,
   MIDSCENE_USE_QWEN3_VL,
@@ -252,6 +259,7 @@ export const MODEL_ENV_KEYS = [
   MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT,
   MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED,
   MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET,
+  MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT,
   // PLANNING
   MIDSCENE_PLANNING_MODEL_NAME,
   MIDSCENE_PLANNING_MODEL_SOCKS_PROXY,
@@ -268,6 +276,7 @@ export const MODEL_ENV_KEYS = [
   MIDSCENE_PLANNING_MODEL_REASONING_EFFORT,
   MIDSCENE_PLANNING_MODEL_REASONING_ENABLED,
   MIDSCENE_PLANNING_MODEL_REASONING_BUDGET,
+  MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT,
   MIDSCENE_MODEL_FAMILY,
 ] as const;
 
@@ -390,6 +399,8 @@ export interface IModelConfigForDefault {
   [MIDSCENE_MODEL_REASONING_ENABLED]?: string;
   // reasoning budget (number as string)
   [MIDSCENE_MODEL_REASONING_BUDGET]?: string;
+  // Response format strategy (none/auto)
+  [MIDSCENE_MODEL_RESPONSE_FORMAT]?: TModelResponseFormat;
 }
 
 export interface IModelConfigForDefaultLegacy {
@@ -490,6 +501,8 @@ export interface IModelConfig {
   /**
    * Number of retries when AI call fails.
    * Default is 1 (retry once after failure).
+   * Retries occur on HTTP errors or when the model response cannot be
+   * structurally parsed.
    */
   retryCount?: number;
   /**
@@ -513,6 +526,11 @@ export interface IModelConfig {
    * Passed through to model-family-specific parameters (e.g., thinking_budget for qwen).
    */
   reasoningBudget?: number;
+  /**
+   * Response format strategy. "auto" lets the model adapter enable a
+   * provider-supported structured response format for eligible intents.
+   */
+  responseFormat?: TModelResponseFormat;
   /**
    * Model family - unified model configuration
    * Maps directly to model families like 'qwen2.5-vl', 'qwen3-vl', 'doubao-vision', 'doubao-seed', etc.