@midscene/playground 1.9.8 → 1.10.0

package/dist/lib/server.js

@@ -37,9 +37,12 @@ var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
     createManualExecutorContext: ()=>createManualExecutorContext,
-    default: ()=>server,
+    buildPlaygroundBrowserUrl: ()=>buildPlaygroundBrowserUrl,
     PlaygroundServer: ()=>PlaygroundServer,
+    default: ()=>server,
+    resolvePlaygroundBrowserHost: ()=>resolvePlaygroundBrowserHost,
     serializeZodField: ()=>serializeZodField,
+    resolvePlaygroundListenHost: ()=>resolvePlaygroundListenHost,
     InteractParamsValidationError: ()=>InteractParamsValidationError,
     buildInteractParams: ()=>buildInteractParams
 });
@@ -48,6 +51,7 @@ const external_node_fs_namespaceObject = require("node:fs");
 const external_node_path_namespaceObject = require("node:path");
 const external_node_url_namespaceObject = require("node:url");
 const core_namespaceObject = require("@midscene/core");
+const ai_model_namespaceObject = require("@midscene/core/ai-model");
 const utils_namespaceObject = require("@midscene/core/utils");
 const common_namespaceObject = require("@midscene/shared/common");
 const constants_namespaceObject = require("@midscene/shared/constants");
@@ -78,6 +82,15 @@ const defaultPort = constants_namespaceObject.PLAYGROUND_SERVER_PORT;
 const RECORDER_CAPTURE_AFTER_INTERACT_DELAY_MS = 250;
 const RECORDER_AI_DESCRIBE_AFTER_INTERACT_TIMEOUT_MS = 30000;
 const RECORDER_AI_DESCRIBE_SCREENSHOT_DUMP_DIR = 'recorder-ai-describe-screenshots';
+function createElementDescriberRuntime(agent) {
+    const modelConfigManager = agent.modelConfigManager;
+    const missingModelRuntime = void 0;
+    return {
+        service: agent.service,
+        describeModelRuntime: modelConfigManager ? (0, ai_model_namespaceObject.getModelRuntime)(modelConfigManager.getModelConfig('insight')) : missingModelRuntime,
+        locateModelRuntime: modelConfigManager ? (0, ai_model_namespaceObject.getModelRuntime)(modelConfigManager.getModelConfig('default')) : missingModelRuntime
+    };
+}
 function extractBase64Payload(value) {
     const dataUrlMatch = value.match(/^data:([^;,]+)?(?:;[^,]*)?,([\s\S]*)$/);
     if (dataUrlMatch) return {
@@ -286,6 +299,17 @@ function createRecorderAiDescribeTraceBase(event, eventScreenshot) {
         screenshotBytes: estimateBase64Bytes(eventScreenshot)
     };
 }
+function resolvePlaygroundListenHost() {
+    return process.env.MIDSCENE_PLAYGROUND_HOST?.trim() || '0.0.0.0';
+}
+function resolvePlaygroundBrowserHost() {
+    const listenHost = resolvePlaygroundListenHost();
+    return '0.0.0.0' === listenHost || '::' === listenHost ? '127.0.0.1' : listenHost;
+}
+function buildPlaygroundBrowserUrl(host, port) {
+    const normalizedHost = host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;
+    return `http://${normalizedHost}:${port}`;
+}
 function serializeAiConfigSignature(aiConfig) {
     return JSON.stringify(Object.entries(aiConfig).sort(([leftKey], [rightKey])=>leftKey.localeCompare(rightKey)));
 }
@@ -1000,7 +1024,7 @@ class PlaygroundServer {
             let screenshotAnnotation;
             let screenshotPersistError;
             let annotatedScreenshotPersistError;
-            if ('failed' === status) try {
+            if ('failed' === status || extra.verifyResult?.pass === false) try {
                 const screenshotDump = await persistRecorderAiDescribeScreenshot(event, eventScreenshot, extra.verifyResult);
                 screenshotRef = screenshotDump?.screenshotRef;
                 annotatedScreenshotRef = screenshotDump?.annotatedScreenshotRef;
@@ -1036,7 +1060,7 @@ class PlaygroundServer {
                 trace
             };
         }
-        if ('function' != typeof agent?.describeElementAtPoint) {
+        if (!agent) {
             const error = 'Active agent does not support describeElementAtPoint.';
             const trace = await finishTrace('failed', {
                 error
@@ -1063,7 +1087,8 @@ class PlaygroundServer {
             if (!event.pageInfo?.width || !event.pageInfo?.height) throw new Error('Skipped aiDescribe because the recorder event has no pageInfo for coordinate mapping.');
             const verifyPrompt = shouldVerifyRecorderAiDescribeEvent(event);
             modelCallStartedAt = Date.now();
-            const describeResult = await withTimeout(agent.describeElementAtPoint([
+            const elementDescriber = createElementDescriberRuntime(agent);
+            const describeResult = await withTimeout((0, core_namespaceObject.describeElementAtPoint)(elementDescriber, [
                 x,
                 y
             ], {
@@ -1082,7 +1107,7 @@ class PlaygroundServer {
             deepLocate = describeResult.deepLocate;
             verifyResult = verifyPrompt ? describeResult.verifyResult : void 0;
             if (!elementDescription) throw new Error("aiDescribe returned an empty element description.");
-            if (verifyResult && false === verifyResult.pass) throw new Error('aiDescribe verification failed.');
+            if (false === describeResult.success) throw new Error(describeResult.error || `aiDescribe ${describeResult.failureStage || 'unknown'} failed.`);
             const semanticAction = buildRecorderSemanticAction(event.actionType || event.type, event.rawPayload || {}, event.url);
             const trace = await finishTrace('ready', {
                 modelCallDurationMs,
@@ -1105,7 +1130,8 @@ class PlaygroundServer {
                                 x,
                                 y
                             ],
-                            actualCenter: verifyResult?.center
+                            actualCenter: verifyResult?.center,
+                            annotatedScreenshotPath: trace.annotatedScreenshotRef?.path
                         }
                     })
                 },
@@ -2069,7 +2095,8 @@ class PlaygroundServer {
         this.port = port || defaultPort;
         return new Promise((resolve)=>{
             const serverPort = this.port ?? defaultPort;
-            this.server = this._app.listen(serverPort, '0.0.0.0', ()=>{
+            const listenHost = resolvePlaygroundListenHost();
+            this.server = this._app.listen(serverPort, listenHost, ()=>{
                 resolve(this);
             });
         });
@@ -2149,15 +2176,21 @@ const server = PlaygroundServer;
 exports.InteractParamsValidationError = __webpack_exports__.InteractParamsValidationError;
 exports.PlaygroundServer = __webpack_exports__.PlaygroundServer;
 exports.buildInteractParams = __webpack_exports__.buildInteractParams;
+exports.buildPlaygroundBrowserUrl = __webpack_exports__.buildPlaygroundBrowserUrl;
 exports.createManualExecutorContext = __webpack_exports__.createManualExecutorContext;
 exports["default"] = __webpack_exports__["default"];
+exports.resolvePlaygroundBrowserHost = __webpack_exports__.resolvePlaygroundBrowserHost;
+exports.resolvePlaygroundListenHost = __webpack_exports__.resolvePlaygroundListenHost;
 exports.serializeZodField = __webpack_exports__.serializeZodField;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
     "InteractParamsValidationError",
     "PlaygroundServer",
     "buildInteractParams",
+    "buildPlaygroundBrowserUrl",
     "createManualExecutorContext",
     "default",
+    "resolvePlaygroundBrowserHost",
+    "resolvePlaygroundListenHost",
     "serializeZodField"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {