@midscene/playground 1.8.0 → 1.8.1

package/dist/es/server.mjs

@@ -1,5 +1,4 @@
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import node_http from "node:http";
 import { dirname, join, resolve as external_node_path_resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { ReportActionDump, runConnectivityTest } from "@midscene/core";
@@ -11,7 +10,8 @@ import { getDebug } from "@midscene/shared/logger";
 import { uuid as utils_uuid } from "@midscene/shared/utils";
 import express from "express";
 import { executeAction, formatErrorMessage } from "./common.mjs";
-import { createInterfaceMjpegHub, writeMjpegFrame } from "./mjpeg-hub.mjs";
+import { MjpegStreamHandler } from "./mjpeg-stream-handler.mjs";
+import { PointerInputError, dispatchPointer } from "./pointer-dispatch.mjs";
 import { buildRuntimeInfo } from "./runtime-metadata.mjs";
 import "dotenv/config";
 function _define_property(obj, key, value) {
@@ -81,6 +81,19 @@ function locateFromPoint(x, y, fieldX, fieldY, description) {
         Math.round(requireNumber(y, fieldY))
     ], description);
 }
+const POINTER_INTERACT_ACTIONS = new Set([
+    'Tap',
+    'DoubleClick',
+    'LongPress',
+    'Swipe',
+    'DragAndDrop',
+    'KeyboardPress',
+    'Input',
+    'Pinch'
+]);
+function isPointerInteractActionType(actionType) {
+    return POINTER_INTERACT_ACTIONS.has(actionType);
+}
 const buildLocateActionParams = (body, actionType)=>{
     const params = {
         locate: locateFromPoint(body.x, body.y, 'x', 'y', `manual ${actionType}`)
@@ -171,6 +184,10 @@ function isRecoverablePageSessionError(error) {
     return RECOVERABLE_PAGE_SESSION_ERROR_PATTERN.test(message);
 }
 class PlaygroundServer {
+    setActiveAgent(agent) {
+        this._activeConnection.agent = agent;
+        this._mjpegHandler.reset();
+    }
     get agent() {
         return this._activeConnection.agent;
     }
@@ -223,6 +240,7 @@ class PlaygroundServer {
             executionHooks: this._baseExecutionHooks,
             sidecars: this._baseSidecars
         };
+        this._mjpegHandler.reset();
         this.syncRuntimeState();
     }
     setPreparedPlatform(prepared) {
@@ -318,7 +336,7 @@ class PlaygroundServer {
         } catch (error) {
             console.warn('Failed to destroy old agent:', error);
         } finally{
-            this._activeConnection.agent = null;
+            this.setActiveAgent(null);
             this._configDirty = false;
         }
     }
@@ -357,6 +375,7 @@ class PlaygroundServer {
                 executionHooks: session.executionHooks || this._baseExecutionHooks,
                 sidecars: sessionSidecars
             };
+            this._mjpegHandler.reset();
             this.sessionSetupState = 'ready';
             this.sessionSetupBlockingReason = void 0;
             this.syncRuntimeState();
@@ -414,7 +433,7 @@ class PlaygroundServer {
         console.log('Recreating agent to cancel current task...');
         await this.destroyCurrentAgent();
         if (this._activeConnection.agentFactory) try {
-            this._activeConnection.agent = await this._activeConnection.agentFactory();
+            this.setActiveAgent(await this._activeConnection.agentFactory());
             this._agentReady = true;
             console.log('Agent recreated successfully');
         } catch (error) {
@@ -431,7 +450,7 @@ class PlaygroundServer {
         if (!this._activeConnection.agentFactory || !isRecoverablePageSessionError(error)) return null;
         debugMjpeg(`Recovering active agent after ${reason}:`, error);
         try {
-            this._interfaceMjpegHub.stopProducer();
+            this._mjpegHandler.reset();
             await this.recreateAgent();
             return this._activeConnection.agent;
         } catch (recreateError) {
@@ -505,7 +524,7 @@ class PlaygroundServer {
                 await this.destroyCurrentSession();
                 const created = await this.sessionManager.createSession(req.body || {});
                 await this.applyCreatedSession(created);
-                if (!this._activeConnection.agent && this._activeConnection.agentFactory) this._activeConnection.agent = await this._activeConnection.agentFactory();
+                if (!this._activeConnection.agent && this._activeConnection.agentFactory) this.setActiveAgent(await this._activeConnection.agentFactory());
                 if (this._configDirty && this._activeConnection.agentFactory) {
                     this._configDirty = false;
                     await this.recreateAgent();
@@ -637,7 +656,7 @@ class PlaygroundServer {
                 console.log('AI config changed, recreating agent...');
                 try {
                     await this.destroyCurrentAgent();
-                    this._activeConnection.agent = await this._activeConnection.agentFactory();
+                    this.setActiveAgent(await this._activeConnection.agentFactory());
                     agent = this.getActiveAgentOrThrow();
                     this._agentReady = true;
                     console.log('Agent recreated with new config');
@@ -807,19 +826,7 @@ class PlaygroundServer {
             if (!agent) return res.status(409).json({
                 error: 'No active session'
             });
-            const nativeUrl = agent.interface?.mjpegStreamUrl;
-            const recentlyFailed = false === this._nativeMjpegAvailable && null !== this._nativeMjpegFailedAt && Date.now() - this._nativeMjpegFailedAt < PlaygroundServer.MJPEG_NEGATIVE_CACHE_MS;
-            if (nativeUrl && !recentlyFailed) {
-                const proxyOk = await this.probeAndProxyNativeMjpeg(nativeUrl, req, res);
-                if (proxyOk) return;
-            }
-            const interfaceStreamStarted = await this._interfaceMjpegHub.streamRequest(req, res, agent.interface, async (startupError)=>(await this.recoverActiveAgentAfterPreviewError(startupError, 'interface MJPEG startup'))?.interface ?? null);
-            if (interfaceStreamStarted) return;
-            const fallbackAgent = this._activeConnection.agent;
-            if ('function' != typeof fallbackAgent?.interface?.screenshotBase64) return res.status(500).json({
-                error: 'Screenshot method not available on current interface'
-            });
-            await this.startPollingMjpegStream(req, res);
+            await this._mjpegHandler.serve(req, res);
         });
         this._app.get('/interface-info', async (_req, res)=>{
             try {
@@ -878,26 +885,26 @@ class PlaygroundServer {
             if ('string' != typeof actionType || !actionType) return res.status(400).json({
                 error: 'actionType is required'
             });
-            if (!this.findInteractAction(agent, actionType) && !this.canRunBrowserChromeInteractAction(agent, actionType)) return res.status(404).json({
-                error: `Action "${actionType}" is not available on the current device`
-            });
-            let params;
             try {
-                params = buildInteractParams(actionType, req.body ?? {});
-            } catch (error) {
-                if (error instanceof InteractParamsValidationError) return res.status(400).json({
-                    error: error.message
-                });
-                const errorMessage = error instanceof Error ? error.message : 'Unknown error';
-                console.error(`Failed to build interact params for "${actionType}": ${errorMessage}`);
-                return res.status(500).json({
-                    error: errorMessage
+                const inputPrimitives = agent.interface.inputPrimitives;
+                if (inputPrimitives) {
+                    await dispatchPointer(inputPrimitives, req.body ?? {}, ()=>agent.interface.size());
+                    res.json({});
+                    return;
+                }
+                if (!this.findInteractAction(agent, actionType) && !this.canRunBrowserChromeInteractAction(agent, actionType)) return res.status(404).json({
+                    error: isPointerInteractActionType(actionType) ? 'Manual control is not supported on this device' : `Action "${actionType}" is not available on the current device`
                 });
-            }
-            try {
+                const params = buildInteractParams(actionType, req.body ?? {});
                 await this.runInteractAction(agent, actionType, params);
                 res.json({});
             } catch (error) {
+                if (error instanceof PointerInputError) return res.status(error.statusCode).json({
+                    error: error.message
+                });
+                if (error instanceof InteractParamsValidationError) return res.status(400).json({
+                    error: error.message
+                });
                 const recoveredAgent = await this.recoverActiveAgentAfterPreviewError(error, `manual interact action "${actionType}"`);
                 if (recoveredAgent) return res.status(409).json({
                     error: 'The page session was closed and has been recreated. Please retry the action.'
@@ -979,123 +986,6 @@ class PlaygroundServer {
             }
         });
     }
-    probeAndProxyNativeMjpeg(nativeUrl, req, res) {
-        return new Promise((resolve)=>{
-            console.log(`MJPEG: trying native stream from ${nativeUrl}`);
-            const proxyReq = node_http.get(nativeUrl, (proxyRes)=>{
-                const statusCode = proxyRes.statusCode ?? 0;
-                if (statusCode >= 400) {
-                    this._nativeMjpegAvailable = false;
-                    this._nativeMjpegFailedAt = Date.now();
-                    proxyRes.resume();
-                    debugMjpeg(`native stream returned HTTP ${statusCode}, using polling mode`);
-                    resolve(false);
-                    return;
-                }
-                this._nativeMjpegAvailable = true;
-                this._nativeMjpegFailedAt = null;
-                console.log('MJPEG: streaming via native WDA MJPEG server');
-                const contentType = proxyRes.headers['content-type'];
-                if (contentType) res.setHeader('Content-Type', contentType);
-                res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate');
-                res.setHeader('Connection', 'keep-alive');
-                proxyRes.pipe(res);
-                req.on('close', ()=>proxyReq.destroy());
-                resolve(true);
-            });
-            proxyReq.on('error', (err)=>{
-                this._nativeMjpegAvailable = false;
-                this._nativeMjpegFailedAt = Date.now();
-                debugMjpeg(`MJPEG: native stream unavailable (${err.message}), using polling mode`);
-                resolve(false);
-            });
-        });
-    }
-    probeNativeMjpegLiveness(nativeUrl) {
-        return new Promise((resolve)=>{
-            const probe = node_http.get(nativeUrl, (probeRes)=>{
-                const statusCode = probeRes.statusCode ?? 0;
-                const reachable = statusCode >= 200 && statusCode < 400;
-                probeRes.destroy();
-                resolve(reachable);
-            });
-            probe.setTimeout(1000, ()=>{
-                probe.destroy();
-                resolve(false);
-            });
-            probe.on('error', ()=>resolve(false));
-        });
-    }
-    async startPollingMjpegStream(req, res) {
-        const defaultMjpegFps = 10;
-        const maxMjpegFps = 30;
-        const maxErrorBackoffMs = 3000;
-        const errorLogThreshold = 3;
-        const nativeProbeIntervalMs = 3000;
-        const parsedFps = Number(req.query.fps);
-        const fps = Math.min(Math.max(Number.isNaN(parsedFps) ? defaultMjpegFps : parsedFps, 1), maxMjpegFps);
-        const interval = Math.round(1000 / fps);
-        const boundary = 'mjpeg-boundary';
-        console.log(`MJPEG: streaming via polling mode (${fps}fps)`);
-        res.setHeader('Content-Type', `multipart/x-mixed-replace; boundary=${boundary}`);
-        res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate');
-        res.setHeader('Connection', 'keep-alive');
-        let stopped = false;
-        let consecutiveErrors = 0;
-        const nativeUrl = this._activeConnection.agent?.interface?.mjpegStreamUrl;
-        let probeTimer;
-        if (nativeUrl) probeTimer = setInterval(async ()=>{
-            if (stopped) return;
-            const reachable = await this.probeNativeMjpegLiveness(nativeUrl);
-            if (reachable && !stopped) {
-                console.log('MJPEG: native stream came online, ending polling so client reconnects');
-                this._nativeMjpegAvailable = true;
-                this._nativeMjpegFailedAt = null;
-                stopped = true;
-                try {
-                    res.destroy();
-                } catch  {}
-            }
-        }, nativeProbeIntervalMs);
-        req.on('close', ()=>{
-            stopped = true;
-            if (probeTimer) clearInterval(probeTimer);
-        });
-        while(!stopped){
-            if (!this._agentReady) {
-                await new Promise((r)=>setTimeout(r, 200));
-                continue;
-            }
-            const frameStart = Date.now();
-            try {
-                const agent = this.getActiveAgentOrThrow();
-                const base64 = await agent.interface.screenshotBase64();
-                if (stopped) break;
-                consecutiveErrors = 0;
-                writeMjpegFrame(res, boundary, {
-                    data: base64,
-                    contentType: 'image/jpeg'
-                });
-            } catch (err) {
-                if (stopped) break;
-                const recoveredAgent = await this.recoverActiveAgentAfterPreviewError(err, 'polling MJPEG frame capture');
-                if (recoveredAgent) {
-                    consecutiveErrors = 0;
-                    continue;
-                }
-                consecutiveErrors++;
-                if (consecutiveErrors <= errorLogThreshold) console.error('MJPEG frame error:', err);
-                else if (consecutiveErrors === errorLogThreshold + 1) console.error('MJPEG: suppressing further errors, retrying silently...');
-                const backoff = Math.min(1000 * consecutiveErrors, maxErrorBackoffMs);
-                await new Promise((r)=>setTimeout(r, backoff));
-                continue;
-            }
-            const elapsed = Date.now() - frameStart;
-            const remaining = interval - elapsed;
-            if (remaining > 0) await new Promise((r)=>setTimeout(r, remaining));
-        }
-        if (probeTimer) clearInterval(probeTimer);
-    }
     setupStaticRoutes() {
         this._app.get('/', (_req, res)=>{
             this.serveHtmlWithPorts(res);
@@ -1129,7 +1019,7 @@ class PlaygroundServer {
     async launch(port) {
         if (this._activeConnection.agentFactory && !this.sessionManager) {
             console.log('Initializing agent from factory function...');
-            this._activeConnection.agent = await this._activeConnection.agentFactory();
+            this.setActiveAgent(await this._activeConnection.agentFactory());
             this._activeConnection.session = {
                 connected: true,
                 metadata: {}
@@ -1151,6 +1041,7 @@ class PlaygroundServer {
         await this.destroyCurrentSession().catch((error)=>{
             console.warn('Failed to destroy current session during shutdown:', error);
         });
+        this._mjpegHandler.shutdown();
         return new Promise((resolve, reject)=>{
             if (this.server) {
                 this.taskExecutionDumps = {};
@@ -1174,12 +1065,13 @@ class PlaygroundServer {
         _define_property(this, "id", void 0);
         _define_property(this, "scrcpyPort", void 0);
         _define_property(this, "_initialized", false);
-        _define_property(this, "_nativeMjpegAvailable", null);
-        _define_property(this, "_nativeMjpegFailedAt", null);
-        _define_property(this, "_interfaceMjpegHub", createInterfaceMjpegHub({
-            initialFrameTimeoutMs: PlaygroundServer.INTERFACE_MJPEG_INITIAL_FRAME_TIMEOUT_MS,
-            idleStopMs: PlaygroundServer.INTERFACE_MJPEG_IDLE_STOP_MS,
-            debug: debugMjpeg
+        _define_property(this, "_mjpegHandler", new MjpegStreamHandler({
+            getNativeUrl: ()=>this._activeConnection.agent?.interface?.mjpegStreamUrl,
+            getActiveInterface: ()=>this._activeConnection.agent?.interface ?? null,
+            takeScreenshot: ()=>this.getActiveAgentOrThrow().interface.screenshotBase64(),
+            canTakeScreenshot: ()=>'function' == typeof this._activeConnection.agent?.interface?.screenshotBase64,
+            isAgentReady: ()=>this._agentReady,
+            recoverFromPreviewError: async (error, reason)=>(await this.recoverActiveAgentAfterPreviewError(error, reason))?.interface ?? null
         }));
         _define_property(this, "sessionManager", void 0);
         _define_property(this, "sessionSetupState", 'ready');
@@ -1206,12 +1098,9 @@ class PlaygroundServer {
         this.taskExecutionDumps = {};
         this.id = id || utils_uuid();
         if ('function' == typeof agent) this._activeConnection.agentFactory = agent;
-        else this._activeConnection.agent = agent || null;
+        else this.setActiveAgent(agent || null);
     }
 }
-_define_property(PlaygroundServer, "MJPEG_NEGATIVE_CACHE_MS", 10000);
-_define_property(PlaygroundServer, "INTERFACE_MJPEG_INITIAL_FRAME_TIMEOUT_MS", 1500);
-_define_property(PlaygroundServer, "INTERFACE_MJPEG_IDLE_STOP_MS", 2000);
 const server = PlaygroundServer;
 export { InteractParamsValidationError, PlaygroundServer, buildInteractParams, createManualExecutorContext, server as default, serializeZodField };