@ozaiya/openclaw-channel 0.10.44 → 0.10.45

package/dist/src/gateway.js

@@ -28,7 +28,8 @@ import * as desktop from "./desktopContainer.js";
 import { createSandboxFileSession, handleSandboxFileMessage } from "./sandboxFileTransfer.js";
 import { acquireCdpScreencast, releaseCdpScreencast } from "./sandboxScreenCdp.js";
 import { createSandboxRtcSession } from "./sandboxScreenRtc.js";
-import { probeFfmpeg, createH264Transcoder } from "./sandboxScreenH264.js";
+import { probeFfmpeg, createH264Transcoder, createBitrateController } from "./sandboxScreenH264.js";
+import { probeDesktopSource, createDesktopVideo, createDesktopInput } from "./sandboxScreenDesktop.js";
 import { unsealScreenKey, encryptScreenPayload, decryptScreenPayload, E2E_INPUT_MARKER } from "./sandboxScreenE2E.js";
 import { readSandboxIdentity } from "./sandboxIdentity.js";
 import { WeChatMonitorSession, getMonitor, getAllMonitors } from "./wechatMonitor.js";
@@ -583,7 +584,7 @@ export async function startGatewayMode(options) {
                     // DataChannel (P2P, or TURN via coturn) and move frames + input onto it —
                     // skipping the double-proxied relay, which is the dominant latency. The
                     // relay stays connected as signaling pipe + automatic fallback.
-                    const { channelId, botId, supportsRtc, supportsE2E, iceServers, requestVnc } = payload.params;
+                    const { channelId, botId, supportsRtc, supportsE2E, iceServers, requestVnc, requestDesktop } = payload.params;
                     // Full-desktop noVNC tier (the bot's screenMode='vnc' → server &vnc=1 → requestVnc). Instead
                     // of the CDP screencast, raw-pipe the sandbox-browser's websockify (x11vnc on Xvfb :1)
                     // straight to the relay — exactly like desktop-screen-open. The viewer runs noVNC/RFB and
@@ -695,7 +696,20 @@ export async function startGatewayMode(options) {
                     let userQuality = null;
                     let userBitrateKbps = null;
                     let userMinIntervalMs = null;
-                    log?.info(`[gateway] sandbox-screen-open channelId=${channelId} botId=${botId} (cdp${supportsRtc ? '+rtc' : ''}${expectE2E ? '+e2e' : ''})`);
+                    // Desktop source (viewer asked with &desktop=1 — screenMode='vnc' bots):
+                    // the whole X display + real X11 input, served by the browser
+                    // container's control server. Compose mode only (host-mode sandboxes
+                    // don't run it). Unavailable ⇒ the page (CDP) source, and the viewer is
+                    // told so it can fall back to its noVNC pipe.
+                    const controlHost = process.env.NOVNC_HOST || '';
+                    const controlPort = Number(process.env.RECORD_CONTROL_PORT || '8091');
+                    const desktopInfo = requestDesktop && desktop.isInContainer && controlHost
+                        ? await probeDesktopSource(controlHost, controlPort)
+                        : null;
+                    const useDesktop = !!desktopInfo;
+                    if (requestDesktop && !useDesktop)
+                        log?.warn(`[gateway] sandbox-screen: desktop source unavailable channelId=${channelId} — page source`);
+                    log?.info(`[gateway] sandbox-screen-open channelId=${channelId} botId=${botId} (${useDesktop ? `desktop ${desktopInfo.width}x${desktopInfo.height}` : 'cdp'}${supportsRtc ? '+rtc' : ''}${expectE2E ? '+e2e' : ''})`);
                     // Resolve the browser container's CDP host:port (the socat-proxied DevTools
                     // endpoint). In-container: NOVNC_HOST + CDP port. On host: the mapped 9222.
                     let cdpHost = process.env.NOVNC_HOST || 'localhost';
@@ -735,7 +749,8 @@ export async function startGatewayMode(options) {
                     if (!desktop.isInContainer && browserContainerId) {
                         ffmpegCandidates.push(['docker', 'exec', '-i', browserContainerId, 'ffmpeg']);
                     }
-                    const ffmpegCmd = await probeFfmpeg(ffmpegCandidates, dockerEnvLocal);
+                    // (The desktop source encodes inside the browser container — no local ffmpeg needed.)
+                    const ffmpegCmd = useDesktop ? null : await probeFfmpeg(ffmpegCandidates, dockerEnvLocal);
                     if (ffmpegCmd)
                         log?.info(`[gateway] sandbox-screen: H.264 encoder available (${ffmpegCmd[0]})`);
                     // Open the relay WS back to the server.
@@ -751,7 +766,7 @@ export async function startGatewayMode(options) {
                         ack({ ok: false, error: `Failed to connect relay WS: ${String(err)}` });
                         return;
                     }
-                    const entry = { relayWs, cdpKey: { host: cdpHost, port: cdpPort } };
+                    const entry = { relayWs, cdpKey: useDesktop ? undefined : { host: cdpHost, port: cdpPort } };
                     screenConnections.set(channelId, entry);
                     let relayOpen = false;
                     const earlyFrames = [];
@@ -770,20 +785,20 @@ export async function startGatewayMode(options) {
                                 if (!pt || pt.length < 1)
                                     return;
                                 if (pt[0] === 0)
-                                    entry.cdp?.handleBinaryInput(pt.subarray(1));
+                                    entry.input?.handleBinaryInput(pt.subarray(1));
                                 else {
                                     try {
                                         const m = JSON.parse(pt.subarray(1).toString());
                                         if (m.type === 'input')
-                                            entry.cdp?.handleInput(m);
+                                            entry.input?.handleInput(m);
                                         else if (m.type === 'viewport-size')
-                                            entry.cdp?.resize(Number(m.width), Number(m.height));
+                                            entry.input?.resize(Number(m.width), Number(m.height));
                                     }
                                     catch { /* ignore */ }
                                 }
                                 return;
                             }
-                            entry.cdp?.handleBinaryInput(buf);
+                            entry.input?.handleBinaryInput(buf);
                             return;
                         }
                         try {
@@ -805,9 +820,9 @@ export async function startGatewayMode(options) {
                                 return;
                             }
                             if (msg.type === 'input')
-                                entry.cdp?.handleInput(msg);
+                                entry.input?.handleInput(msg);
                             else if (msg.type === 'viewport-size')
-                                entry.cdp?.resize(Number(msg.width), Number(msg.height));
+                                entry.input?.resize(Number(msg.width), Number(msg.height));
                             // The viewer pings every second for its latency indicator — answer on
                             // the active transport (silently dropped before, which left the
                             // latency stat permanently empty on the bot view).
@@ -827,13 +842,17 @@ export async function startGatewayMode(options) {
                                     if (isFinite(mbps) && mbps > 0) {
                                         userBitrateKbps = Math.max(300, Math.min(40000, Math.round(mbps * 1000)));
                                         userQuality = Math.max(40, Math.min(95, Math.round(45 + Math.log2(mbps) * 10)));
-                                        transcoder?.updateBitrate(userBitrateKbps);
+                                        // The pick is the adaptation ceiling; it jumps there and
+                                        // backs off from it only on loss.
+                                        bitrate.setCeiling(userBitrateKbps);
                                     }
                                 }
                                 else if (msg.setting === 'frameRate') {
                                     const f = Number(msg.value);
-                                    if (isFinite(f) && f > 0)
+                                    if (isFinite(f) && f > 0) {
                                         userMinIntervalMs = Math.round(1000 / Math.max(1, Math.min(30, f)));
+                                        desktopVideo?.setFps(f);
+                                    }
                                 }
                             }
                         }
@@ -847,6 +866,17 @@ export async function startGatewayMode(options) {
                         t?.close();
                     }
                     catch { /* ignore */ } };
+                    // Desktop source halves (useDesktop only): the x11grab stream and the XTest input.
+                    let desktopVideo = null;
+                    let desktopInput = null;
+                    // Media-track bitrate follows the viewer's RTCP loss reports + PLIs,
+                    // between a floor and the quality pick (or a default ceiling).
+                    const bitrate = createBitrateController({
+                        initialKbps: 1200,
+                        maxKbps: 2500,
+                        apply: (kbps) => { transcoder?.updateBitrate(kbps); desktopVideo?.updateBitrate(kbps); },
+                        log: (m) => log?.info(m),
+                    });
                     if (supportsRtc) {
                         // The container NAT is symmetric on Docker-Desktop-Mac, so a missing
                         // TURN entry here means the direct tier can't ever connect — make
@@ -870,16 +900,25 @@ export async function startGatewayMode(options) {
                             onData: (data) => handleViewerData(data, typeof data !== 'string'),
                             onActive: () => log?.info(`[gateway] sandbox-screen: P2P DataChannel active channelId=${channelId} — frames direct, relay on standby`),
                             onClosed: () => log?.info(`[gateway] sandbox-screen: P2P closed channelId=${channelId} — frames back on relay`),
-                            enableVideo: !!ffmpegCmd,
+                            enableVideo: !!ffmpegCmd || useDesktop,
                             onMediaStateChange: (ready) => {
+                                if (ready)
+                                    bitrate.settle();
+                                if (useDesktop) {
+                                    // Same x11grab stream, re-dialed as H.264 for the media track
+                                    // or JPEG for the DataChannel/relay.
+                                    log?.info(`[gateway] sandbox-screen: desktop ${ready ? 'H.264 media track up' : 'media track down — JPEG'} channelId=${channelId}`);
+                                    desktopVideo?.setMode(ready ? 'h264' : 'jpeg');
+                                    return;
+                                }
                                 if (ready && ffmpegCmd && !transcoder && !transcoderDead) {
                                     log?.info(`[gateway] sandbox-screen: H.264 media track up channelId=${channelId} — starting encoder`);
                                     transcoder = createH264Transcoder({
                                         cmdPrefix: ffmpegCmd,
                                         env: dockerEnvLocal,
                                         fps: 25,
-                                        // Honor a quality pick made before the media track came up.
-                                        bitrateKbps: userBitrateKbps ?? 1200,
+                                        // Current adaptive target (includes a quality pick made before the track came up).
+                                        bitrateKbps: bitrate.current(),
                                         onAccessUnit: (au) => { rtc?.sendVideoFrame(au); },
                                         onExit: () => { transcoder = null; transcoderDead = true; log?.warn(`[gateway] sandbox-screen: encoder gone channelId=${channelId} — JPEG fallback`); },
                                         log: { info: (m) => log?.info(m), warn: (m) => log?.warn(m) },
@@ -891,7 +930,8 @@ export async function startGatewayMode(options) {
                             },
                             // Viewer PLI → force the encoder to emit a keyframe now (debounced
                             // inside the transcoder). No transcoder ⇒ no-op (GOP recovers).
-                            onKeyframeRequest: () => transcoder?.forceKeyframe(),
+                            onKeyframeRequest: () => { bitrate.onPli(); transcoder?.forceKeyframe(); desktopVideo?.forceKeyframe(); },
+                            onLossReport: (fraction) => bitrate.onLoss(fraction),
                             log: { info: (m) => log?.info(m), warn: (m) => log?.warn(m) },
                         });
                     }
@@ -900,10 +940,22 @@ export async function startGatewayMode(options) {
                             return;
                         screenConnections.delete(channelId);
                         stopTranscoder();
+                        try {
+                            desktopVideo?.close();
+                        }
+                        catch { /* ignore */ }
+                        try {
+                            desktopInput?.close();
+                        }
+                        catch { /* ignore */ }
+                        desktopVideo = null;
+                        desktopInput = null;
                         // The CDP session is SHARED across viewers of this browser — release
                         // our slot; it closes only when the last viewer leaves.
-                        releaseCdpScreencast(cdpHost, cdpPort, channelId);
+                        if (!useDesktop)
+                            releaseCdpScreencast(cdpHost, cdpPort, channelId);
                         entry.cdp = undefined;
+                        entry.input = undefined;
                         try {
                             rtc?.close();
                         }
@@ -955,88 +1007,118 @@ export async function startGatewayMode(options) {
                             earlyFrames.push(data);
                         }
                     };
-                    // Attach to the (shared, per-browser) CDP screencast — one capture loop
-                    // no matter how many viewers are watching this sandbox.
-                    acquireCdpScreencast({
-                        host: cdpHost,
-                        port: cdpPort,
-                        minIntervalMs: 55,
-                        quality: 40,
-                        log: { info: (m) => log?.info(m), warn: (m) => log?.warn(m) },
-                    }, channelId, {
-                        // Transport-adaptive tuning. Relay tier: ~18fps / q40 — capture is only
-                        // ~38ms, so the bottleneck is per-frame transfer over the (Cloudflare-
-                        // proxied) relay; smaller frames transfer faster = lower latency. Direct
-                        // tier (P2P/TURN DataChannel): the proxy hop is gone, so spend the
-                        // headroom on fidelity + smoothness (~25fps / q60). H.264 tier: the JPEG
-                        // is only encoder input (bandwidth decouples from its size), so feed it
-                        // high quality at the encoder's frame rate. The server drops frames a
-                        // slow viewer can't take (dropFramesOnBackpressure) and the screencast
-                        // self-throttles on the active transport's buffer, so these stay
-                        // ceilings. (Shared session: max quality / min interval across viewers.)
-                        getQuality: () => (userQuality ?? (transcoder && rtc?.isMediaReady() ? 80 : rtc?.isActive() ? 60 : 40)),
-                        getMinIntervalMs: () => (userMinIntervalMs ?? (transcoder && rtc?.isMediaReady() ? 40 : rtc?.isActive() ? 40 : 55)),
-                        onFrame: (jpeg) => {
-                            // H.264 tier: the poll feeds the encoder; the WebRTC media track is
-                            // already DTLS-E2E, so no app-layer crypto there.
-                            if (transcoder && rtc?.isMediaReady()) {
-                                transcoder.feed(jpeg);
+                    // A JPEG frame for the DataChannel/relay tiers. With E2E negotiated,
+                    // encrypt with the viewer's key; until it arrives, WITHHOLD (never leak a
+                    // plaintext frame to the server). Frames are stateless, so the first one
+                    // just shows a beat later.
+                    const sendJpegFrame = (jpeg) => {
+                        if (expectE2E) {
+                            if (!e2eKey)
                                 return;
-                            }
-                            // JPEG over the relay/DC. With E2E negotiated, encrypt with the
-                            // viewer's key; until it arrives, WITHHOLD (never leak a plaintext
-                            // frame to the server). Frames are stateless, so the first one just
-                            // shows a beat later.
-                            if (expectE2E) {
-                                if (!e2eKey)
+                            sendToViewer(encryptScreenPayload(e2eKey, jpeg));
+                            return;
+                        }
+                        sendToViewer(jpeg);
+                    };
+                    if (useDesktop) {
+                        desktopVideo = createDesktopVideo({
+                            host: controlHost,
+                            port: controlPort,
+                            mode: 'jpeg', // the media track (if any) comes up later
+                            fps: 25,
+                            bitrateKbps: bitrate.current(),
+                            onAccessUnit: (au) => { rtc?.sendVideoFrame(au); },
+                            onJpeg: sendJpegFrame,
+                            onExit: () => {
+                                // The display can't be streamed after all — tell the viewer (it
+                                // falls back to noVNC) and end this session.
+                                try {
+                                    if (relayWs.readyState === 1)
+                                        relayWs.send(JSON.stringify({ type: 'screen-source', source: 'page' }));
+                                }
+                                catch { /* ignore */ }
+                                cleanupScreen('desktop-stream-failed');
+                            },
+                            log: { info: (m) => log?.info(m), warn: (m) => log?.warn(m) },
+                        });
+                        desktopInput = createDesktopInput({ host: controlHost, port: controlPort, log: { info: (m) => log?.info(m), warn: (m) => log?.warn(m) } });
+                        entry.input = desktopInput;
+                    }
+                    else {
+                        // Attach to the (shared, per-browser) CDP screencast — one capture loop
+                        // no matter how many viewers are watching this sandbox.
+                        acquireCdpScreencast({
+                            host: cdpHost,
+                            port: cdpPort,
+                            minIntervalMs: 55,
+                            quality: 40,
+                            log: { info: (m) => log?.info(m), warn: (m) => log?.warn(m) },
+                        }, channelId, {
+                            // Transport-adaptive tuning. Relay tier: ~18fps / q40 — capture is only
+                            // ~38ms, so the bottleneck is per-frame transfer over the (Cloudflare-
+                            // proxied) relay; smaller frames transfer faster = lower latency. Direct
+                            // tier (P2P/TURN DataChannel): the proxy hop is gone, so spend the
+                            // headroom on fidelity + smoothness (~25fps / q60). H.264 tier: the JPEG
+                            // is only encoder input (bandwidth decouples from its size), so feed it
+                            // high quality at the encoder's frame rate. The server drops frames a
+                            // slow viewer can't take (dropFramesOnBackpressure) and the screencast
+                            // self-throttles on the active transport's buffer, so these stay
+                            // ceilings. (Shared session: max quality / min interval across viewers.)
+                            getQuality: () => (userQuality ?? (transcoder && rtc?.isMediaReady() ? 80 : rtc?.isActive() ? 60 : 40)),
+                            getMinIntervalMs: () => (userMinIntervalMs ?? (transcoder && rtc?.isMediaReady() ? 40 : rtc?.isActive() ? 40 : 55)),
+                            onFrame: (jpeg) => {
+                                // H.264 tier: the poll feeds the encoder; the WebRTC media track is
+                                // already DTLS-E2E, so no app-layer crypto there.
+                                if (transcoder && rtc?.isMediaReady()) {
+                                    transcoder.feed(jpeg);
                                     return;
-                                sendToViewer(encryptScreenPayload(e2eKey, jpeg));
+                                }
+                                sendJpegFrame(jpeg);
+                            },
+                            // Frames carry no pointer (headless screenshots have no OS cursor), so
+                            // stream the page's pointer position — the viewer draws the arrow. The
+                            // shape (CSS cursor under the pointer) gives the hover affordance.
+                            onCursor: (x, y, click, shape) => {
+                                const msg = { type: 'cursor', x, y };
+                                if (click)
+                                    msg.click = true;
+                                if (shape)
+                                    msg.shape = shape;
+                                sendToViewer(JSON.stringify(msg));
+                            },
+                            // Text copied inside the sandbox page → viewer clipboard. Clipboard is
+                            // sensitive (passwords), so encrypt it under E2E — sent as a base64
+                            // `enc` field (text, so the server's frame-drop never touches it). When
+                            // E2E is expected but the key hasn't landed, drop rather than leak.
+                            onClipboard: (text) => {
+                                if (expectE2E) {
+                                    if (!e2eKey)
+                                        return;
+                                    sendToViewer(JSON.stringify({ type: 'clipboard', enc: encryptScreenPayload(e2eKey, Buffer.from(text, 'utf-8')).toString('base64') }));
+                                }
+                                else {
+                                    sendToViewer(JSON.stringify({ type: 'clipboard', text }));
+                                }
+                            },
+                            getBackpressure: () => {
+                                if (transcoder && rtc?.isMediaReady())
+                                    return 0; // encoder drops internally; RTP self-paces
+                                return rtc?.isActive() ? rtc.getBufferedAmount() : (relayOpen ? relayWs.bufferedAmount : 0);
+                            },
+                        }).then((session) => {
+                            if (!session) {
+                                cleanupScreen('cdp-init-failed');
                                 return;
                             }
-                            sendToViewer(jpeg);
-                        },
-                        // Frames carry no pointer (headless screenshots have no OS cursor), so
-                        // stream the page's pointer position — the viewer draws the arrow. The
-                        // shape (CSS cursor under the pointer) gives the hover affordance.
-                        onCursor: (x, y, click, shape) => {
-                            const msg = { type: 'cursor', x, y };
-                            if (click)
-                                msg.click = true;
-                            if (shape)
-                                msg.shape = shape;
-                            sendToViewer(JSON.stringify(msg));
-                        },
-                        // Text copied inside the sandbox page → viewer clipboard. Clipboard is
-                        // sensitive (passwords), so encrypt it under E2E — sent as a base64
-                        // `enc` field (text, so the server's frame-drop never touches it). When
-                        // E2E is expected but the key hasn't landed, drop rather than leak.
-                        onClipboard: (text) => {
-                            if (expectE2E) {
-                                if (!e2eKey)
-                                    return;
-                                sendToViewer(JSON.stringify({ type: 'clipboard', enc: encryptScreenPayload(e2eKey, Buffer.from(text, 'utf-8')).toString('base64') }));
-                            }
-                            else {
-                                sendToViewer(JSON.stringify({ type: 'clipboard', text }));
-                            }
-                        },
-                        getBackpressure: () => {
-                            if (transcoder && rtc?.isMediaReady())
-                                return 0; // encoder drops internally; RTP self-paces
-                            return rtc?.isActive() ? rtc.getBufferedAmount() : (relayOpen ? relayWs.bufferedAmount : 0);
-                        },
-                    }).then((session) => {
-                        if (!session) {
-                            cleanupScreen('cdp-init-failed');
-                            return;
-                        }
-                        if (!screenConnections.has(channelId)) {
-                            releaseCdpScreencast(cdpHost, cdpPort, channelId);
-                            return;
-                        } // closed during init
-                        entry.cdp = session;
-                        log?.info(`[gateway] sandbox-screen: CDP screencast attached channelId=${channelId} (${cdpHost}:${cdpPort})`);
-                    }).catch((err) => { log?.warn(`[gateway] sandbox-screen: CDP init error ${String(err)}`); cleanupScreen('cdp-error'); });
+                            if (!screenConnections.has(channelId)) {
+                                releaseCdpScreencast(cdpHost, cdpPort, channelId);
+                                return;
+                            } // closed during init
+                            entry.cdp = session;
+                            entry.input = session;
+                            log?.info(`[gateway] sandbox-screen: CDP screencast attached channelId=${channelId} (${cdpHost}:${cdpPort})`);
+                        }).catch((err) => { log?.warn(`[gateway] sandbox-screen: CDP init error ${String(err)}`); cleanupScreen('cdp-error'); });
+                    }
                     relayWs.on("open", () => {
                         try {
                             relayWs._socket?.setNoDelay?.(true);
@@ -1044,6 +1126,10 @@ export async function startGatewayMode(options) {
                         catch { /* ignore */ }
                         log?.info(`[gateway] sandbox-screen: relay WS connected channelId=${channelId}`);
                         relayOpen = true;
+                        // Which source this session streams: 'desktop' (whole X display, X11
+                        // input) or 'page' (CDP). A viewer that asked for the desktop and
+                        // hears 'page' falls back to its noVNC pipe.
+                        relayWs.send(JSON.stringify({ type: 'screen-source', source: useDesktop ? 'desktop' : 'page' }));
                         for (const f of earlyFrames) {
                             if (relayWs.readyState === 1)
                                 relayWs.send(f);