neoagent 2.4.1-beta.6 → 2.4.1-beta.8

package/server/routes/stream.js

@@ -0,0 +1,178 @@
+'use strict';
+
+const express = require('express');
+const { requireAuth } = require('../middleware/auth');
+const { sanitizeError } = require('../utils/security');
+const { AndroidStream } = require('../services/streaming/android-stream');
+const { BrowserStream } = require('../services/streaming/browser-stream');
+
+const router = express.Router();
+router.use(requireAuth);
+
+function boundedInt(value, fallback, min, max) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return fallback;
+  return Math.min(max, Math.max(min, Math.floor(parsed)));
+}
+
+function streamHub(req) {
+  const hub = req.app?.locals?.streamHub;
+  if (!hub) throw new Error('Stream hub is unavailable.');
+  return hub;
+}
+
+async function desktopProvider(req) {
+  const factory = req.app?.locals?.getDesktopProviderForUser;
+  if (typeof factory !== 'function') {
+    throw new Error('Desktop provider is unavailable.');
+  }
+  return factory(req.session?.userId);
+}
+
+async function androidController(req) {
+  const runtimeManager = req.app?.locals?.runtimeManager;
+  if (runtimeManager && typeof runtimeManager.getAndroidProviderForUser === 'function') {
+    return runtimeManager.getAndroidProviderForUser(req.session?.userId);
+  }
+  throw new Error('Android controller is unavailable.');
+}
+
+async function browserController(req) {
+  const runtimeManager = req.app?.locals?.runtimeManager;
+  if (runtimeManager && typeof runtimeManager.getBrowserProviderForUser === 'function') {
+    return runtimeManager.getBrowserProviderForUser(req.session?.userId);
+  }
+  throw new Error('Browser controller is unavailable.');
+}
+
+function normalizePlatform(value) {
+  const platform = String(value || '').trim().toLowerCase();
+  if (platform === 'desktop' || platform === 'android' || platform === 'browser') return platform;
+  const error = new Error('platform must be desktop, android, or browser.');
+  error.status = 400;
+  throw error;
+}
+
+function normalizeDeviceId(platform, value) {
+  const deviceId = String(value || '').trim();
+  if (deviceId) return deviceId;
+  if (platform === 'browser') return 'browser';
+  const error = new Error('deviceId is required.');
+  error.status = 400;
+  throw error;
+}
+
+async function resolveStartDeviceId(platform, req) {
+  const rawDeviceId = String(req.body?.deviceId || '').trim();
+  if (platform === 'browser') return { deviceId: 'browser', controller: null };
+  if (platform === 'android') {
+    const controller = await androidController(req);
+    const status = typeof controller.getStatus === 'function'
+      ? await controller.getStatus().catch(() => ({}))
+      : {};
+    const adbSerial = String(status?.adbSerial || controller.adbSerial || '').trim();
+    if (rawDeviceId && adbSerial && rawDeviceId !== adbSerial) {
+      const error = new Error('Android deviceId does not match the active emulator serial.');
+      error.status = 400;
+      throw error;
+    }
+    if (rawDeviceId) return { deviceId: rawDeviceId, controller };
+    if (!adbSerial) throw new Error('Android deviceId is required when no emulator serial is available.');
+    return { deviceId: adbSerial, controller };
+  }
+  if (rawDeviceId) return { deviceId: rawDeviceId, controller: null };
+  return { deviceId: normalizeDeviceId(platform, rawDeviceId), controller: null };
+}
+
+async function resolveStopDeviceId(platform, req) {
+  const rawDeviceId = String(req.body?.deviceId || '').trim();
+  if (rawDeviceId) return rawDeviceId;
+  if (platform === 'browser') return 'browser';
+  if (platform === 'android') {
+    const controller = await androidController(req);
+    const status = typeof controller.getStatus === 'function'
+      ? await controller.getStatus().catch(() => ({}))
+      : {};
+    const adbSerial = String(status?.adbSerial || controller.adbSerial || '').trim();
+    if (adbSerial) return adbSerial;
+  }
+  const error = new Error('deviceId is required.');
+  error.status = 400;
+  throw error;
+}
+
+function sendError(res, err) {
+  res.status(Number(err?.status || 500)).json({ error: sanitizeError(err) });
+}
+
+router.post('/start', async (req, res) => {
+  try {
+    const userId = req.session?.userId;
+    const platform = normalizePlatform(req.body?.platform);
+    const resolved = await resolveStartDeviceId(platform, req);
+    const deviceId = resolved.deviceId;
+    const fps = boundedInt(req.body?.fps, platform === 'android' ? 10 : 15, 1, platform === 'android' ? 15 : 20);
+    const quality = boundedInt(req.body?.quality, platform === 'android' ? 75 : 80, 30, 95);
+    const hub = streamHub(req);
+    await hub.stopStream(userId, platform, deviceId, 'restart');
+
+    if (platform === 'desktop') {
+      const provider = await desktopProvider(req);
+      const result = await provider.startStream({
+        deviceId,
+        fps,
+        quality,
+        displayId: req.body?.displayId || null,
+      });
+      const resolvedDeviceId = result?.deviceId || result?.device?.deviceId || deviceId;
+      hub.markStarted(userId, resolvedDeviceId, platform, { fps, quality }, () =>
+        provider.stopStream({ deviceId: resolvedDeviceId }));
+      return res.json({ ok: true, platform, deviceId: resolvedDeviceId, fps, quality });
+    }
+
+    if (platform === 'android') {
+      const controller = resolved.controller || await androidController(req);
+      const stream = new AndroidStream({ userId, deviceId, controller, streamHub: hub, fps, quality });
+      stream.start();
+      hub.markStarted(userId, deviceId, platform, { fps, quality }, () => stream.stop());
+      return res.json({ ok: true, platform, deviceId, fps, quality });
+    }
+
+    const controller = await browserController(req);
+    const stream = new BrowserStream({ userId, deviceId, controller, streamHub: hub, fps, quality });
+    stream.start();
+    hub.markStarted(userId, deviceId, platform, { fps, quality }, () => stream.stop());
+    return res.json({ ok: true, platform, deviceId, fps, quality });
+  } catch (err) {
+    sendError(res, err);
+  }
+});
+
+router.post('/stop', async (req, res) => {
+  try {
+    const platform = normalizePlatform(req.body?.platform);
+    const deviceId = await resolveStopDeviceId(platform, req);
+    const stopped = await streamHub(req).stopStream(req.session?.userId, platform, deviceId, 'api_stop');
+    res.json({ ok: true, stopped, platform, deviceId });
+  } catch (err) {
+    sendError(res, err);
+  }
+});
+
+router.get('/status', async (req, res) => {
+  try {
+    const platform = req.query?.platform ? normalizePlatform(req.query.platform) : null;
+    const deviceId = req.query?.deviceId
+      ? normalizeDeviceId(platform || 'desktop', req.query.deviceId)
+      : null;
+    const hub = streamHub(req);
+    if (deviceId) {
+      return res.json(hub.status(req.session?.userId, platform || 'desktop', deviceId));
+    }
+    return res.json({ streams: hub.listStatus(req.session?.userId), platform });
+  } catch (err) {
+    sendError(res, err);
+  }
+});
+
+module.exports = router;

package/server/services/android/controller.js

@@ -347,12 +347,16 @@ class AndroidController {
   // ── Actions ───────────────────────────────────────────────────────────────
 
   async screenshot(_opts = {}) {
-    const serial = this.#requireSerial();
-    const r = this.#adbCapture(serial, ['exec-out', 'screencap', '-p']);
+    const r = await this.capturePng();
     if (!r?.length) throw new Error('screencap returned no data');
     return { screenshotPath: this.#saveArtifact(r) };
   }
 
+  async capturePng(_opts = {}) {
+    const serial = this.#requireSerial();
+    return this.#adbCapture(serial, ['exec-out', 'screencap', '-p']);
+  }
+
   async observe(_opts = {}) { return this.screenshot(); }
 
   async tap({ x, y } = {}) {

package/server/services/browser/controller.js

@@ -453,6 +453,22 @@ class BrowserController {
     };
   }
 
+  async screenshotJpeg(quality = 80, options = {}) {
+    const page = await this.ensurePage();
+    const screenshotOptions = {
+      type: 'jpeg',
+      quality: Math.min(95, Math.max(30, Math.floor(Number(quality) || 80))),
+      fullPage: options.fullPage === true,
+    };
+    if (options.selector) {
+      const element = await page.$(options.selector);
+      if (element) {
+        return element.screenshot(screenshotOptions);
+      }
+    }
+    return page.screenshot(screenshotOptions);
+  }
+
   async navigate(url, options = {}) {
     const page = await this.ensurePage();
 

package/server/services/desktop/gateway.js

@@ -1,5 +1,11 @@
 const { WebSocketServer } = require('ws');
-const { DESKTOP_COMPANION_WS_PATH, parseDesktopMessage } = require('./protocol');
+const {
+  DESKTOP_COMPANION_WS_PATH,
+  FRAME_TYPE_VIDEO,
+  MAX_DESKTOP_STREAM_FRAME_BYTES,
+  parseBinaryFrame,
+  parseDesktopMessage,
+} = require('./protocol');
 const {
   assertDesktopHelloAuth,
   isDesktopCompanionHello,
@@ -72,8 +78,11 @@ function createUpgradeThrottleObserver() {
   return { record, snapshot };
 }
 
-function bindDesktopCompanionGateway(httpServer, app, sessionMiddleware) {
-  const wss = new WebSocketServer({ noServer: true });
+function bindDesktopCompanionGateway(httpServer, app, sessionMiddleware, streamHub = null) {
+  const wss = new WebSocketServer({
+    noServer: true,
+    maxPayload: MAX_DESKTOP_STREAM_FRAME_BYTES,
+  });
   const upgradeAttempts = new Map();
   const upgradeThrottleObserver = createUpgradeThrottleObserver();
 
@@ -173,6 +182,22 @@ function bindDesktopCompanionGateway(httpServer, app, sessionMiddleware) {
               device,
             }));
             ws.on('message', (nextData) => {
+              const activeStreamHub = streamHub || app?.locals?.streamHub || null;
+              if (
+                activeStreamHub
+                && Buffer.isBuffer(nextData)
+                && nextData.length > 10
+                && nextData[0] === FRAME_TYPE_VIDEO
+              ) {
+                const frame = parseBinaryFrame(nextData);
+                if (frame) {
+                  activeStreamHub.handleFrame(req.session.userId, device.deviceId, {
+                    ...frame,
+                    platform: 'desktop',
+                  });
+                }
+                return;
+              }
               let parsed;
               try {
                 parsed = parseDesktopMessage(nextData);

package/server/services/desktop/protocol.js

@@ -3,6 +3,8 @@ const DESKTOP_COMPANION_WS_PATH = '/api/desktop/ws';
 const DESKTOP_COMMANDS = Object.freeze({
   GET_STATUS: 'getStatus',
   CAPTURE_FRAME: 'captureFrame',
+  STREAM_START: 'startStream',
+  STREAM_STOP: 'stopStream',
   OBSERVE: 'observe',
   CLICK: 'click',
   DRAG: 'drag',
@@ -18,6 +20,9 @@ const DESKTOP_COMMANDS = Object.freeze({
   PING: 'ping',
 });
 
+const FRAME_TYPE_VIDEO = 0x01;
+const MAX_DESKTOP_STREAM_FRAME_BYTES = 8 * 1024 * 1024;
+
 class DesktopCompanionUnavailableError extends Error {
   constructor(message = 'Desktop companion is not connected.') {
     super(message);
@@ -55,11 +60,38 @@ function parseDesktopMessage(data) {
   }
 }
 
+function parseBinaryFrame(buffer) {
+  if (
+    !Buffer.isBuffer(buffer)
+    || buffer.length <= 10
+    || buffer.length > MAX_DESKTOP_STREAM_FRAME_BYTES
+    || buffer[0] !== FRAME_TYPE_VIDEO
+  ) {
+    return null;
+  }
+  const jpeg = buffer.subarray(10);
+  const hasJpegMarkers = jpeg.length >= 4
+    && jpeg[0] === 0xff
+    && jpeg[1] === 0xd8
+    && jpeg[jpeg.length - 2] === 0xff
+    && jpeg[jpeg.length - 1] === 0xd9;
+  if (!hasJpegMarkers) return null;
+  return {
+    seq: buffer.readUInt32BE(1),
+    ts: buffer.readUInt32BE(5),
+    flags: buffer.readUInt8(9),
+    jpeg,
+  };
+}
+
 module.exports = {
   DESKTOP_COMPANION_WS_PATH,
   DESKTOP_COMMANDS,
+  FRAME_TYPE_VIDEO,
+  MAX_DESKTOP_STREAM_FRAME_BYTES,
   DesktopCompanionUnavailableError,
   DesktopCompanionSelectionError,
   createDesktopCommandMessage,
+  parseBinaryFrame,
   parseDesktopMessage,
 };

package/server/services/desktop/provider.js

@@ -130,6 +130,16 @@ class DesktopProvider {
     return this._dispatch(DESKTOP_COMMANDS.CAPTURE_FRAME, options);
   }
 
+  startStream(options = {}) {
+    this._assertReady();
+    return this.registry.startStream(this.userId, options.deviceId || null, options);
+  }
+
+  stopStream(options = {}) {
+    this._assertReady();
+    return this.registry.stopStream(this.userId, options.deviceId || null);
+  }
+
   observe(options = {}) {
     return this._dispatch(DESKTOP_COMMANDS.OBSERVE, options);
   }

package/server/services/desktop/registry.js

@@ -1,6 +1,8 @@
 const crypto = require('crypto');
 const db = require('../../db/database');
 const {
+  DESKTOP_COMMANDS,
+  FRAME_TYPE_VIDEO,
   DesktopCompanionSelectionError,
   DesktopCompanionUnavailableError,
   createDesktopCommandMessage,
@@ -373,6 +375,42 @@ class DesktopCompanionRegistry {
     };
   }
 
+  async startStream(userId, deviceId, options = {}) {
+    const device = this.resolveDevice(userId, deviceId);
+    const connection = this.getConnection(userId, device.deviceId);
+    if (!connection || !connection.isOpen()) {
+      throw new DesktopCompanionUnavailableError();
+    }
+    const result = await connection.sendCommand(DESKTOP_COMMANDS.STREAM_START, {
+      fps: options.fps,
+      quality: options.quality,
+      displayId: options.displayId || device.activeDisplayId || null,
+    }, options);
+    connection._streaming = true;
+    return {
+      ...result,
+      success: result?.success !== false,
+      deviceId: device.deviceId,
+      device: this.getDeviceRecordByDeviceId(userId, device.deviceId),
+    };
+  }
+
+  async stopStream(userId, deviceId) {
+    const device = this.resolveDevice(userId, deviceId);
+    const connection = this.getConnection(userId, device.deviceId);
+    if (!connection || !connection.isOpen()) {
+      throw new DesktopCompanionUnavailableError();
+    }
+    const result = await connection.sendCommand(DESKTOP_COMMANDS.STREAM_STOP, {});
+    connection._streaming = false;
+    return {
+      ...result,
+      success: result?.success !== false,
+      deviceId: device.deviceId,
+      device: this.getDeviceRecordByDeviceId(userId, device.deviceId),
+    };
+  }
+
   getStatus(userId) {
     const devices = this.listDevices(userId);
     const onlineDevices = devices.filter((device) => device.online);
@@ -511,6 +549,9 @@ class DesktopCompanionConnection {
   }
 
   _handleMessage(data) {
+    if (Buffer.isBuffer(data) && data.length > 0 && data[0] === FRAME_TYPE_VIDEO) {
+      return;
+    }
     let message;
     try {
       message = parseDesktopMessage(data);

package/server/services/manager.js

@@ -438,6 +438,7 @@ function configureRealtime(app, io, services) {
     recordingManager: services.recordingManager,
     memoryManager: services.memoryManager,
     voiceRuntimeManager: services.voiceRuntimeManager,
+    streamHub: app.locals.streamHub || services.streamHub || null,
     app,
   });
   app.locals.io = io;
@@ -516,6 +517,7 @@ async function startServices(app, io) {
       recordingManager,
       memoryManager,
       voiceRuntimeManager,
+      streamHub: app.locals.streamHub || null,
     });
 
     resumePendingRecordingSessions(recordingManager);
@@ -541,6 +543,15 @@ async function stopServices(app) {
     }
   }
 
+  if (app.locals.streamHub) {
+    try {
+      await app.locals.streamHub.shutdown();
+      logServiceReady('Stream hub stopped');
+    } catch (err) {
+      console.error('[StreamHub] Shutdown error:', getErrorMessage(err));
+    }
+  }
+
   if (app.locals.memoryIngestionService) {
     try {
       app.locals.memoryIngestionService.stop();

package/server/services/runtime/backends/local-vm.js

@@ -253,6 +253,12 @@ class VmBrowserProvider {
   extract(selector, attribute, all = false) { return this.client.request('POST', '/browser/extract', { selector, attribute, all }); }
   evaluate(script) { return this.client.request('POST', '/browser/execute', { code: script }); }
   async screenshot(options = {}) { return this.#materialize(await this.client.request('POST', '/browser/screenshot', options)); }
+  async screenshotJpeg(quality = 80, options = {}) {
+    const result = await this.client.request('POST', '/browser/screenshot-jpeg', { ...options, quality });
+    const content = String(result?.contentBase64 || '');
+    if (!content) throw new Error('VM browser screenshot-jpeg returned no data.');
+    return Buffer.from(content, 'base64');
+  }
   launch(options = {}) { return this.client.request('POST', '/browser/launch', options); }
   closeBrowser() { return this.client.request('POST', '/browser/close'); }
   fill(selector, value) { return this.type(selector, value); }

package/server/services/runtime/manager.js

@@ -101,6 +101,10 @@ class RuntimeManager {
     const settings = this.getSettings(userId);
     if (settings.cli_backend === 'desktop' && this.hasActiveDesktopCompanion(userId)) {
       const desktopProvider = this.getDesktopCliProvider(userId);
+      const preferredDeviceId = settings.cli_desktop_device_id;
+      if (preferredDeviceId) {
+        desktopProvider.selectDevice(preferredDeviceId);
+      }
       return {
         backend: 'desktop-companion',
         execute: (command, options = {}) => desktopProvider.executeCommand(command, options),

package/server/services/streaming/android-stream.js

@@ -0,0 +1,72 @@
+'use strict';
+
+const sharp = require('sharp');
+
+function clampInt(value, fallback, min, max) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return fallback;
+  return Math.min(max, Math.max(min, Math.floor(parsed)));
+}
+
+class AndroidStream {
+  constructor({ userId, deviceId, controller, streamHub, fps = 10, quality = 75 }) {
+    this.userId = String(userId || '');
+    this.deviceId = String(deviceId || '');
+    this.controller = controller;
+    this.streamHub = streamHub;
+    this.fps = clampInt(fps, 10, 1, 15);
+    this.quality = clampInt(quality, 75, 30, 95);
+    this._timer = null;
+    this._capturing = false;
+    this._seq = 0;
+  }
+
+  start() {
+    if (this._timer) return;
+    const interval = Math.max(1, Math.floor(1000 / this.fps));
+    this._timer = setInterval(() => {
+      void this._captureOnce();
+    }, interval);
+    this._timer.unref?.();
+    void this._captureOnce();
+  }
+
+  stop() {
+    if (this._timer) {
+      clearInterval(this._timer);
+      this._timer = null;
+    }
+  }
+
+  async _captureOnce() {
+    if (this._capturing) return;
+    this._capturing = true;
+    try {
+      if (!this.controller || typeof this.controller.capturePng !== 'function') {
+        throw new Error('Android streaming requires a controller with capturePng().');
+      }
+      const png = await this.controller.capturePng({ deviceId: this.deviceId });
+      if (!png?.length) return;
+      const jpeg = await sharp(png).jpeg({ quality: this.quality }).toBuffer();
+      this.streamHub.handleFrame(this.userId, this.deviceId, {
+        jpeg,
+        platform: 'android',
+        seq: this._seq++ >>> 0,
+        ts: Date.now() >>> 0,
+        flags: 1,
+      });
+    } catch (error) {
+      console.warn('[AndroidStream] frame capture failed', {
+        userId: this.userId,
+        deviceId: this.deviceId,
+        error: String(error?.message || error),
+      });
+    } finally {
+      this._capturing = false;
+    }
+  }
+}
+
+module.exports = {
+  AndroidStream,
+};

package/server/services/streaming/browser-stream.js

@@ -0,0 +1,87 @@
+'use strict';
+
+const fs = require('fs');
+const sharp = require('sharp');
+
+function clampInt(value, fallback, min, max) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return fallback;
+  return Math.min(max, Math.max(min, Math.floor(parsed)));
+}
+
+class BrowserStream {
+  constructor({ userId, deviceId = 'browser', controller, streamHub, fps = 15, quality = 80 }) {
+    this.userId = String(userId || '');
+    this.deviceId = String(deviceId || 'browser');
+    this.controller = controller;
+    this.streamHub = streamHub;
+    this.fps = clampInt(fps, 15, 1, 20);
+    this.quality = clampInt(quality, 80, 30, 95);
+    this._timer = null;
+    this._capturing = false;
+    this._seq = 0;
+  }
+
+  start() {
+    if (this._timer) return;
+    const interval = Math.max(1, Math.floor(1000 / this.fps));
+    this._timer = setInterval(() => {
+      void this._captureOnce();
+    }, interval);
+    this._timer.unref?.();
+    void this._captureOnce();
+  }
+
+  stop() {
+    if (this._timer) {
+      clearInterval(this._timer);
+      this._timer = null;
+    }
+  }
+
+  async _captureOnce() {
+    if (this._capturing) return;
+    this._capturing = true;
+    try {
+      const jpeg = await this._captureJpeg();
+      if (!jpeg?.length) return;
+      this.streamHub.handleFrame(this.userId, this.deviceId, {
+        jpeg,
+        platform: 'browser',
+        seq: this._seq++ >>> 0,
+        ts: Date.now() >>> 0,
+        flags: 1,
+      });
+    } catch (error) {
+      console.warn('[BrowserStream] frame capture failed', {
+        userId: this.userId,
+        deviceId: this.deviceId,
+        error: String(error?.message || error),
+      });
+    } finally {
+      this._capturing = false;
+    }
+  }
+
+  async _captureJpeg() {
+    if (!this.controller) {
+      throw new Error('Browser controller is unavailable.');
+    }
+    if (typeof this.controller.screenshotJpeg === 'function') {
+      return this.controller.screenshotJpeg(this.quality);
+    }
+    if (typeof this.controller.screenshot !== 'function') {
+      throw new Error('Browser streaming requires a screenshot-capable controller.');
+    }
+    const result = await this.controller.screenshot({ fullPage: false });
+    if (!result?.fullPath || !fs.existsSync(result.fullPath)) {
+      throw new Error('Browser screenshot did not produce a readable file.');
+    }
+    const png = fs.readFileSync(result.fullPath);
+    return sharp(png).jpeg({ quality: this.quality }).toBuffer();
+  }
+}
+
+module.exports = {
+  BrowserStream,
+};