neoagent 2.4.1-beta.7 → 2.4.1-beta.8

package/flutter_app/lib/src/backend_client.dart

@@ -615,6 +615,50 @@ class BackendClient {
     return _postEmpty(baseUrl, '/api/browser/close');
   }
 
+  Future<Map<String, dynamic>> startStream(
+    String baseUrl, {
+    required String platform,
+    required String deviceId,
+    int fps = 15,
+    int quality = 80,
+    String? displayId,
+  }) async {
+    return postMap(baseUrl, '/api/stream/start', <String, dynamic>{
+      'platform': platform,
+      'deviceId': deviceId,
+      'fps': fps,
+      'quality': quality,
+      if (displayId != null && displayId.isNotEmpty) 'displayId': displayId,
+    });
+  }
+
+  Future<Map<String, dynamic>> stopStream(
+    String baseUrl, {
+    required String platform,
+    required String deviceId,
+  }) async {
+    return postMap(baseUrl, '/api/stream/stop', <String, dynamic>{
+      'platform': platform,
+      'deviceId': deviceId,
+    });
+  }
+
+  Future<Map<String, dynamic>> fetchStreamStatus(
+    String baseUrl, {
+    String? platform,
+    String? deviceId,
+  }) async {
+    final query = <String>[];
+    if (platform != null && platform.isNotEmpty) {
+      query.add('platform=${Uri.encodeQueryComponent(platform)}');
+    }
+    if (deviceId != null && deviceId.isNotEmpty) {
+      query.add('deviceId=${Uri.encodeQueryComponent(deviceId)}');
+    }
+    final suffix = query.isEmpty ? '' : '?${query.join('&')}';
+    return getMap(baseUrl, '/api/stream/status$suffix');
+  }
+
   Future<Map<String, dynamic>> fetchAndroidStatus(String baseUrl) async {
     return getMap(baseUrl, '/api/android/status');
   }

package/flutter_app/lib/src/desktop_companion_actions.dart

@@ -177,6 +177,18 @@ class DesktopCompanionActions {
     };
   }
 
+  Future<Uint8List> compressToJpeg(
+    DesktopCompanionSnapshot snapshot,
+    int quality,
+  ) async {
+    final raw = _decodeScreenshotBytes(snapshot.screenshotBase64);
+    if (_looksLikeJpeg(raw)) return raw;
+    final decoded = img.decodeImage(raw);
+    if (decoded == null) return raw;
+    final normalizedQuality = quality.clamp(30, 95);
+    return Uint8List.fromList(img.encodeJpg(decoded, quality: normalizedQuality));
+  }
+
   Future<Map<String, Object?>> observe({
     bool includeTree = false,
     String? activeDisplayId,
@@ -450,15 +462,15 @@ class DesktopCompanionActions {
     await stdoutSub.cancel();
     await stderrSub.cancel();
 
-    String _trim(StringBuffer buf) {
+    String trimOutput(StringBuffer buf) {
       final s = buf.toString().trim();
       return s.length > maxChars ? '${s.substring(0, maxChars)}\n...[truncated, ${s.length} total chars]' : s;
     }
 
     return <String, Object?>{
       'exitCode': exitCode,
-      'stdout': _trim(stdoutBuf),
-      'stderr': _trim(stderrBuf),
+      'stdout': trimOutput(stdoutBuf),
+      'stderr': trimOutput(stderrBuf),
       'timedOut': timedOut,
       'killed': timedOut,
       'durationMs': DateTime.now().difference(startedAt).inMilliseconds,
@@ -588,6 +600,23 @@ class DesktopCompanionActions {
     }
   }
 
+  Uint8List _decodeScreenshotBytes(String screenshotBase64) {
+    final trimmed = screenshotBase64.trim();
+    final commaIndex = trimmed.indexOf(',');
+    final encoded = trimmed.startsWith('data:image/') && commaIndex >= 0
+        ? trimmed.substring(commaIndex + 1)
+        : trimmed;
+    return Uint8List.fromList(base64Decode(encoded));
+  }
+
+  bool _looksLikeJpeg(Uint8List bytes) {
+    return bytes.length >= 4 &&
+        bytes[0] == 0xff &&
+        bytes[1] == 0xd8 &&
+        bytes[bytes.length - 2] == 0xff &&
+        bytes[bytes.length - 1] == 0xd9;
+  }
+
   String _normalizeMouseButton(String button) {
     final value = button.trim().toLowerCase();
     if (value == 'left' || value == 'right' || value == 'middle') {

package/flutter_app/lib/src/desktop_companion_io.dart

@@ -25,6 +25,10 @@ class DesktopCompanionManager extends ChangeNotifier {
   final DesktopCompanionActions _actions;
   WebSocket? _socket;
   Timer? _reconnectTimer;
+  Timer? _streamTimer;
+  bool _streamCaptureInFlight = false;
+  int _frameSeq = 0;
+  int _streamGeneration = 0;
 
   String _backendUrl = '';
   String _sessionCookie = '';
@@ -126,6 +130,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   Future<void> disconnect() async {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _stopStreaming();
     _connecting = false;
     _connected = false;
     final socket = _socket;
@@ -327,6 +332,10 @@ class DesktopCompanionManager extends ChangeNotifier {
         );
       case 'captureFrame':
         return _actions.captureFrame(activeDisplayId: _activeDisplayId);
+      case 'startStream':
+        return _startStreaming(payload);
+      case 'stopStream':
+        return _stopStreaming();
       case 'observe':
         return _actions.observe(
           includeTree: payload['includeTree'] == true,
@@ -404,6 +413,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   }
 
   void _handleSocketClosed() {
+    _stopStreaming();
     _socket = null;
     _connecting = false;
     _connected = false;
@@ -415,6 +425,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   void dispose() {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _stopStreaming();
     _connecting = false;
     _connected = false;
     _enabled = false;
@@ -448,6 +459,78 @@ class DesktopCompanionManager extends ChangeNotifier {
     );
   }
 
+  Future<Map<String, Object?>> _startStreaming(
+    Map<String, Object?> payload,
+  ) async {
+    _streamTimer?.cancel();
+    final generation = ++_streamGeneration;
+    final fps = ((payload['fps'] as num?)?.round() ?? 15).clamp(1, 20);
+    final quality = ((payload['quality'] as num?)?.round() ?? 80).clamp(30, 95);
+    final displayId = payload['displayId']?.toString().trim();
+    if (displayId != null && displayId.isNotEmpty) {
+      _activeDisplayId = displayId;
+    }
+    final interval = Duration(milliseconds: max(1, (1000 / fps).floor()));
+    _frameSeq = 0;
+    _streamTimer = Timer.periodic(interval, (_) {
+      unawaited(_captureAndSendBinaryFrame(quality, generation));
+    });
+    unawaited(_captureAndSendBinaryFrame(quality, generation));
+    return <String, Object?>{
+      'success': true,
+      'fps': fps,
+      'quality': quality,
+      'displayId': _activeDisplayId,
+    };
+  }
+
+  Map<String, Object?> _stopStreaming() {
+    _streamTimer?.cancel();
+    _streamTimer = null;
+    _streamGeneration++;
+    _streamCaptureInFlight = false;
+    return <String, Object?>{'success': true};
+  }
+
+  Future<void> _captureAndSendBinaryFrame(int quality, int generation) async {
+    final socket = _socket;
+    if (socket == null ||
+        !_connected ||
+        _streamCaptureInFlight ||
+        generation != _streamGeneration) {
+      return;
+    }
+    _streamCaptureInFlight = true;
+    try {
+      final snapshot = await _actions.captureSnapshot(
+        activeDisplayId: _activeDisplayId,
+      );
+      if (snapshot == null) return;
+      final jpeg = await _actions.compressToJpeg(snapshot, quality);
+      if (jpeg.isEmpty) return;
+      if (!_connected || generation != _streamGeneration || _socket != socket) {
+        return;
+      }
+      final frame = Uint8List(10 + jpeg.length);
+      final header = ByteData.sublistView(frame, 0, 10);
+      header.setUint8(0, 0x01);
+      header.setUint32(1, _frameSeq++ & 0xffffffff, Endian.big);
+      header.setUint32(
+        5,
+        DateTime.now().millisecondsSinceEpoch & 0xffffffff,
+        Endian.big,
+      );
+      header.setUint8(9, 0x01);
+      frame.setRange(10, frame.length, jpeg);
+      socket.add(frame);
+    } catch (error) {
+      _errorMessage = 'Desktop stream capture failed: $error';
+      notifyListeners();
+    } finally {
+      _streamCaptureInFlight = false;
+    }
+  }
+
   Future<void> _openMacPermissionSettings(String key) async {
     final uri = switch (key) {
       'screencapture' =>

package/flutter_app/lib/src/stream_renderer.dart

@@ -0,0 +1,155 @@
+import 'dart:typed_data';
+
+import 'package:flutter/material.dart';
+import 'package:socket_io_client/socket_io_client.dart' as io;
+
+class StreamRenderer extends StatefulWidget {
+  const StreamRenderer({
+    super.key,
+    required this.socket,
+    required this.deviceId,
+    required this.platform,
+    required this.remoteResolution,
+    this.onTap,
+    this.onType,
+    this.fit = BoxFit.contain,
+  });
+
+  final io.Socket socket;
+  final String deviceId;
+  final String platform;
+  final Size remoteResolution;
+  final void Function(double x, double y)? onTap;
+  final void Function(String text)? onType;
+  final BoxFit fit;
+
+  @override
+  State<StreamRenderer> createState() => _StreamRendererState();
+}
+
+class _StreamRendererState extends State<StreamRenderer> {
+  Uint8List? _frame;
+
+  @override
+  void initState() {
+    super.initState();
+    widget.socket.on('stream:frame', _onFrame);
+    widget.socket.emit('stream:subscribe', <String, Object?>{
+      'deviceId': widget.deviceId,
+      'platform': widget.platform,
+    });
+  }
+
+  @override
+  void didUpdateWidget(StreamRenderer oldWidget) {
+    super.didUpdateWidget(oldWidget);
+    if (oldWidget.socket == widget.socket &&
+        oldWidget.deviceId == widget.deviceId &&
+        oldWidget.platform == widget.platform) {
+      return;
+    }
+    oldWidget.socket.off('stream:frame', _onFrame);
+    oldWidget.socket.emit('stream:unsubscribe', <String, Object?>{
+      'deviceId': oldWidget.deviceId,
+      'platform': oldWidget.platform,
+    });
+    widget.socket.on('stream:frame', _onFrame);
+    widget.socket.emit('stream:subscribe', <String, Object?>{
+      'deviceId': widget.deviceId,
+      'platform': widget.platform,
+    });
+  }
+
+  void _onFrame(dynamic data) {
+    Object? meta;
+    Object? bytes;
+    if (data is List && data.length >= 2) {
+      meta = data[0];
+      bytes = data[1];
+    }
+    if (meta is Map) {
+      final frameDeviceId = meta['deviceId']?.toString() ?? '';
+      final framePlatform = meta['platform']?.toString() ?? '';
+      if (frameDeviceId.isNotEmpty && frameDeviceId != widget.deviceId) {
+        return;
+      }
+      if (framePlatform.isNotEmpty && framePlatform != widget.platform) {
+        return;
+      }
+    }
+    final frame = switch (bytes) {
+      Uint8List value => value,
+      List<int> value => Uint8List.fromList(value),
+      ByteBuffer value => Uint8List.view(value),
+      _ => null,
+    };
+    if (frame == null || frame.isEmpty || !mounted) return;
+    setState(() => _frame = frame);
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    final frame = _frame;
+    if (frame == null) {
+      return const Center(child: CircularProgressIndicator());
+    }
+    return LayoutBuilder(
+      builder: (context, constraints) {
+        return GestureDetector(
+          behavior: HitTestBehavior.opaque,
+          onTapDown: widget.onTap == null
+              ? null
+              : (details) => _handleTap(details, constraints.biggest),
+          child: Image.memory(
+            frame,
+            gaplessPlayback: true,
+            fit: widget.fit,
+            width: constraints.maxWidth,
+            height: constraints.maxHeight,
+          ),
+        );
+      },
+    );
+  }
+
+  void _handleTap(TapDownDetails details, Size boxSize) {
+    final remote = widget.remoteResolution;
+    if (remote.width <= 0 || remote.height <= 0 || boxSize.width <= 0 || boxSize.height <= 0) {
+      return;
+    }
+    final imageAspect = remote.width / remote.height;
+    final boxAspect = boxSize.width / boxSize.height;
+    double renderWidth;
+    double renderHeight;
+    double offsetX = 0;
+    double offsetY = 0;
+    if (boxAspect > imageAspect) {
+      renderHeight = boxSize.height;
+      renderWidth = renderHeight * imageAspect;
+      offsetX = (boxSize.width - renderWidth) / 2;
+    } else {
+      renderWidth = boxSize.width;
+      renderHeight = renderWidth / imageAspect;
+      offsetY = (boxSize.height - renderHeight) / 2;
+    }
+    final localX = details.localPosition.dx - offsetX;
+    final localY = details.localPosition.dy - offsetY;
+    if (localX < 0 || localY < 0 || localX > renderWidth || localY > renderHeight) {
+      return;
+    }
+    widget.onTap?.call(
+      localX * remote.width / renderWidth,
+      localY * remote.height / renderHeight,
+    );
+  }
+
+  @override
+  void dispose() {
+    widget.socket.emit('stream:unsubscribe', <String, Object?>{
+      'deviceId': widget.deviceId,
+      'platform': widget.platform,
+    });
+    widget.socket.off('stream:frame', _onFrame);
+    super.dispose();
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "neoagent",
-  "version": "2.4.1-beta.7",
+  "version": "2.4.1-beta.8",
   "description": "Proactive personal AI agent with no limits",
   "license": "AGPL-3.0-only",
   "main": "server/index.js",

package/server/guest_agent.js

@@ -195,6 +195,13 @@ function requireCapability(controller, name) {
 app.post('/browser/launch', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').launch(req.body || {})));
 app.post('/browser/navigate', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').navigate(req.body?.url, req.body || {})));
 app.post('/browser/screenshot', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').screenshot(req.body || {})));
+app.post('/browser/screenshot-jpeg', async (req, res) => handle(res, async () => {
+  const jpeg = await requireCapability(browserController, 'browser').screenshotJpeg(req.body?.quality, req.body || {});
+  return {
+    contentType: 'image/jpeg',
+    contentBase64: Buffer.from(jpeg).toString('base64'),
+  };
+}));
 app.post('/browser/click', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').click(req.body?.selector, req.body?.text, req.body?.screenshot !== false)));
 app.post('/browser/click-point', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').clickPoint(req.body?.x, req.body?.y, req.body?.screenshot !== false)));
 app.post('/browser/fill', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').type(req.body?.selector, String(req.body?.value ?? req.body?.text ?? ''), req.body || {})));

package/server/http/routes.js

@@ -25,6 +25,7 @@ const routeRegistry = [
   { basePath: '/api/browser-extension', modulePath: '../routes/browser_extension' },
   { basePath: '/api/android', modulePath: '../routes/android' },
   { basePath: '/api/desktop', modulePath: '../routes/desktop' },
+  { basePath: '/api/stream', modulePath: '../routes/stream' },
   { basePath: '/api/recordings', modulePath: '../routes/recordings' },
   { basePath: '/api/social-video', modulePath: '../routes/social_video' },
   { basePath: '/api/voice-assistant', modulePath: '../routes/voice_assistant' },

package/server/http/socket.js

@@ -8,7 +8,7 @@ function createSocketServer(httpServer, { validateOrigin }) {
     pingInterval: Number(process.env.NEOAGENT_SOCKET_PING_INTERVAL_MS || 25000),
     pingTimeout: Number(process.env.NEOAGENT_SOCKET_PING_TIMEOUT_MS || 20000),
     connectTimeout: Number(process.env.NEOAGENT_SOCKET_CONNECT_TIMEOUT_MS || 10000),
-    maxHttpBufferSize: Number(process.env.NEOAGENT_SOCKET_MAX_HTTP_BUFFER_BYTES || 1_000_000),
+    maxHttpBufferSize: Number(process.env.NEOAGENT_SOCKET_MAX_HTTP_BUFFER_BYTES || 8 * 1024 * 1024),
     cors: {
       origin(origin, callback) {
         return validateOrigin(origin, callback, { allowMissingOrigin: true });

package/server/index.js

@@ -34,6 +34,7 @@ const { startServices, stopServices } = require('./services/manager');
 const { bindBrowserExtensionGateway } = require('./services/browser/extension/gateway');
 const { bindDesktopCompanionGateway } = require('./services/desktop/gateway');
 const { bindWearableGateway } = require('./services/wearable/gateway');
+const { StreamHub } = require('./services/streaming/stream-hub');
 
 function parseBooleanFlag(value, fallback = false) {
   const normalized = String(value || '').trim().toLowerCase();
@@ -89,6 +90,8 @@ const app = express();
 app.disable('x-powered-by');
 const httpServer = createServer(app);
 const io = createSocketServer(httpServer, { validateOrigin });
+const streamHub = new StreamHub(io);
+app.locals.streamHub = streamHub;
 app.locals.httpRuntimeConfig = {
   secureCookies: SECURE_COOKIES,
   trustProxy: TRUST_PROXY,
@@ -112,7 +115,7 @@ registerApiRoutes(app);
 registerStaticRoutes(app);
 registerErrorHandler(app);
 bindBrowserExtensionGateway(httpServer, app);
-bindDesktopCompanionGateway(httpServer, app, sessionMiddleware);
+bindDesktopCompanionGateway(httpServer, app, sessionMiddleware, streamHub);
 bindWearableGateway(httpServer, app, sessionMiddleware);
 
 let shuttingDown = false;

package/server/public/.last_build_id

@@ -1 +1 @@
-547ecdc48f374b4b525ad319e60c921b
+f09d02f54b922317d6897b53a29a5868

package/server/public/flutter_bootstrap.js

@@ -37,6 +37,6 @@ _flutter.buildConfig = {"engineRevision":"4c525dac5ebe5971c5708ef73558ed8edcf4a3
 
 _flutter.loader.load({
   serviceWorkerSettings: {
-    serviceWorkerVersion: "730484345" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
+    serviceWorkerVersion: "1854193544" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
   }
 });

package/server/public/main.dart.js

@@ -131149,7 +131149,7 @@ r===$&&A.b()
 o.push(A.iq(p,A.j2(!1,new A.a4(B.u0,A.e_(new A.cN(B.hf,new A.a6E(r,p),p),p,p),p),!1,B.I,!0),p,p,0,0,0,p))}r=!1
 if(!s.ay)if(!s.ch){r=s.e
 r===$&&A.b()
-r=B.b.v("mpeldus3-e9a9387").length!==0&&r.b}if(r){r=s.d
+r=B.b.v("mpem9vkq-e415031").length!==0&&r.b}if(r){r=s.d
 r===$&&A.b()
 r=r.ab&&!r.Z?84:0
 q=s.e
@@ -136201,7 +136201,7 @@ $S:338}
 A.Z6.prototype={}
 A.S_.prototype={
 n4(a){var s=this
-if(B.b.v("mpeldus3-e9a9387").length===0||s.a!=null)return
+if(B.b.v("mpem9vkq-e415031").length===0||s.a!=null)return
 s.Aq()
 s.a=A.q9(B.Qi,new A.b7t(s))},
 Aq(){var s=0,r=A.l(t.H),q,p=2,o=[],n=this,m,l,k,j,i,h,g,f
@@ -136219,7 +136219,7 @@ if(!t.f.b(k)){s=1
 break}i=J.X(k,"buildId")
 h=i==null?null:B.b.v(J.q(i))
 j=h==null?"":h
-if(J.bt(j)===0||J.c(j,"mpeldus3-e9a9387")){s=1
+if(J.bt(j)===0||J.c(j,"mpem9vkq-e415031")){s=1
 break}n.b=!0
 n.F()
 p=2
@@ -136236,7 +136236,7 @@ case 2:return A.i(o.at(-1),r)}})
 return A.k($async$Aq,r)},
 vl(){var s=0,r=A.l(t.H),q,p=2,o=[],n=this,m,l,k,j,i,h,g,f,e,d,c,b,a,a0,a1
 var $async$vl=A.h(function(a2,a3){if(a2===1){o.push(a3)
-s=p}for(;;)switch(s){case 0:if(B.b.v("mpeldus3-e9a9387").length===0||n.c){s=1
+s=p}for(;;)switch(s){case 0:if(B.b.v("mpem9vkq-e415031").length===0||n.c){s=1
 break}n.c=!0
 n.F()
 p=4