neoagent 2.4.1-beta.7 → 2.4.1-beta.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/extensions/chrome-browser/background.mjs +9 -3
  2. package/extensions/chrome-browser/manifest.json +2 -1
  3. package/extensions/chrome-browser/popup.js +2 -2
  4. package/flutter_app/lib/main.dart +1 -0
  5. package/flutter_app/lib/main_controller.dart +141 -4
  6. package/flutter_app/lib/main_devices.dart +416 -119
  7. package/flutter_app/lib/main_settings.dart +89 -14
  8. package/flutter_app/lib/src/backend_client.dart +77 -0
  9. package/flutter_app/lib/src/desktop_companion_actions.dart +58 -3
  10. package/flutter_app/lib/src/desktop_companion_io.dart +89 -0
  11. package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
  12. package/flutter_app/lib/src/stream_renderer.dart +286 -0
  13. package/flutter_app/macos/Runner/AppDelegate.swift +44 -0
  14. package/package.json +1 -1
  15. package/server/guest_agent.js +7 -0
  16. package/server/http/routes.js +1 -0
  17. package/server/http/socket.js +1 -1
  18. package/server/index.js +4 -1
  19. package/server/public/.last_build_id +1 -1
  20. package/server/public/flutter_bootstrap.js +1 -1
  21. package/server/public/main.dart.js +67339 -66716
  22. package/server/routes/browser.js +14 -0
  23. package/server/routes/browser_extension.js +21 -4
  24. package/server/routes/desktop.js +10 -0
  25. package/server/routes/settings.js +4 -0
  26. package/server/routes/stream.js +187 -0
  27. package/server/services/ai/tools.js +2 -2
  28. package/server/services/android/controller.js +41 -2
  29. package/server/services/browser/controller.js +34 -0
  30. package/server/services/browser/extension/manifest.js +33 -0
  31. package/server/services/browser/extension/provider.js +12 -6
  32. package/server/services/browser/extension/registry.js +144 -18
  33. package/server/services/desktop/gateway.js +28 -3
  34. package/server/services/desktop/protocol.js +33 -0
  35. package/server/services/desktop/provider.js +14 -0
  36. package/server/services/desktop/registry.js +41 -0
  37. package/server/services/manager.js +19 -2
  38. package/server/services/runtime/backends/local-vm.js +6 -0
  39. package/server/services/runtime/manager.js +7 -3
  40. package/server/services/runtime/settings.js +17 -0
  41. package/server/services/streaming/android-stream.js +83 -0
  42. package/server/services/streaming/browser-stream.js +87 -0
  43. package/server/services/streaming/stream-hub.js +231 -0
  44. package/server/services/websocket.js +73 -0
@@ -90,6 +90,29 @@ class _DevicesPanelState extends State<DevicesPanel> {
90
90
  bool get _androidStarting =>
91
91
  widget.controller.androidRuntime['starting'] == true;
92
92
 
93
+ String? get _androidDeviceId {
94
+ final status = widget.controller.androidRuntime;
95
+ final direct = status['adbSerial']?.toString().trim();
96
+ if (direct != null && direct.isNotEmpty) {
97
+ return direct;
98
+ }
99
+ final devices = _jsonMapList(status['devices'], fallbackToMapValues: true);
100
+ for (final device in devices) {
101
+ if (device['status']?.toString() != 'device') {
102
+ continue;
103
+ }
104
+ final serial = device['serial']?.toString().trim();
105
+ if (serial != null && serial.isNotEmpty) {
106
+ return serial;
107
+ }
108
+ final id = device['deviceId']?.toString().trim();
109
+ if (id != null && id.isNotEmpty) {
110
+ return id;
111
+ }
112
+ }
113
+ return null;
114
+ }
115
+
93
116
  List<Map<String, dynamic>> get _onlineDesktopDevices => widget
94
117
  .controller
95
118
  .desktopDevices
@@ -107,6 +130,9 @@ class _DevicesPanelState extends State<DevicesPanel> {
107
130
  widget.controller.browserBackend == 'extension' &&
108
131
  !widget.controller.browserExtensionConnected;
109
132
 
133
+ List<Map<String, dynamic>> get _browserExtensionDevices =>
134
+ widget.controller.browserExtensionTokens;
135
+
110
136
  String? get _activeScreenshotPath {
111
137
  if (_isBrowser) {
112
138
  if (_extensionPreferredButOffline) return null;
@@ -296,6 +322,23 @@ class _DevicesPanelState extends State<DevicesPanel> {
296
322
  }
297
323
  });
298
324
 
325
+ void _handleHover(Offset point) {
326
+ if (_isBrowser) {
327
+ unawaited(widget.controller.hoverBrowserPointRuntime(
328
+ x: point.dx.round(),
329
+ y: point.dy.round(),
330
+ ));
331
+ } else if (_isDesktop) {
332
+ if (_desktopRequiresSelection) {
333
+ return;
334
+ }
335
+ unawaited(widget.controller.hoverDesktopRuntime(
336
+ x: point.dx.round(),
337
+ y: point.dy.round(),
338
+ ));
339
+ }
340
+ }
341
+
299
342
  Future<void> _handleTap(Offset point) => _runOnSurface(() async {
300
343
  if (_isBrowser) {
301
344
  await widget.controller.clickBrowserPointRuntime(
@@ -324,36 +367,37 @@ class _DevicesPanelState extends State<DevicesPanel> {
324
367
  });
325
368
  });
326
369
 
327
- Future<void> _handleSwipe(Offset start, Offset end) => _runOnSurface(() async {
328
- if (_isBrowser) {
329
- await widget.controller.scrollBrowserRuntime(
330
- deltaY: (start.dy - end.dy).round(),
331
- );
332
- return;
333
- }
334
- if (_isDesktop) {
335
- if (_desktopRequiresSelection) {
336
- return;
337
- }
338
- await widget.controller.dragDesktopRuntime(
339
- x1: start.dx.round(),
340
- y1: start.dy.round(),
341
- x2: end.dx.round(),
342
- y2: end.dy.round(),
343
- );
344
- return;
345
- }
346
- if (!_androidOnline) {
347
- return;
348
- }
349
- await widget.controller.swipeAndroidRuntime(<String, dynamic>{
350
- 'x1': start.dx.round(),
351
- 'y1': start.dy.round(),
352
- 'x2': end.dx.round(),
353
- 'y2': end.dy.round(),
354
- 'durationMs': 280,
355
- });
356
- });
370
+ Future<void> _handleSwipe(Offset start, Offset end) =>
371
+ _runOnSurface(() async {
372
+ if (_isBrowser) {
373
+ await widget.controller.scrollBrowserRuntime(
374
+ deltaY: (start.dy - end.dy).round(),
375
+ );
376
+ return;
377
+ }
378
+ if (_isDesktop) {
379
+ if (_desktopRequiresSelection) {
380
+ return;
381
+ }
382
+ await widget.controller.dragDesktopRuntime(
383
+ x1: start.dx.round(),
384
+ y1: start.dy.round(),
385
+ x2: end.dx.round(),
386
+ y2: end.dy.round(),
387
+ );
388
+ return;
389
+ }
390
+ if (!_androidOnline) {
391
+ return;
392
+ }
393
+ await widget.controller.swipeAndroidRuntime(<String, dynamic>{
394
+ 'x1': start.dx.round(),
395
+ 'y1': start.dy.round(),
396
+ 'x2': end.dx.round(),
397
+ 'y2': end.dy.round(),
398
+ 'durationMs': 280,
399
+ });
400
+ });
357
401
 
358
402
  Future<void> _runQuickAction(String action) => _runOnSurface(() async {
359
403
  final controller = widget.controller;
@@ -398,6 +442,13 @@ class _DevicesPanelState extends State<DevicesPanel> {
398
442
  final prefersExtension = controller.browserBackend == 'extension';
399
443
  final extensionConnected = controller.browserExtensionConnected;
400
444
  final usingExtension = prefersExtension && extensionConnected;
445
+ final selectedBrowserExtension = controller.browserExtensionTokens
446
+ .where(
447
+ (device) =>
448
+ device['tokenId'] == controller.selectedBrowserExtensionTokenId,
449
+ )
450
+ .cast<Map<String, dynamic>?>()
451
+ .firstWhere((device) => device != null, orElse: () => null);
401
452
  final browserFallbackLabel = 'cloud browser runtime';
402
453
  final browserPageInfo = browserStatus['pageInfo'] is Map<dynamic, dynamic>
403
454
  ? Map<String, dynamic>.from(browserStatus['pageInfo'] as Map)
@@ -454,12 +505,63 @@ class _DevicesPanelState extends State<DevicesPanel> {
454
505
  desktopDevices: controller.desktopDevices,
455
506
  selectedDesktopDeviceId:
456
507
  controller.selectedDesktopDeviceId,
508
+ browserExtensionDevices:
509
+ controller.browserExtensionTokens,
510
+ selectedBrowserExtensionTokenId:
511
+ controller.selectedBrowserExtensionTokenId,
457
512
  browserExtensionPreferred: prefersExtension,
458
513
  browserExtensionActive: usingExtension,
459
514
  browserFallbackLabel: browserFallbackLabel,
460
515
  ),
461
516
  if (_isBrowser && prefersExtension) ...<Widget>[
462
517
  const SizedBox(height: 14),
518
+ if (_browserExtensionDevices.isNotEmpty) ...<Widget>[
519
+ DropdownButtonFormField<String>(
520
+ isExpanded: true,
521
+ initialValue: selectedBrowserExtension?['tokenId']
522
+ ?.toString(),
523
+ decoration: const InputDecoration(
524
+ labelText: 'Chrome extension device',
525
+ prefixIcon: Icon(Icons.extension_outlined),
526
+ ),
527
+ hint: const Text('Select a paired extension'),
528
+ items: _browserExtensionDevices.map((device) {
529
+ final tokenId = device['tokenId']?.toString() ?? '';
530
+ final label =
531
+ device['name']?.toString().trim().isNotEmpty ==
532
+ true
533
+ ? device['name'].toString()
534
+ : tokenId;
535
+ final state =
536
+ device['online'] == true ||
537
+ device['connected'] == true
538
+ ? 'online'
539
+ : 'offline';
540
+ return DropdownMenuItem<String>(
541
+ value: tokenId,
542
+ child: Text(
543
+ '$label · $state',
544
+ maxLines: 1,
545
+ overflow: TextOverflow.ellipsis,
546
+ softWrap: false,
547
+ ),
548
+ );
549
+ }).toList(),
550
+ onChanged: _isCurrentSurfaceBusy
551
+ ? null
552
+ : (value) {
553
+ if (value == null || value.isEmpty) {
554
+ return;
555
+ }
556
+ unawaited(
557
+ controller.selectBrowserExtensionRuntime(
558
+ value,
559
+ ),
560
+ );
561
+ },
562
+ ),
563
+ const SizedBox(height: 10),
564
+ ],
463
565
  _ExtensionStatusBar(
464
566
  connected: extensionConnected,
465
567
  onDownload: controller.downloadBrowserExtension,
@@ -570,6 +672,21 @@ class _DevicesPanelState extends State<DevicesPanel> {
570
672
  surface: _surface,
571
673
  controller: controller,
572
674
  screenshotPath: _activeScreenshotPath,
675
+ streamPlatform: _isBrowser && browserStatus['launched'] == true
676
+ ? 'browser'
677
+ : (_isDesktop && _desktopOnline
678
+ ? 'desktop'
679
+ : (!_isBrowser && !_isDesktop && _androidOnline
680
+ ? 'android'
681
+ : null)),
682
+ streamDeviceId: _isBrowser && browserStatus['launched'] == true
683
+ ? 'browser'
684
+ : (_isDesktop && _desktopOnline
685
+ ? (widget.controller.selectedDesktopDeviceId ??
686
+ (_onlineDesktopDevices.isNotEmpty ? _onlineDesktopDevices.first['deviceId']?.toString() : null))
687
+ : (!_isBrowser && !_isDesktop && _androidOnline
688
+ ? _androidDeviceId
689
+ : null)),
573
690
  busy: _isCurrentSurfaceBusy,
574
691
  wakingUp: !_isBrowser && !_isDesktop && _androidStarting,
575
692
  enabled: _isBrowser || _isDesktop || _androidOnline,
@@ -578,6 +695,7 @@ class _DevicesPanelState extends State<DevicesPanel> {
578
695
  : _desktopRequiresSelection,
579
696
  onTapPoint: _handleTap,
580
697
  onSwipe: _handleSwipe,
698
+ onHover: _handleHover,
581
699
  onWakeRequested: _openPrimary,
582
700
  ),
583
701
  if (!_isBrowser && !_isDesktop) ...<Widget>[
@@ -666,6 +784,8 @@ class _DeviceSurfaceHeader extends StatelessWidget {
666
784
  required this.desktopRuntime,
667
785
  required this.desktopDevices,
668
786
  required this.selectedDesktopDeviceId,
787
+ required this.browserExtensionDevices,
788
+ required this.selectedBrowserExtensionTokenId,
669
789
  required this.browserExtensionPreferred,
670
790
  required this.browserExtensionActive,
671
791
  required this.browserFallbackLabel,
@@ -679,6 +799,8 @@ class _DeviceSurfaceHeader extends StatelessWidget {
679
799
  final Map<String, dynamic> desktopRuntime;
680
800
  final List<Map<String, dynamic>> desktopDevices;
681
801
  final String? selectedDesktopDeviceId;
802
+ final List<Map<String, dynamic>> browserExtensionDevices;
803
+ final String? selectedBrowserExtensionTokenId;
682
804
  final bool browserExtensionPreferred;
683
805
  final bool browserExtensionActive;
684
806
  final String browserFallbackLabel;
@@ -694,12 +816,29 @@ class _DeviceSurfaceHeader extends StatelessWidget {
694
816
  .where((device) => device['deviceId'] == selectedDesktopDeviceId)
695
817
  .cast<Map<String, dynamic>?>()
696
818
  .firstWhere((device) => device != null, orElse: () => null);
819
+ final selectedExtension = browserExtensionDevices
820
+ .where(
821
+ (device) => device['tokenId'] == selectedBrowserExtensionTokenId,
822
+ )
823
+ .cast<Map<String, dynamic>?>()
824
+ .firstWhere((device) => device != null, orElse: () => null);
825
+ final extensionOnlineCount = browserExtensionDevices
826
+ .where(
827
+ (device) =>
828
+ device['online'] == true || device['connected'] == true,
829
+ )
830
+ .length;
697
831
  final desktopOnlineCount = desktopDevices
698
832
  .where((device) => device['online'] == true)
699
833
  .length;
700
834
  final title = switch (surface) {
701
835
  _DeviceSurface.browser =>
702
- (browserPageInfo['title']?.toString().trim().isNotEmpty ?? false)
836
+ browserExtensionPreferred &&
837
+ selectedExtension?['name']?.toString().trim().isNotEmpty ==
838
+ true
839
+ ? selectedExtension!['name'].toString()
840
+ : (browserPageInfo['title']?.toString().trim().isNotEmpty ??
841
+ false)
703
842
  ? browserPageInfo['title'].toString()
704
843
  : 'Live Browser',
705
844
  _DeviceSurface.android => 'Android Phone',
@@ -709,9 +848,13 @@ class _DeviceSurfaceHeader extends StatelessWidget {
709
848
  : 'Desktop Companion',
710
849
  };
711
850
  final subtitle = switch (surface) {
712
- _DeviceSurface.browser =>
713
- browserExtensionPreferred && !browserExtensionActive
714
- ? 'No extension device is active. Using the $browserFallbackLabel.'
851
+ _DeviceSurface.browser =>
852
+ browserExtensionPreferred && selectedExtension == null
853
+ ? extensionOnlineCount > 1
854
+ ? 'Multiple extension devices are online. Pick the browser you want to control.'
855
+ : 'No extension device is active. Using the $browserFallbackLabel.'
856
+ : browserExtensionPreferred && !browserExtensionActive
857
+ ? 'Selected extension is offline. Using the $browserFallbackLabel.'
715
858
  : (browserPageInfo['url']?.toString() ??
716
859
  'Ready for navigation'),
717
860
  _DeviceSurface.android =>
@@ -746,7 +889,9 @@ class _DeviceSurfaceHeader extends StatelessWidget {
746
889
  : '${selectedDesktop['platform'] ?? 'desktop'} · ${selectedDesktop['hostname'] ?? 'unknown host'}',
747
890
  };
748
891
  final statusLabel = surface == _DeviceSurface.browser
749
- ? browserExtensionPreferred && !browserExtensionActive
892
+ ? browserExtensionPreferred && selectedExtension == null
893
+ ? (extensionOnlineCount > 0 ? 'Select Device' : 'Fallback')
894
+ : browserExtensionPreferred && !browserExtensionActive
750
895
  ? 'Fallback'
751
896
  : browserExtensionActive
752
897
  ? 'Extension'
@@ -1120,10 +1265,7 @@ class _AndroidNavDock extends StatelessWidget {
1120
1265
  }
1121
1266
 
1122
1267
  class _SurfaceSwitcher extends StatelessWidget {
1123
- const _SurfaceSwitcher({
1124
- required this.surface,
1125
- required this.onSelect,
1126
- });
1268
+ const _SurfaceSwitcher({required this.surface, required this.onSelect});
1127
1269
 
1128
1270
  final _DeviceSurface surface;
1129
1271
  final Future<void> Function(_DeviceSurface) onSelect;
@@ -1157,8 +1299,7 @@ class _SurfaceSwitcher extends StatelessWidget {
1157
1299
  ),
1158
1300
  labelStyle: TextStyle(
1159
1301
  color: selected ? _textPrimary : _textSecondary,
1160
- fontWeight:
1161
- selected ? FontWeight.w700 : FontWeight.w500,
1302
+ fontWeight: selected ? FontWeight.w700 : FontWeight.w500,
1162
1303
  ),
1163
1304
  );
1164
1305
  }).toList(),
@@ -1179,24 +1320,30 @@ class _InteractiveSurfacePreview extends StatefulWidget {
1179
1320
  required this.surface,
1180
1321
  required this.controller,
1181
1322
  required this.screenshotPath,
1323
+ required this.streamPlatform,
1324
+ required this.streamDeviceId,
1182
1325
  required this.busy,
1183
1326
  required this.wakingUp,
1184
1327
  required this.enabled,
1185
1328
  required this.connectRequired,
1186
1329
  required this.onTapPoint,
1187
1330
  required this.onSwipe,
1331
+ this.onHover,
1188
1332
  required this.onWakeRequested,
1189
1333
  });
1190
1334
 
1191
1335
  final _DeviceSurface surface;
1192
1336
  final NeoAgentController controller;
1193
1337
  final String? screenshotPath;
1338
+ final String? streamPlatform;
1339
+ final String? streamDeviceId;
1194
1340
  final bool busy;
1195
1341
  final bool wakingUp;
1196
1342
  final bool enabled;
1197
1343
  final bool connectRequired;
1198
1344
  final Future<void> Function(Offset point) onTapPoint;
1199
1345
  final Future<void> Function(Offset start, Offset end) onSwipe;
1346
+ final void Function(Offset point)? onHover;
1200
1347
  final Future<void> Function() onWakeRequested;
1201
1348
 
1202
1349
  @override
@@ -1213,11 +1360,15 @@ class _InteractiveSurfacePreviewState
1213
1360
  Object? _imageError;
1214
1361
  Offset? _dragStart;
1215
1362
  Offset? _dragEnd;
1363
+ bool _streamStarting = false;
1364
+ String? _activeStreamKey;
1365
+ String? _streamFailedKey;
1216
1366
 
1217
1367
  @override
1218
1368
  void initState() {
1219
1369
  super.initState();
1220
1370
  unawaited(_loadImage());
1371
+ unawaited(_syncStream());
1221
1372
  }
1222
1373
 
1223
1374
  @override
@@ -1226,14 +1377,95 @@ class _InteractiveSurfacePreviewState
1226
1377
  if (oldWidget.screenshotPath != widget.screenshotPath) {
1227
1378
  unawaited(_loadImage());
1228
1379
  }
1380
+ if (oldWidget.streamPlatform != widget.streamPlatform ||
1381
+ oldWidget.streamDeviceId != widget.streamDeviceId ||
1382
+ oldWidget.controller.streamSocket != widget.controller.streamSocket) {
1383
+ unawaited(_syncStream());
1384
+ }
1229
1385
  }
1230
1386
 
1231
1387
  @override
1232
1388
  void dispose() {
1389
+ unawaited(_stopActiveStream());
1233
1390
  _detachImageListener();
1234
1391
  super.dispose();
1235
1392
  }
1236
1393
 
1394
+ String? get _requestedStreamKey {
1395
+ final platform = widget.streamPlatform?.trim();
1396
+ final deviceId = widget.streamDeviceId?.trim();
1397
+ if (platform == null ||
1398
+ platform.isEmpty ||
1399
+ deviceId == null ||
1400
+ deviceId.isEmpty ||
1401
+ widget.controller.streamSocket == null) {
1402
+ return null;
1403
+ }
1404
+ return '$platform:$deviceId';
1405
+ }
1406
+
1407
+ Future<void> _syncStream() async {
1408
+ final requested = _requestedStreamKey;
1409
+ if (_activeStreamKey == requested || _streamStarting) {
1410
+ return;
1411
+ }
1412
+ if (requested != _streamFailedKey) {
1413
+ _streamFailedKey = null;
1414
+ }
1415
+ await _stopActiveStream();
1416
+ if (requested == null) {
1417
+ return;
1418
+ }
1419
+ final parts = requested.split(':');
1420
+ _streamStarting = true;
1421
+ try {
1422
+ await widget.controller.startStreamRuntime(
1423
+ platform: parts[0],
1424
+ deviceId: parts.sublist(1).join(':'),
1425
+ fps: 10,
1426
+ quality: 70,
1427
+ );
1428
+ if (mounted && _requestedStreamKey == requested) {
1429
+ _activeStreamKey = requested;
1430
+ _streamFailedKey = null;
1431
+ } else {
1432
+ await widget.controller.stopStreamRuntime(
1433
+ platform: parts[0],
1434
+ deviceId: parts.sublist(1).join(':'),
1435
+ );
1436
+ }
1437
+ } catch (_) {
1438
+ if (mounted) {
1439
+ _streamFailedKey = requested;
1440
+ }
1441
+ if (mounted && (widget.screenshotPath ?? '').isEmpty) {
1442
+ unawaited(_loadImage());
1443
+ }
1444
+ } finally {
1445
+ _streamStarting = false;
1446
+ if (mounted &&
1447
+ _activeStreamKey != _requestedStreamKey &&
1448
+ _streamFailedKey != _requestedStreamKey) {
1449
+ unawaited(_syncStream());
1450
+ }
1451
+ }
1452
+ }
1453
+
1454
+ Future<void> _stopActiveStream() async {
1455
+ final active = _activeStreamKey;
1456
+ _activeStreamKey = null;
1457
+ if (active == null) {
1458
+ return;
1459
+ }
1460
+ final parts = active.split(':');
1461
+ try {
1462
+ await widget.controller.stopStreamRuntime(
1463
+ platform: parts[0],
1464
+ deviceId: parts.sublist(1).join(':'),
1465
+ );
1466
+ } catch (_) {}
1467
+ }
1468
+
1237
1469
  void _detachImageListener() {
1238
1470
  if (_imageStream != null && _imageListener != null) {
1239
1471
  _imageStream!.removeListener(_imageListener!);
@@ -1368,82 +1600,41 @@ class _InteractiveSurfacePreviewState
1368
1600
  constraints.maxWidth,
1369
1601
  constraints.maxHeight,
1370
1602
  );
1371
- if (!hasImage) {
1372
- return _EmptySurfaceState(
1373
- surface: widget.surface,
1374
- enabled: widget.enabled,
1375
- busy: widget.busy,
1376
- isLoadingPreview: widget.wakingUp,
1377
- connectRequired: widget.connectRequired,
1378
- errorMessage: _imageError?.toString(),
1379
- onPressed: widget.onWakeRequested,
1380
- );
1381
- }
1382
- final imageBytes = _imageBytes;
1383
- if (imageBytes == null) {
1384
- return _EmptySurfaceState(
1385
- surface: widget.surface,
1386
- enabled: widget.enabled,
1387
- busy: widget.busy,
1388
- isLoadingPreview: _imageError == null,
1389
- connectRequired: widget.connectRequired,
1390
- errorMessage: _imageError?.toString(),
1391
- onPressed: widget.onWakeRequested,
1392
- );
1393
- }
1394
- return Semantics(
1395
- button: true,
1396
- label: 'Device surface preview — tap to interact, swipe to scroll',
1397
- child: GestureDetector(
1398
- onTapUp: widget.busy
1399
- ? null
1400
- : (details) async {
1401
- final point = _mapToPixels(
1402
- details.localPosition,
1403
- boxSize,
1404
- );
1405
- if (point != null) {
1406
- await widget.onTapPoint(point);
1407
- }
1408
- },
1409
- onPanStart: widget.busy
1410
- ? null
1411
- : (details) {
1412
- _dragStart = details.localPosition;
1413
- _dragEnd = details.localPosition;
1414
- },
1415
- onPanUpdate: widget.busy
1416
- ? null
1417
- : (details) {
1418
- _dragEnd = details.localPosition;
1419
- },
1420
- onPanEnd: widget.busy
1421
- ? null
1422
- : (_) async {
1423
- final start = _dragStart;
1424
- final end = _dragEnd;
1425
- _dragStart = null;
1426
- _dragEnd = null;
1427
- if (start == null || end == null) {
1428
- return;
1429
- }
1430
- if ((start - end).distance < 12) {
1431
- return;
1432
- }
1433
- final mappedStart = _mapToPixels(start, boxSize);
1434
- final mappedEnd = _mapToPixels(end, boxSize);
1435
- if (mappedStart != null && mappedEnd != null) {
1436
- await widget.onSwipe(mappedStart, mappedEnd);
1437
- }
1438
- },
1439
- child: Stack(
1603
+ final socket = widget.controller.streamSocket;
1604
+ final streamPlatform = widget.streamPlatform;
1605
+ final streamDeviceId = widget.streamDeviceId;
1606
+ final streamKey = _requestedStreamKey;
1607
+ if (socket != null &&
1608
+ streamPlatform != null &&
1609
+ streamPlatform.isNotEmpty &&
1610
+ streamDeviceId != null &&
1611
+ streamDeviceId.isNotEmpty &&
1612
+ streamKey != _streamFailedKey) {
1613
+ return Stack(
1440
1614
  fit: StackFit.expand,
1441
1615
  children: <Widget>[
1442
1616
  Container(color: _bgSecondary),
1443
- Image.memory(
1444
- imageBytes,
1445
- fit: BoxFit.contain,
1446
- gaplessPlayback: true,
1617
+ StreamRenderer(
1618
+ socket: socket,
1619
+ deviceId: streamDeviceId,
1620
+ platform: streamPlatform,
1621
+ remoteResolution: _pixelSize,
1622
+ onTap: widget.busy
1623
+ ? null
1624
+ : (x, y) => unawaited(
1625
+ widget.onTapPoint(Offset(x, y)),
1626
+ ),
1627
+ onSwipe: widget.busy
1628
+ ? null
1629
+ : (x1, y1, x2, y2) => unawaited(
1630
+ widget.onSwipe(
1631
+ Offset(x1, y1),
1632
+ Offset(x2, y2),
1633
+ ),
1634
+ ),
1635
+ onHover: widget.busy
1636
+ ? null
1637
+ : (x, y) => widget.onHover?.call(Offset(x, y)),
1447
1638
  ),
1448
1639
  Positioned(
1449
1640
  left: 12,
@@ -1469,8 +1660,115 @@ class _InteractiveSurfacePreviewState
1469
1660
  if (widget.busy)
1470
1661
  const Center(child: CircularProgressIndicator()),
1471
1662
  ],
1663
+ );
1664
+ }
1665
+ if (!hasImage) {
1666
+ return _EmptySurfaceState(
1667
+ surface: widget.surface,
1668
+ enabled: widget.enabled,
1669
+ busy: widget.busy,
1670
+ isLoadingPreview: widget.wakingUp,
1671
+ connectRequired: widget.connectRequired,
1672
+ errorMessage: _imageError?.toString(),
1673
+ onPressed: widget.onWakeRequested,
1674
+ );
1675
+ }
1676
+ final imageBytes = _imageBytes;
1677
+ if (imageBytes == null) {
1678
+ return _EmptySurfaceState(
1679
+ surface: widget.surface,
1680
+ enabled: widget.enabled,
1681
+ busy: widget.busy,
1682
+ isLoadingPreview: _imageError == null,
1683
+ connectRequired: widget.connectRequired,
1684
+ errorMessage: _imageError?.toString(),
1685
+ onPressed: widget.onWakeRequested,
1686
+ );
1687
+ }
1688
+ return Semantics(
1689
+ button: true,
1690
+ label:
1691
+ 'Device surface preview — tap to interact, swipe to scroll',
1692
+ child: GestureDetector(
1693
+ onTapUp: widget.busy
1694
+ ? null
1695
+ : (details) async {
1696
+ final point = _mapToPixels(
1697
+ details.localPosition,
1698
+ boxSize,
1699
+ );
1700
+ if (point != null) {
1701
+ await widget.onTapPoint(point);
1702
+ }
1703
+ },
1704
+ onPanStart: widget.busy
1705
+ ? null
1706
+ : (details) {
1707
+ _dragStart = details.localPosition;
1708
+ _dragEnd = details.localPosition;
1709
+ },
1710
+ onPanUpdate: widget.busy
1711
+ ? null
1712
+ : (details) {
1713
+ _dragEnd = details.localPosition;
1714
+ },
1715
+ onPanEnd: widget.busy
1716
+ ? null
1717
+ : (_) async {
1718
+ final start = _dragStart;
1719
+ final end = _dragEnd;
1720
+ _dragStart = null;
1721
+ _dragEnd = null;
1722
+ if (start == null || end == null) {
1723
+ return;
1724
+ }
1725
+ if ((start - end).distance < 12) {
1726
+ return;
1727
+ }
1728
+ final mappedStart = _mapToPixels(
1729
+ start,
1730
+ boxSize,
1731
+ );
1732
+ final mappedEnd = _mapToPixels(end, boxSize);
1733
+ if (mappedStart != null && mappedEnd != null) {
1734
+ await widget.onSwipe(mappedStart, mappedEnd);
1735
+ }
1736
+ },
1737
+ child: Stack(
1738
+ fit: StackFit.expand,
1739
+ children: <Widget>[
1740
+ Container(color: _bgSecondary),
1741
+ Image.memory(
1742
+ imageBytes,
1743
+ fit: BoxFit.contain,
1744
+ gaplessPlayback: true,
1745
+ ),
1746
+ Positioned(
1747
+ left: 12,
1748
+ right: 12,
1749
+ bottom: 12,
1750
+ child: Container(
1751
+ padding: const EdgeInsets.symmetric(
1752
+ horizontal: 12,
1753
+ vertical: 10,
1754
+ ),
1755
+ decoration: BoxDecoration(
1756
+ color: const Color(0xB205080D),
1757
+ borderRadius: BorderRadius.circular(14),
1758
+ border: Border.all(color: _borderLight),
1759
+ ),
1760
+ child: Text(
1761
+ widget.surface.helper,
1762
+ textAlign: TextAlign.center,
1763
+ style: TextStyle(color: _textPrimary),
1764
+ ),
1765
+ ),
1766
+ ),
1767
+ if (widget.busy)
1768
+ const Center(child: CircularProgressIndicator()),
1769
+ ],
1770
+ ),
1472
1771
  ),
1473
- ),
1474
1772
  );
1475
1773
  },
1476
1774
  ),
@@ -1592,9 +1890,6 @@ class _EmptySurfaceState extends StatelessWidget {
1592
1890
  }
1593
1891
  }
1594
1892
 
1595
-
1596
-
1597
-
1598
1893
  class _DeviceFieldRow extends StatelessWidget {
1599
1894
  const _DeviceFieldRow({required this.children});
1600
1895
 
@@ -1741,7 +2036,9 @@ class _ExtensionStatusBar extends StatelessWidget {
1741
2036
  child: Row(
1742
2037
  children: <Widget>[
1743
2038
  _DotStatus(
1744
- label: connected ? 'Extension connected' : 'Extension not connected',
2039
+ label: connected
2040
+ ? 'Extension connected'
2041
+ : 'Extension not connected',
1745
2042
  color: connected ? _success : _warning,
1746
2043
  ),
1747
2044
  const Spacer(),