neoagent 2.4.0 → 2.4.1-beta.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/LICENSE +619 -21
  2. package/README.md +1 -1
  3. package/extensions/chrome-browser/background.mjs +19 -7
  4. package/extensions/chrome-browser/icons/icon128.png +0 -0
  5. package/extensions/chrome-browser/icons/icon16.png +0 -0
  6. package/extensions/chrome-browser/icons/icon48.png +0 -0
  7. package/extensions/chrome-browser/icons/logo.svg +12 -0
  8. package/extensions/chrome-browser/manifest.json +13 -2
  9. package/extensions/chrome-browser/popup.css +5 -0
  10. package/extensions/chrome-browser/popup.html +7 -5
  11. package/extensions/chrome-browser/popup.js +16 -7
  12. package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +721 -0
  13. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +6 -0
  14. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +1 -1
  15. package/flutter_app/lib/main.dart +1 -0
  16. package/flutter_app/lib/main_controller.dart +156 -3
  17. package/flutter_app/lib/main_devices.dart +485 -119
  18. package/flutter_app/lib/main_settings.dart +289 -30
  19. package/flutter_app/lib/src/backend_client.dart +89 -0
  20. package/flutter_app/lib/src/desktop_companion_actions.dart +153 -3
  21. package/flutter_app/lib/src/desktop_companion_io.dart +145 -4
  22. package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
  23. package/flutter_app/lib/src/stream_renderer.dart +286 -0
  24. package/flutter_app/macos/Runner/AppDelegate.swift +56 -1
  25. package/package.json +2 -2
  26. package/server/guest_agent.js +19 -1
  27. package/server/http/routes.js +191 -0
  28. package/server/http/socket.js +1 -1
  29. package/server/index.js +4 -1
  30. package/server/public/.last_build_id +1 -1
  31. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  32. package/server/public/flutter_bootstrap.js +1 -1
  33. package/server/public/main.dart.js +75438 -74005
  34. package/server/routes/browser.js +14 -0
  35. package/server/routes/browser_extension.js +21 -4
  36. package/server/routes/desktop.js +10 -0
  37. package/server/routes/settings.js +4 -0
  38. package/server/routes/stream.js +187 -0
  39. package/server/services/ai/tools.js +40 -29
  40. package/server/services/android/controller.js +41 -2
  41. package/server/services/browser/controller.js +34 -0
  42. package/server/services/browser/extension/manifest.js +33 -0
  43. package/server/services/browser/extension/provider.js +12 -6
  44. package/server/services/browser/extension/registry.js +188 -18
  45. package/server/services/desktop/gateway.js +28 -3
  46. package/server/services/desktop/protocol.js +34 -0
  47. package/server/services/desktop/provider.js +25 -0
  48. package/server/services/desktop/registry.js +92 -10
  49. package/server/services/manager.js +19 -2
  50. package/server/services/runtime/backends/local-vm.js +6 -0
  51. package/server/services/runtime/docker-vm-manager.js +26 -3
  52. package/server/services/runtime/manager.js +36 -5
  53. package/server/services/runtime/settings.js +17 -0
  54. package/server/services/streaming/android-stream.js +298 -0
  55. package/server/services/streaming/browser-stream.js +87 -0
  56. package/server/services/streaming/stream-hub.js +231 -0
  57. package/server/services/websocket.js +73 -0
@@ -90,6 +90,29 @@ class _DevicesPanelState extends State<DevicesPanel> {
90
90
  bool get _androidStarting =>
91
91
  widget.controller.androidRuntime['starting'] == true;
92
92
 
93
+ String? get _androidDeviceId {
94
+ final status = widget.controller.androidRuntime;
95
+ final direct = status['adbSerial']?.toString().trim();
96
+ if (direct != null && direct.isNotEmpty) {
97
+ return direct;
98
+ }
99
+ final devices = _jsonMapList(status['devices'], fallbackToMapValues: true);
100
+ for (final device in devices) {
101
+ if (device['status']?.toString() != 'device') {
102
+ continue;
103
+ }
104
+ final serial = device['serial']?.toString().trim();
105
+ if (serial != null && serial.isNotEmpty) {
106
+ return serial;
107
+ }
108
+ final id = device['deviceId']?.toString().trim();
109
+ if (id != null && id.isNotEmpty) {
110
+ return id;
111
+ }
112
+ }
113
+ return null;
114
+ }
115
+
93
116
  List<Map<String, dynamic>> get _onlineDesktopDevices => widget
94
117
  .controller
95
118
  .desktopDevices
@@ -103,8 +126,16 @@ class _DevicesPanelState extends State<DevicesPanel> {
103
126
  _onlineDesktopDevices.length > 1 &&
104
127
  (widget.controller.selectedDesktopDeviceId ?? '').isEmpty;
105
128
 
129
+ bool get _extensionPreferredButOffline =>
130
+ widget.controller.browserBackend == 'extension' &&
131
+ !widget.controller.browserExtensionConnected;
132
+
133
+ List<Map<String, dynamic>> get _browserExtensionDevices =>
134
+ widget.controller.browserExtensionTokens;
135
+
106
136
  String? get _activeScreenshotPath {
107
137
  if (_isBrowser) {
138
+ if (_extensionPreferredButOffline) return null;
108
139
  return widget.controller.browserScreenshotPath;
109
140
  }
110
141
  if (_isDesktop) {
@@ -121,6 +152,7 @@ class _DevicesPanelState extends State<DevicesPanel> {
121
152
  Future<void> _ensurePreview() async {
122
153
  final controller = widget.controller;
123
154
  if (_isBrowser) {
155
+ if (_extensionPreferredButOffline) return;
124
156
  if (controller.browserRuntime['launched'] != true) {
125
157
  return;
126
158
  }
@@ -161,6 +193,7 @@ class _DevicesPanelState extends State<DevicesPanel> {
161
193
  return;
162
194
  }
163
195
  if (_isBrowser) {
196
+ if (_extensionPreferredButOffline) return;
164
197
  await widget.controller.refreshBrowserFrameRuntime();
165
198
  return;
166
199
  }
@@ -289,6 +322,23 @@ class _DevicesPanelState extends State<DevicesPanel> {
289
322
  }
290
323
  });
291
324
 
325
+ void _handleHover(Offset point) {
326
+ if (_isBrowser) {
327
+ unawaited(widget.controller.hoverBrowserPointRuntime(
328
+ x: point.dx.round(),
329
+ y: point.dy.round(),
330
+ ));
331
+ } else if (_isDesktop) {
332
+ if (_desktopRequiresSelection) {
333
+ return;
334
+ }
335
+ unawaited(widget.controller.hoverDesktopRuntime(
336
+ x: point.dx.round(),
337
+ y: point.dy.round(),
338
+ ));
339
+ }
340
+ }
341
+
292
342
  Future<void> _handleTap(Offset point) => _runOnSurface(() async {
293
343
  if (_isBrowser) {
294
344
  await widget.controller.clickBrowserPointRuntime(
@@ -317,36 +367,37 @@ class _DevicesPanelState extends State<DevicesPanel> {
317
367
  });
318
368
  });
319
369
 
320
- Future<void> _handleSwipe(Offset start, Offset end) => _runOnSurface(() async {
321
- if (_isBrowser) {
322
- await widget.controller.scrollBrowserRuntime(
323
- deltaY: (start.dy - end.dy).round(),
324
- );
325
- return;
326
- }
327
- if (_isDesktop) {
328
- if (_desktopRequiresSelection) {
329
- return;
330
- }
331
- await widget.controller.dragDesktopRuntime(
332
- x1: start.dx.round(),
333
- y1: start.dy.round(),
334
- x2: end.dx.round(),
335
- y2: end.dy.round(),
336
- );
337
- return;
338
- }
339
- if (!_androidOnline) {
340
- return;
341
- }
342
- await widget.controller.swipeAndroidRuntime(<String, dynamic>{
343
- 'x1': start.dx.round(),
344
- 'y1': start.dy.round(),
345
- 'x2': end.dx.round(),
346
- 'y2': end.dy.round(),
347
- 'durationMs': 280,
348
- });
349
- });
370
+ Future<void> _handleSwipe(Offset start, Offset end) =>
371
+ _runOnSurface(() async {
372
+ if (_isBrowser) {
373
+ await widget.controller.scrollBrowserRuntime(
374
+ deltaY: (start.dy - end.dy).round(),
375
+ );
376
+ return;
377
+ }
378
+ if (_isDesktop) {
379
+ if (_desktopRequiresSelection) {
380
+ return;
381
+ }
382
+ await widget.controller.dragDesktopRuntime(
383
+ x1: start.dx.round(),
384
+ y1: start.dy.round(),
385
+ x2: end.dx.round(),
386
+ y2: end.dy.round(),
387
+ );
388
+ return;
389
+ }
390
+ if (!_androidOnline) {
391
+ return;
392
+ }
393
+ await widget.controller.swipeAndroidRuntime(<String, dynamic>{
394
+ 'x1': start.dx.round(),
395
+ 'y1': start.dy.round(),
396
+ 'x2': end.dx.round(),
397
+ 'y2': end.dy.round(),
398
+ 'durationMs': 280,
399
+ });
400
+ });
350
401
 
351
402
  Future<void> _runQuickAction(String action) => _runOnSurface(() async {
352
403
  final controller = widget.controller;
@@ -391,6 +442,13 @@ class _DevicesPanelState extends State<DevicesPanel> {
391
442
  final prefersExtension = controller.browserBackend == 'extension';
392
443
  final extensionConnected = controller.browserExtensionConnected;
393
444
  final usingExtension = prefersExtension && extensionConnected;
445
+ final selectedBrowserExtension = controller.browserExtensionTokens
446
+ .where(
447
+ (device) =>
448
+ device['tokenId'] == controller.selectedBrowserExtensionTokenId,
449
+ )
450
+ .cast<Map<String, dynamic>?>()
451
+ .firstWhere((device) => device != null, orElse: () => null);
394
452
  final browserFallbackLabel = 'cloud browser runtime';
395
453
  final browserPageInfo = browserStatus['pageInfo'] is Map<dynamic, dynamic>
396
454
  ? Map<String, dynamic>.from(browserStatus['pageInfo'] as Map)
@@ -447,10 +505,69 @@ class _DevicesPanelState extends State<DevicesPanel> {
447
505
  desktopDevices: controller.desktopDevices,
448
506
  selectedDesktopDeviceId:
449
507
  controller.selectedDesktopDeviceId,
508
+ browserExtensionDevices:
509
+ controller.browserExtensionTokens,
510
+ selectedBrowserExtensionTokenId:
511
+ controller.selectedBrowserExtensionTokenId,
450
512
  browserExtensionPreferred: prefersExtension,
451
513
  browserExtensionActive: usingExtension,
452
514
  browserFallbackLabel: browserFallbackLabel,
453
515
  ),
516
+ if (_isBrowser && prefersExtension) ...<Widget>[
517
+ const SizedBox(height: 14),
518
+ if (_browserExtensionDevices.isNotEmpty) ...<Widget>[
519
+ DropdownButtonFormField<String>(
520
+ isExpanded: true,
521
+ initialValue: selectedBrowserExtension?['tokenId']
522
+ ?.toString(),
523
+ decoration: const InputDecoration(
524
+ labelText: 'Chrome extension device',
525
+ prefixIcon: Icon(Icons.extension_outlined),
526
+ ),
527
+ hint: const Text('Select a paired extension'),
528
+ items: _browserExtensionDevices.map((device) {
529
+ final tokenId = device['tokenId']?.toString() ?? '';
530
+ final label =
531
+ device['name']?.toString().trim().isNotEmpty ==
532
+ true
533
+ ? device['name'].toString()
534
+ : tokenId;
535
+ final state =
536
+ device['online'] == true ||
537
+ device['connected'] == true
538
+ ? 'online'
539
+ : 'offline';
540
+ return DropdownMenuItem<String>(
541
+ value: tokenId,
542
+ child: Text(
543
+ '$label · $state',
544
+ maxLines: 1,
545
+ overflow: TextOverflow.ellipsis,
546
+ softWrap: false,
547
+ ),
548
+ );
549
+ }).toList(),
550
+ onChanged: _isCurrentSurfaceBusy
551
+ ? null
552
+ : (value) {
553
+ if (value == null || value.isEmpty) {
554
+ return;
555
+ }
556
+ unawaited(
557
+ controller.selectBrowserExtensionRuntime(
558
+ value,
559
+ ),
560
+ );
561
+ },
562
+ ),
563
+ const SizedBox(height: 10),
564
+ ],
565
+ _ExtensionStatusBar(
566
+ connected: extensionConnected,
567
+ onDownload: controller.downloadBrowserExtension,
568
+ onRefresh: controller.refreshBrowserExtensionStatus,
569
+ ),
570
+ ],
454
571
  if (_isDesktop) ...<Widget>[
455
572
  const SizedBox(height: 14),
456
573
  DropdownButtonFormField<String>(
@@ -555,12 +672,30 @@ class _DevicesPanelState extends State<DevicesPanel> {
555
672
  surface: _surface,
556
673
  controller: controller,
557
674
  screenshotPath: _activeScreenshotPath,
675
+ streamPlatform: _isBrowser && browserStatus['launched'] == true
676
+ ? 'browser'
677
+ : (_isDesktop && _desktopOnline
678
+ ? 'desktop'
679
+ : (!_isBrowser && !_isDesktop && _androidOnline
680
+ ? 'android'
681
+ : null)),
682
+ streamDeviceId: _isBrowser && browserStatus['launched'] == true
683
+ ? 'browser'
684
+ : (_isDesktop && _desktopOnline
685
+ ? (widget.controller.selectedDesktopDeviceId ??
686
+ (_onlineDesktopDevices.isNotEmpty ? _onlineDesktopDevices.first['deviceId']?.toString() : null))
687
+ : (!_isBrowser && !_isDesktop && _androidOnline
688
+ ? _androidDeviceId
689
+ : null)),
558
690
  busy: _isCurrentSurfaceBusy,
559
691
  wakingUp: !_isBrowser && !_isDesktop && _androidStarting,
560
692
  enabled: _isBrowser || _isDesktop || _androidOnline,
561
- connectRequired: _desktopRequiresSelection,
693
+ connectRequired: _isBrowser
694
+ ? _extensionPreferredButOffline
695
+ : _desktopRequiresSelection,
562
696
  onTapPoint: _handleTap,
563
697
  onSwipe: _handleSwipe,
698
+ onHover: _handleHover,
564
699
  onWakeRequested: _openPrimary,
565
700
  ),
566
701
  if (!_isBrowser && !_isDesktop) ...<Widget>[
@@ -649,6 +784,8 @@ class _DeviceSurfaceHeader extends StatelessWidget {
649
784
  required this.desktopRuntime,
650
785
  required this.desktopDevices,
651
786
  required this.selectedDesktopDeviceId,
787
+ required this.browserExtensionDevices,
788
+ required this.selectedBrowserExtensionTokenId,
652
789
  required this.browserExtensionPreferred,
653
790
  required this.browserExtensionActive,
654
791
  required this.browserFallbackLabel,
@@ -662,6 +799,8 @@ class _DeviceSurfaceHeader extends StatelessWidget {
662
799
  final Map<String, dynamic> desktopRuntime;
663
800
  final List<Map<String, dynamic>> desktopDevices;
664
801
  final String? selectedDesktopDeviceId;
802
+ final List<Map<String, dynamic>> browserExtensionDevices;
803
+ final String? selectedBrowserExtensionTokenId;
665
804
  final bool browserExtensionPreferred;
666
805
  final bool browserExtensionActive;
667
806
  final String browserFallbackLabel;
@@ -677,12 +816,29 @@ class _DeviceSurfaceHeader extends StatelessWidget {
677
816
  .where((device) => device['deviceId'] == selectedDesktopDeviceId)
678
817
  .cast<Map<String, dynamic>?>()
679
818
  .firstWhere((device) => device != null, orElse: () => null);
819
+ final selectedExtension = browserExtensionDevices
820
+ .where(
821
+ (device) => device['tokenId'] == selectedBrowserExtensionTokenId,
822
+ )
823
+ .cast<Map<String, dynamic>?>()
824
+ .firstWhere((device) => device != null, orElse: () => null);
825
+ final extensionOnlineCount = browserExtensionDevices
826
+ .where(
827
+ (device) =>
828
+ device['online'] == true || device['connected'] == true,
829
+ )
830
+ .length;
680
831
  final desktopOnlineCount = desktopDevices
681
832
  .where((device) => device['online'] == true)
682
833
  .length;
683
834
  final title = switch (surface) {
684
835
  _DeviceSurface.browser =>
685
- (browserPageInfo['title']?.toString().trim().isNotEmpty ?? false)
836
+ browserExtensionPreferred &&
837
+ selectedExtension?['name']?.toString().trim().isNotEmpty ==
838
+ true
839
+ ? selectedExtension!['name'].toString()
840
+ : (browserPageInfo['title']?.toString().trim().isNotEmpty ??
841
+ false)
686
842
  ? browserPageInfo['title'].toString()
687
843
  : 'Live Browser',
688
844
  _DeviceSurface.android => 'Android Phone',
@@ -692,9 +848,13 @@ class _DeviceSurfaceHeader extends StatelessWidget {
692
848
  : 'Desktop Companion',
693
849
  };
694
850
  final subtitle = switch (surface) {
695
- _DeviceSurface.browser =>
696
- browserExtensionPreferred && !browserExtensionActive
697
- ? 'No extension device is active. Using the $browserFallbackLabel.'
851
+ _DeviceSurface.browser =>
852
+ browserExtensionPreferred && selectedExtension == null
853
+ ? extensionOnlineCount > 1
854
+ ? 'Multiple extension devices are online. Pick the browser you want to control.'
855
+ : 'No extension device is active. Using the $browserFallbackLabel.'
856
+ : browserExtensionPreferred && !browserExtensionActive
857
+ ? 'Selected extension is offline. Using the $browserFallbackLabel.'
698
858
  : (browserPageInfo['url']?.toString() ??
699
859
  'Ready for navigation'),
700
860
  _DeviceSurface.android =>
@@ -729,7 +889,9 @@ class _DeviceSurfaceHeader extends StatelessWidget {
729
889
  : '${selectedDesktop['platform'] ?? 'desktop'} · ${selectedDesktop['hostname'] ?? 'unknown host'}',
730
890
  };
731
891
  final statusLabel = surface == _DeviceSurface.browser
732
- ? browserExtensionPreferred && !browserExtensionActive
892
+ ? browserExtensionPreferred && selectedExtension == null
893
+ ? (extensionOnlineCount > 0 ? 'Select Device' : 'Fallback')
894
+ : browserExtensionPreferred && !browserExtensionActive
733
895
  ? 'Fallback'
734
896
  : browserExtensionActive
735
897
  ? 'Extension'
@@ -1103,10 +1265,7 @@ class _AndroidNavDock extends StatelessWidget {
1103
1265
  }
1104
1266
 
1105
1267
  class _SurfaceSwitcher extends StatelessWidget {
1106
- const _SurfaceSwitcher({
1107
- required this.surface,
1108
- required this.onSelect,
1109
- });
1268
+ const _SurfaceSwitcher({required this.surface, required this.onSelect});
1110
1269
 
1111
1270
  final _DeviceSurface surface;
1112
1271
  final Future<void> Function(_DeviceSurface) onSelect;
@@ -1140,8 +1299,7 @@ class _SurfaceSwitcher extends StatelessWidget {
1140
1299
  ),
1141
1300
  labelStyle: TextStyle(
1142
1301
  color: selected ? _textPrimary : _textSecondary,
1143
- fontWeight:
1144
- selected ? FontWeight.w700 : FontWeight.w500,
1302
+ fontWeight: selected ? FontWeight.w700 : FontWeight.w500,
1145
1303
  ),
1146
1304
  );
1147
1305
  }).toList(),
@@ -1162,24 +1320,30 @@ class _InteractiveSurfacePreview extends StatefulWidget {
1162
1320
  required this.surface,
1163
1321
  required this.controller,
1164
1322
  required this.screenshotPath,
1323
+ required this.streamPlatform,
1324
+ required this.streamDeviceId,
1165
1325
  required this.busy,
1166
1326
  required this.wakingUp,
1167
1327
  required this.enabled,
1168
1328
  required this.connectRequired,
1169
1329
  required this.onTapPoint,
1170
1330
  required this.onSwipe,
1331
+ this.onHover,
1171
1332
  required this.onWakeRequested,
1172
1333
  });
1173
1334
 
1174
1335
  final _DeviceSurface surface;
1175
1336
  final NeoAgentController controller;
1176
1337
  final String? screenshotPath;
1338
+ final String? streamPlatform;
1339
+ final String? streamDeviceId;
1177
1340
  final bool busy;
1178
1341
  final bool wakingUp;
1179
1342
  final bool enabled;
1180
1343
  final bool connectRequired;
1181
1344
  final Future<void> Function(Offset point) onTapPoint;
1182
1345
  final Future<void> Function(Offset start, Offset end) onSwipe;
1346
+ final void Function(Offset point)? onHover;
1183
1347
  final Future<void> Function() onWakeRequested;
1184
1348
 
1185
1349
  @override
@@ -1196,11 +1360,15 @@ class _InteractiveSurfacePreviewState
1196
1360
  Object? _imageError;
1197
1361
  Offset? _dragStart;
1198
1362
  Offset? _dragEnd;
1363
+ bool _streamStarting = false;
1364
+ String? _activeStreamKey;
1365
+ String? _streamFailedKey;
1199
1366
 
1200
1367
  @override
1201
1368
  void initState() {
1202
1369
  super.initState();
1203
1370
  unawaited(_loadImage());
1371
+ unawaited(_syncStream());
1204
1372
  }
1205
1373
 
1206
1374
  @override
@@ -1209,14 +1377,95 @@ class _InteractiveSurfacePreviewState
1209
1377
  if (oldWidget.screenshotPath != widget.screenshotPath) {
1210
1378
  unawaited(_loadImage());
1211
1379
  }
1380
+ if (oldWidget.streamPlatform != widget.streamPlatform ||
1381
+ oldWidget.streamDeviceId != widget.streamDeviceId ||
1382
+ oldWidget.controller.streamSocket != widget.controller.streamSocket) {
1383
+ unawaited(_syncStream());
1384
+ }
1212
1385
  }
1213
1386
 
1214
1387
  @override
1215
1388
  void dispose() {
1389
+ unawaited(_stopActiveStream());
1216
1390
  _detachImageListener();
1217
1391
  super.dispose();
1218
1392
  }
1219
1393
 
1394
+ String? get _requestedStreamKey {
1395
+ final platform = widget.streamPlatform?.trim();
1396
+ final deviceId = widget.streamDeviceId?.trim();
1397
+ if (platform == null ||
1398
+ platform.isEmpty ||
1399
+ deviceId == null ||
1400
+ deviceId.isEmpty ||
1401
+ widget.controller.streamSocket == null) {
1402
+ return null;
1403
+ }
1404
+ return '$platform:$deviceId';
1405
+ }
1406
+
1407
+ Future<void> _syncStream() async {
1408
+ final requested = _requestedStreamKey;
1409
+ if (_activeStreamKey == requested || _streamStarting) {
1410
+ return;
1411
+ }
1412
+ if (requested != _streamFailedKey) {
1413
+ _streamFailedKey = null;
1414
+ }
1415
+ await _stopActiveStream();
1416
+ if (requested == null) {
1417
+ return;
1418
+ }
1419
+ final parts = requested.split(':');
1420
+ _streamStarting = true;
1421
+ try {
1422
+ await widget.controller.startStreamRuntime(
1423
+ platform: parts[0],
1424
+ deviceId: parts.sublist(1).join(':'),
1425
+ fps: 10,
1426
+ quality: 70,
1427
+ );
1428
+ if (mounted && _requestedStreamKey == requested) {
1429
+ _activeStreamKey = requested;
1430
+ _streamFailedKey = null;
1431
+ } else {
1432
+ await widget.controller.stopStreamRuntime(
1433
+ platform: parts[0],
1434
+ deviceId: parts.sublist(1).join(':'),
1435
+ );
1436
+ }
1437
+ } catch (_) {
1438
+ if (mounted) {
1439
+ _streamFailedKey = requested;
1440
+ }
1441
+ if (mounted && (widget.screenshotPath ?? '').isEmpty) {
1442
+ unawaited(_loadImage());
1443
+ }
1444
+ } finally {
1445
+ _streamStarting = false;
1446
+ if (mounted &&
1447
+ _activeStreamKey != _requestedStreamKey &&
1448
+ _streamFailedKey != _requestedStreamKey) {
1449
+ unawaited(_syncStream());
1450
+ }
1451
+ }
1452
+ }
1453
+
1454
+ Future<void> _stopActiveStream() async {
1455
+ final active = _activeStreamKey;
1456
+ _activeStreamKey = null;
1457
+ if (active == null) {
1458
+ return;
1459
+ }
1460
+ final parts = active.split(':');
1461
+ try {
1462
+ await widget.controller.stopStreamRuntime(
1463
+ platform: parts[0],
1464
+ deviceId: parts.sublist(1).join(':'),
1465
+ );
1466
+ } catch (_) {}
1467
+ }
1468
+
1220
1469
  void _detachImageListener() {
1221
1470
  if (_imageStream != null && _imageListener != null) {
1222
1471
  _imageStream!.removeListener(_imageListener!);
@@ -1351,82 +1600,41 @@ class _InteractiveSurfacePreviewState
1351
1600
  constraints.maxWidth,
1352
1601
  constraints.maxHeight,
1353
1602
  );
1354
- if (!hasImage) {
1355
- return _EmptySurfaceState(
1356
- surface: widget.surface,
1357
- enabled: widget.enabled,
1358
- busy: widget.busy,
1359
- isLoadingPreview: widget.wakingUp,
1360
- connectRequired: widget.connectRequired,
1361
- errorMessage: _imageError?.toString(),
1362
- onPressed: widget.onWakeRequested,
1363
- );
1364
- }
1365
- final imageBytes = _imageBytes;
1366
- if (imageBytes == null) {
1367
- return _EmptySurfaceState(
1368
- surface: widget.surface,
1369
- enabled: widget.enabled,
1370
- busy: widget.busy,
1371
- isLoadingPreview: _imageError == null,
1372
- connectRequired: widget.connectRequired,
1373
- errorMessage: _imageError?.toString(),
1374
- onPressed: widget.onWakeRequested,
1375
- );
1376
- }
1377
- return Semantics(
1378
- button: true,
1379
- label: 'Device surface preview — tap to interact, swipe to scroll',
1380
- child: GestureDetector(
1381
- onTapUp: widget.busy
1382
- ? null
1383
- : (details) async {
1384
- final point = _mapToPixels(
1385
- details.localPosition,
1386
- boxSize,
1387
- );
1388
- if (point != null) {
1389
- await widget.onTapPoint(point);
1390
- }
1391
- },
1392
- onPanStart: widget.busy
1393
- ? null
1394
- : (details) {
1395
- _dragStart = details.localPosition;
1396
- _dragEnd = details.localPosition;
1397
- },
1398
- onPanUpdate: widget.busy
1399
- ? null
1400
- : (details) {
1401
- _dragEnd = details.localPosition;
1402
- },
1403
- onPanEnd: widget.busy
1404
- ? null
1405
- : (_) async {
1406
- final start = _dragStart;
1407
- final end = _dragEnd;
1408
- _dragStart = null;
1409
- _dragEnd = null;
1410
- if (start == null || end == null) {
1411
- return;
1412
- }
1413
- if ((start - end).distance < 12) {
1414
- return;
1415
- }
1416
- final mappedStart = _mapToPixels(start, boxSize);
1417
- final mappedEnd = _mapToPixels(end, boxSize);
1418
- if (mappedStart != null && mappedEnd != null) {
1419
- await widget.onSwipe(mappedStart, mappedEnd);
1420
- }
1421
- },
1422
- child: Stack(
1603
+ final socket = widget.controller.streamSocket;
1604
+ final streamPlatform = widget.streamPlatform;
1605
+ final streamDeviceId = widget.streamDeviceId;
1606
+ final streamKey = _requestedStreamKey;
1607
+ if (socket != null &&
1608
+ streamPlatform != null &&
1609
+ streamPlatform.isNotEmpty &&
1610
+ streamDeviceId != null &&
1611
+ streamDeviceId.isNotEmpty &&
1612
+ streamKey != _streamFailedKey) {
1613
+ return Stack(
1423
1614
  fit: StackFit.expand,
1424
1615
  children: <Widget>[
1425
1616
  Container(color: _bgSecondary),
1426
- Image.memory(
1427
- imageBytes,
1428
- fit: BoxFit.contain,
1429
- gaplessPlayback: true,
1617
+ StreamRenderer(
1618
+ socket: socket,
1619
+ deviceId: streamDeviceId,
1620
+ platform: streamPlatform,
1621
+ remoteResolution: _pixelSize,
1622
+ onTap: widget.busy
1623
+ ? null
1624
+ : (x, y) => unawaited(
1625
+ widget.onTapPoint(Offset(x, y)),
1626
+ ),
1627
+ onSwipe: widget.busy
1628
+ ? null
1629
+ : (x1, y1, x2, y2) => unawaited(
1630
+ widget.onSwipe(
1631
+ Offset(x1, y1),
1632
+ Offset(x2, y2),
1633
+ ),
1634
+ ),
1635
+ onHover: widget.busy
1636
+ ? null
1637
+ : (x, y) => widget.onHover?.call(Offset(x, y)),
1430
1638
  ),
1431
1639
  Positioned(
1432
1640
  left: 12,
@@ -1452,8 +1660,115 @@ class _InteractiveSurfacePreviewState
1452
1660
  if (widget.busy)
1453
1661
  const Center(child: CircularProgressIndicator()),
1454
1662
  ],
1663
+ );
1664
+ }
1665
+ if (!hasImage) {
1666
+ return _EmptySurfaceState(
1667
+ surface: widget.surface,
1668
+ enabled: widget.enabled,
1669
+ busy: widget.busy,
1670
+ isLoadingPreview: widget.wakingUp,
1671
+ connectRequired: widget.connectRequired,
1672
+ errorMessage: _imageError?.toString(),
1673
+ onPressed: widget.onWakeRequested,
1674
+ );
1675
+ }
1676
+ final imageBytes = _imageBytes;
1677
+ if (imageBytes == null) {
1678
+ return _EmptySurfaceState(
1679
+ surface: widget.surface,
1680
+ enabled: widget.enabled,
1681
+ busy: widget.busy,
1682
+ isLoadingPreview: _imageError == null,
1683
+ connectRequired: widget.connectRequired,
1684
+ errorMessage: _imageError?.toString(),
1685
+ onPressed: widget.onWakeRequested,
1686
+ );
1687
+ }
1688
+ return Semantics(
1689
+ button: true,
1690
+ label:
1691
+ 'Device surface preview — tap to interact, swipe to scroll',
1692
+ child: GestureDetector(
1693
+ onTapUp: widget.busy
1694
+ ? null
1695
+ : (details) async {
1696
+ final point = _mapToPixels(
1697
+ details.localPosition,
1698
+ boxSize,
1699
+ );
1700
+ if (point != null) {
1701
+ await widget.onTapPoint(point);
1702
+ }
1703
+ },
1704
+ onPanStart: widget.busy
1705
+ ? null
1706
+ : (details) {
1707
+ _dragStart = details.localPosition;
1708
+ _dragEnd = details.localPosition;
1709
+ },
1710
+ onPanUpdate: widget.busy
1711
+ ? null
1712
+ : (details) {
1713
+ _dragEnd = details.localPosition;
1714
+ },
1715
+ onPanEnd: widget.busy
1716
+ ? null
1717
+ : (_) async {
1718
+ final start = _dragStart;
1719
+ final end = _dragEnd;
1720
+ _dragStart = null;
1721
+ _dragEnd = null;
1722
+ if (start == null || end == null) {
1723
+ return;
1724
+ }
1725
+ if ((start - end).distance < 12) {
1726
+ return;
1727
+ }
1728
+ final mappedStart = _mapToPixels(
1729
+ start,
1730
+ boxSize,
1731
+ );
1732
+ final mappedEnd = _mapToPixels(end, boxSize);
1733
+ if (mappedStart != null && mappedEnd != null) {
1734
+ await widget.onSwipe(mappedStart, mappedEnd);
1735
+ }
1736
+ },
1737
+ child: Stack(
1738
+ fit: StackFit.expand,
1739
+ children: <Widget>[
1740
+ Container(color: _bgSecondary),
1741
+ Image.memory(
1742
+ imageBytes,
1743
+ fit: BoxFit.contain,
1744
+ gaplessPlayback: true,
1745
+ ),
1746
+ Positioned(
1747
+ left: 12,
1748
+ right: 12,
1749
+ bottom: 12,
1750
+ child: Container(
1751
+ padding: const EdgeInsets.symmetric(
1752
+ horizontal: 12,
1753
+ vertical: 10,
1754
+ ),
1755
+ decoration: BoxDecoration(
1756
+ color: const Color(0xB205080D),
1757
+ borderRadius: BorderRadius.circular(14),
1758
+ border: Border.all(color: _borderLight),
1759
+ ),
1760
+ child: Text(
1761
+ widget.surface.helper,
1762
+ textAlign: TextAlign.center,
1763
+ style: TextStyle(color: _textPrimary),
1764
+ ),
1765
+ ),
1766
+ ),
1767
+ if (widget.busy)
1768
+ const Center(child: CircularProgressIndicator()),
1769
+ ],
1770
+ ),
1455
1771
  ),
1456
- ),
1457
1772
  );
1458
1773
  },
1459
1774
  ),
@@ -1575,9 +1890,6 @@ class _EmptySurfaceState extends StatelessWidget {
1575
1890
  }
1576
1891
  }
1577
1892
 
1578
-
1579
-
1580
-
1581
1893
  class _DeviceFieldRow extends StatelessWidget {
1582
1894
  const _DeviceFieldRow({required this.children});
1583
1895
 
@@ -1701,6 +2013,60 @@ class _RuntimePreview extends StatelessWidget {
1701
2013
  }
1702
2014
  }
1703
2015
 
2016
+ class _ExtensionStatusBar extends StatelessWidget {
2017
+ const _ExtensionStatusBar({
2018
+ required this.connected,
2019
+ required this.onDownload,
2020
+ required this.onRefresh,
2021
+ });
2022
+
2023
+ final bool connected;
2024
+ final Future<void> Function() onDownload;
2025
+ final Future<void> Function() onRefresh;
2026
+
2027
+ @override
2028
+ Widget build(BuildContext context) {
2029
+ return Container(
2030
+ padding: const EdgeInsets.symmetric(horizontal: 14, vertical: 12),
2031
+ decoration: BoxDecoration(
2032
+ color: _bgSecondary,
2033
+ borderRadius: BorderRadius.circular(16),
2034
+ border: Border.all(color: _borderLight),
2035
+ ),
2036
+ child: Row(
2037
+ children: <Widget>[
2038
+ _DotStatus(
2039
+ label: connected
2040
+ ? 'Extension connected'
2041
+ : 'Extension not connected',
2042
+ color: connected ? _success : _warning,
2043
+ ),
2044
+ const Spacer(),
2045
+ OutlinedButton.icon(
2046
+ onPressed: onDownload,
2047
+ icon: const Icon(Icons.download_outlined, size: 18),
2048
+ label: const Text('Download'),
2049
+ style: OutlinedButton.styleFrom(
2050
+ visualDensity: VisualDensity.compact,
2051
+ padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8),
2052
+ ),
2053
+ ),
2054
+ const SizedBox(width: 8),
2055
+ OutlinedButton.icon(
2056
+ onPressed: onRefresh,
2057
+ icon: const Icon(Icons.sync, size: 18),
2058
+ label: const Text('Refresh'),
2059
+ style: OutlinedButton.styleFrom(
2060
+ visualDensity: VisualDensity.compact,
2061
+ padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8),
2062
+ ),
2063
+ ),
2064
+ ],
2065
+ ),
2066
+ );
2067
+ }
2068
+ }
2069
+
1704
2070
  class _ResultBlock extends StatelessWidget {
1705
2071
  const _ResultBlock({required this.label, required this.value});
1706
2072