neoagent 2.4.1-beta.8 → 2.4.1-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/chrome-browser/background.mjs +9 -3
- package/extensions/chrome-browser/manifest.json +2 -1
- package/extensions/chrome-browser/popup.js +2 -2
- package/flutter_app/lib/main.dart +1 -0
- package/flutter_app/lib/main_controller.dart +141 -4
- package/flutter_app/lib/main_devices.dart +416 -119
- package/flutter_app/lib/main_settings.dart +89 -14
- package/flutter_app/lib/src/backend_client.dart +33 -0
- package/flutter_app/lib/src/desktop_companion_actions.dart +26 -0
- package/flutter_app/lib/src/desktop_companion_io.dart +6 -0
- package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
- package/flutter_app/lib/src/stream_renderer.dart +166 -35
- package/flutter_app/macos/Runner/AppDelegate.swift +44 -0
- package/package.json +1 -1
- package/server/public/.last_build_id +1 -1
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +67339 -66716
- package/server/routes/browser.js +14 -0
- package/server/routes/browser_extension.js +21 -4
- package/server/routes/desktop.js +10 -0
- package/server/routes/settings.js +4 -0
- package/server/routes/stream.js +11 -2
- package/server/services/ai/tools.js +2 -2
- package/server/services/android/controller.js +36 -1
- package/server/services/browser/controller.js +18 -0
- package/server/services/browser/extension/manifest.js +33 -0
- package/server/services/browser/extension/provider.js +12 -6
- package/server/services/browser/extension/registry.js +144 -18
- package/server/services/desktop/protocol.js +1 -0
- package/server/services/desktop/provider.js +4 -0
- package/server/services/manager.js +8 -2
- package/server/services/runtime/manager.js +7 -3
- package/server/services/runtime/settings.js +17 -0
- package/server/services/streaming/android-stream.js +18 -7
- package/server/services/streaming/stream-hub.js +1 -1
|
@@ -90,6 +90,29 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
90
90
|
bool get _androidStarting =>
|
|
91
91
|
widget.controller.androidRuntime['starting'] == true;
|
|
92
92
|
|
|
93
|
+
String? get _androidDeviceId {
|
|
94
|
+
final status = widget.controller.androidRuntime;
|
|
95
|
+
final direct = status['adbSerial']?.toString().trim();
|
|
96
|
+
if (direct != null && direct.isNotEmpty) {
|
|
97
|
+
return direct;
|
|
98
|
+
}
|
|
99
|
+
final devices = _jsonMapList(status['devices'], fallbackToMapValues: true);
|
|
100
|
+
for (final device in devices) {
|
|
101
|
+
if (device['status']?.toString() != 'device') {
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
final serial = device['serial']?.toString().trim();
|
|
105
|
+
if (serial != null && serial.isNotEmpty) {
|
|
106
|
+
return serial;
|
|
107
|
+
}
|
|
108
|
+
final id = device['deviceId']?.toString().trim();
|
|
109
|
+
if (id != null && id.isNotEmpty) {
|
|
110
|
+
return id;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
93
116
|
List<Map<String, dynamic>> get _onlineDesktopDevices => widget
|
|
94
117
|
.controller
|
|
95
118
|
.desktopDevices
|
|
@@ -107,6 +130,9 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
107
130
|
widget.controller.browserBackend == 'extension' &&
|
|
108
131
|
!widget.controller.browserExtensionConnected;
|
|
109
132
|
|
|
133
|
+
List<Map<String, dynamic>> get _browserExtensionDevices =>
|
|
134
|
+
widget.controller.browserExtensionTokens;
|
|
135
|
+
|
|
110
136
|
String? get _activeScreenshotPath {
|
|
111
137
|
if (_isBrowser) {
|
|
112
138
|
if (_extensionPreferredButOffline) return null;
|
|
@@ -296,6 +322,23 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
296
322
|
}
|
|
297
323
|
});
|
|
298
324
|
|
|
325
|
+
void _handleHover(Offset point) {
|
|
326
|
+
if (_isBrowser) {
|
|
327
|
+
unawaited(widget.controller.hoverBrowserPointRuntime(
|
|
328
|
+
x: point.dx.round(),
|
|
329
|
+
y: point.dy.round(),
|
|
330
|
+
));
|
|
331
|
+
} else if (_isDesktop) {
|
|
332
|
+
if (_desktopRequiresSelection) {
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
unawaited(widget.controller.hoverDesktopRuntime(
|
|
336
|
+
x: point.dx.round(),
|
|
337
|
+
y: point.dy.round(),
|
|
338
|
+
));
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
299
342
|
Future<void> _handleTap(Offset point) => _runOnSurface(() async {
|
|
300
343
|
if (_isBrowser) {
|
|
301
344
|
await widget.controller.clickBrowserPointRuntime(
|
|
@@ -324,36 +367,37 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
324
367
|
});
|
|
325
368
|
});
|
|
326
369
|
|
|
327
|
-
Future<void> _handleSwipe(Offset start, Offset end) =>
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
370
|
+
Future<void> _handleSwipe(Offset start, Offset end) =>
|
|
371
|
+
_runOnSurface(() async {
|
|
372
|
+
if (_isBrowser) {
|
|
373
|
+
await widget.controller.scrollBrowserRuntime(
|
|
374
|
+
deltaY: (start.dy - end.dy).round(),
|
|
375
|
+
);
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
if (_isDesktop) {
|
|
379
|
+
if (_desktopRequiresSelection) {
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
await widget.controller.dragDesktopRuntime(
|
|
383
|
+
x1: start.dx.round(),
|
|
384
|
+
y1: start.dy.round(),
|
|
385
|
+
x2: end.dx.round(),
|
|
386
|
+
y2: end.dy.round(),
|
|
387
|
+
);
|
|
388
|
+
return;
|
|
389
|
+
}
|
|
390
|
+
if (!_androidOnline) {
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
await widget.controller.swipeAndroidRuntime(<String, dynamic>{
|
|
394
|
+
'x1': start.dx.round(),
|
|
395
|
+
'y1': start.dy.round(),
|
|
396
|
+
'x2': end.dx.round(),
|
|
397
|
+
'y2': end.dy.round(),
|
|
398
|
+
'durationMs': 280,
|
|
399
|
+
});
|
|
400
|
+
});
|
|
357
401
|
|
|
358
402
|
Future<void> _runQuickAction(String action) => _runOnSurface(() async {
|
|
359
403
|
final controller = widget.controller;
|
|
@@ -398,6 +442,13 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
398
442
|
final prefersExtension = controller.browserBackend == 'extension';
|
|
399
443
|
final extensionConnected = controller.browserExtensionConnected;
|
|
400
444
|
final usingExtension = prefersExtension && extensionConnected;
|
|
445
|
+
final selectedBrowserExtension = controller.browserExtensionTokens
|
|
446
|
+
.where(
|
|
447
|
+
(device) =>
|
|
448
|
+
device['tokenId'] == controller.selectedBrowserExtensionTokenId,
|
|
449
|
+
)
|
|
450
|
+
.cast<Map<String, dynamic>?>()
|
|
451
|
+
.firstWhere((device) => device != null, orElse: () => null);
|
|
401
452
|
final browserFallbackLabel = 'cloud browser runtime';
|
|
402
453
|
final browserPageInfo = browserStatus['pageInfo'] is Map<dynamic, dynamic>
|
|
403
454
|
? Map<String, dynamic>.from(browserStatus['pageInfo'] as Map)
|
|
@@ -454,12 +505,63 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
454
505
|
desktopDevices: controller.desktopDevices,
|
|
455
506
|
selectedDesktopDeviceId:
|
|
456
507
|
controller.selectedDesktopDeviceId,
|
|
508
|
+
browserExtensionDevices:
|
|
509
|
+
controller.browserExtensionTokens,
|
|
510
|
+
selectedBrowserExtensionTokenId:
|
|
511
|
+
controller.selectedBrowserExtensionTokenId,
|
|
457
512
|
browserExtensionPreferred: prefersExtension,
|
|
458
513
|
browserExtensionActive: usingExtension,
|
|
459
514
|
browserFallbackLabel: browserFallbackLabel,
|
|
460
515
|
),
|
|
461
516
|
if (_isBrowser && prefersExtension) ...<Widget>[
|
|
462
517
|
const SizedBox(height: 14),
|
|
518
|
+
if (_browserExtensionDevices.isNotEmpty) ...<Widget>[
|
|
519
|
+
DropdownButtonFormField<String>(
|
|
520
|
+
isExpanded: true,
|
|
521
|
+
initialValue: selectedBrowserExtension?['tokenId']
|
|
522
|
+
?.toString(),
|
|
523
|
+
decoration: const InputDecoration(
|
|
524
|
+
labelText: 'Chrome extension device',
|
|
525
|
+
prefixIcon: Icon(Icons.extension_outlined),
|
|
526
|
+
),
|
|
527
|
+
hint: const Text('Select a paired extension'),
|
|
528
|
+
items: _browserExtensionDevices.map((device) {
|
|
529
|
+
final tokenId = device['tokenId']?.toString() ?? '';
|
|
530
|
+
final label =
|
|
531
|
+
device['name']?.toString().trim().isNotEmpty ==
|
|
532
|
+
true
|
|
533
|
+
? device['name'].toString()
|
|
534
|
+
: tokenId;
|
|
535
|
+
final state =
|
|
536
|
+
device['online'] == true ||
|
|
537
|
+
device['connected'] == true
|
|
538
|
+
? 'online'
|
|
539
|
+
: 'offline';
|
|
540
|
+
return DropdownMenuItem<String>(
|
|
541
|
+
value: tokenId,
|
|
542
|
+
child: Text(
|
|
543
|
+
'$label · $state',
|
|
544
|
+
maxLines: 1,
|
|
545
|
+
overflow: TextOverflow.ellipsis,
|
|
546
|
+
softWrap: false,
|
|
547
|
+
),
|
|
548
|
+
);
|
|
549
|
+
}).toList(),
|
|
550
|
+
onChanged: _isCurrentSurfaceBusy
|
|
551
|
+
? null
|
|
552
|
+
: (value) {
|
|
553
|
+
if (value == null || value.isEmpty) {
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
unawaited(
|
|
557
|
+
controller.selectBrowserExtensionRuntime(
|
|
558
|
+
value,
|
|
559
|
+
),
|
|
560
|
+
);
|
|
561
|
+
},
|
|
562
|
+
),
|
|
563
|
+
const SizedBox(height: 10),
|
|
564
|
+
],
|
|
463
565
|
_ExtensionStatusBar(
|
|
464
566
|
connected: extensionConnected,
|
|
465
567
|
onDownload: controller.downloadBrowserExtension,
|
|
@@ -570,6 +672,21 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
570
672
|
surface: _surface,
|
|
571
673
|
controller: controller,
|
|
572
674
|
screenshotPath: _activeScreenshotPath,
|
|
675
|
+
streamPlatform: _isBrowser && browserStatus['launched'] == true
|
|
676
|
+
? 'browser'
|
|
677
|
+
: (_isDesktop && _desktopOnline
|
|
678
|
+
? 'desktop'
|
|
679
|
+
: (!_isBrowser && !_isDesktop && _androidOnline
|
|
680
|
+
? 'android'
|
|
681
|
+
: null)),
|
|
682
|
+
streamDeviceId: _isBrowser && browserStatus['launched'] == true
|
|
683
|
+
? 'browser'
|
|
684
|
+
: (_isDesktop && _desktopOnline
|
|
685
|
+
? (widget.controller.selectedDesktopDeviceId ??
|
|
686
|
+
(_onlineDesktopDevices.isNotEmpty ? _onlineDesktopDevices.first['deviceId']?.toString() : null))
|
|
687
|
+
: (!_isBrowser && !_isDesktop && _androidOnline
|
|
688
|
+
? _androidDeviceId
|
|
689
|
+
: null)),
|
|
573
690
|
busy: _isCurrentSurfaceBusy,
|
|
574
691
|
wakingUp: !_isBrowser && !_isDesktop && _androidStarting,
|
|
575
692
|
enabled: _isBrowser || _isDesktop || _androidOnline,
|
|
@@ -578,6 +695,7 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
578
695
|
: _desktopRequiresSelection,
|
|
579
696
|
onTapPoint: _handleTap,
|
|
580
697
|
onSwipe: _handleSwipe,
|
|
698
|
+
onHover: _handleHover,
|
|
581
699
|
onWakeRequested: _openPrimary,
|
|
582
700
|
),
|
|
583
701
|
if (!_isBrowser && !_isDesktop) ...<Widget>[
|
|
@@ -666,6 +784,8 @@ class _DeviceSurfaceHeader extends StatelessWidget {
|
|
|
666
784
|
required this.desktopRuntime,
|
|
667
785
|
required this.desktopDevices,
|
|
668
786
|
required this.selectedDesktopDeviceId,
|
|
787
|
+
required this.browserExtensionDevices,
|
|
788
|
+
required this.selectedBrowserExtensionTokenId,
|
|
669
789
|
required this.browserExtensionPreferred,
|
|
670
790
|
required this.browserExtensionActive,
|
|
671
791
|
required this.browserFallbackLabel,
|
|
@@ -679,6 +799,8 @@ class _DeviceSurfaceHeader extends StatelessWidget {
|
|
|
679
799
|
final Map<String, dynamic> desktopRuntime;
|
|
680
800
|
final List<Map<String, dynamic>> desktopDevices;
|
|
681
801
|
final String? selectedDesktopDeviceId;
|
|
802
|
+
final List<Map<String, dynamic>> browserExtensionDevices;
|
|
803
|
+
final String? selectedBrowserExtensionTokenId;
|
|
682
804
|
final bool browserExtensionPreferred;
|
|
683
805
|
final bool browserExtensionActive;
|
|
684
806
|
final String browserFallbackLabel;
|
|
@@ -694,12 +816,29 @@ class _DeviceSurfaceHeader extends StatelessWidget {
|
|
|
694
816
|
.where((device) => device['deviceId'] == selectedDesktopDeviceId)
|
|
695
817
|
.cast<Map<String, dynamic>?>()
|
|
696
818
|
.firstWhere((device) => device != null, orElse: () => null);
|
|
819
|
+
final selectedExtension = browserExtensionDevices
|
|
820
|
+
.where(
|
|
821
|
+
(device) => device['tokenId'] == selectedBrowserExtensionTokenId,
|
|
822
|
+
)
|
|
823
|
+
.cast<Map<String, dynamic>?>()
|
|
824
|
+
.firstWhere((device) => device != null, orElse: () => null);
|
|
825
|
+
final extensionOnlineCount = browserExtensionDevices
|
|
826
|
+
.where(
|
|
827
|
+
(device) =>
|
|
828
|
+
device['online'] == true || device['connected'] == true,
|
|
829
|
+
)
|
|
830
|
+
.length;
|
|
697
831
|
final desktopOnlineCount = desktopDevices
|
|
698
832
|
.where((device) => device['online'] == true)
|
|
699
833
|
.length;
|
|
700
834
|
final title = switch (surface) {
|
|
701
835
|
_DeviceSurface.browser =>
|
|
702
|
-
|
|
836
|
+
browserExtensionPreferred &&
|
|
837
|
+
selectedExtension?['name']?.toString().trim().isNotEmpty ==
|
|
838
|
+
true
|
|
839
|
+
? selectedExtension!['name'].toString()
|
|
840
|
+
: (browserPageInfo['title']?.toString().trim().isNotEmpty ??
|
|
841
|
+
false)
|
|
703
842
|
? browserPageInfo['title'].toString()
|
|
704
843
|
: 'Live Browser',
|
|
705
844
|
_DeviceSurface.android => 'Android Phone',
|
|
@@ -709,9 +848,13 @@ class _DeviceSurfaceHeader extends StatelessWidget {
|
|
|
709
848
|
: 'Desktop Companion',
|
|
710
849
|
};
|
|
711
850
|
final subtitle = switch (surface) {
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
?
|
|
851
|
+
_DeviceSurface.browser =>
|
|
852
|
+
browserExtensionPreferred && selectedExtension == null
|
|
853
|
+
? extensionOnlineCount > 1
|
|
854
|
+
? 'Multiple extension devices are online. Pick the browser you want to control.'
|
|
855
|
+
: 'No extension device is active. Using the $browserFallbackLabel.'
|
|
856
|
+
: browserExtensionPreferred && !browserExtensionActive
|
|
857
|
+
? 'Selected extension is offline. Using the $browserFallbackLabel.'
|
|
715
858
|
: (browserPageInfo['url']?.toString() ??
|
|
716
859
|
'Ready for navigation'),
|
|
717
860
|
_DeviceSurface.android =>
|
|
@@ -746,7 +889,9 @@ class _DeviceSurfaceHeader extends StatelessWidget {
|
|
|
746
889
|
: '${selectedDesktop['platform'] ?? 'desktop'} · ${selectedDesktop['hostname'] ?? 'unknown host'}',
|
|
747
890
|
};
|
|
748
891
|
final statusLabel = surface == _DeviceSurface.browser
|
|
749
|
-
? browserExtensionPreferred &&
|
|
892
|
+
? browserExtensionPreferred && selectedExtension == null
|
|
893
|
+
? (extensionOnlineCount > 0 ? 'Select Device' : 'Fallback')
|
|
894
|
+
: browserExtensionPreferred && !browserExtensionActive
|
|
750
895
|
? 'Fallback'
|
|
751
896
|
: browserExtensionActive
|
|
752
897
|
? 'Extension'
|
|
@@ -1120,10 +1265,7 @@ class _AndroidNavDock extends StatelessWidget {
|
|
|
1120
1265
|
}
|
|
1121
1266
|
|
|
1122
1267
|
class _SurfaceSwitcher extends StatelessWidget {
|
|
1123
|
-
const _SurfaceSwitcher({
|
|
1124
|
-
required this.surface,
|
|
1125
|
-
required this.onSelect,
|
|
1126
|
-
});
|
|
1268
|
+
const _SurfaceSwitcher({required this.surface, required this.onSelect});
|
|
1127
1269
|
|
|
1128
1270
|
final _DeviceSurface surface;
|
|
1129
1271
|
final Future<void> Function(_DeviceSurface) onSelect;
|
|
@@ -1157,8 +1299,7 @@ class _SurfaceSwitcher extends StatelessWidget {
|
|
|
1157
1299
|
),
|
|
1158
1300
|
labelStyle: TextStyle(
|
|
1159
1301
|
color: selected ? _textPrimary : _textSecondary,
|
|
1160
|
-
fontWeight:
|
|
1161
|
-
selected ? FontWeight.w700 : FontWeight.w500,
|
|
1302
|
+
fontWeight: selected ? FontWeight.w700 : FontWeight.w500,
|
|
1162
1303
|
),
|
|
1163
1304
|
);
|
|
1164
1305
|
}).toList(),
|
|
@@ -1179,24 +1320,30 @@ class _InteractiveSurfacePreview extends StatefulWidget {
|
|
|
1179
1320
|
required this.surface,
|
|
1180
1321
|
required this.controller,
|
|
1181
1322
|
required this.screenshotPath,
|
|
1323
|
+
required this.streamPlatform,
|
|
1324
|
+
required this.streamDeviceId,
|
|
1182
1325
|
required this.busy,
|
|
1183
1326
|
required this.wakingUp,
|
|
1184
1327
|
required this.enabled,
|
|
1185
1328
|
required this.connectRequired,
|
|
1186
1329
|
required this.onTapPoint,
|
|
1187
1330
|
required this.onSwipe,
|
|
1331
|
+
this.onHover,
|
|
1188
1332
|
required this.onWakeRequested,
|
|
1189
1333
|
});
|
|
1190
1334
|
|
|
1191
1335
|
final _DeviceSurface surface;
|
|
1192
1336
|
final NeoAgentController controller;
|
|
1193
1337
|
final String? screenshotPath;
|
|
1338
|
+
final String? streamPlatform;
|
|
1339
|
+
final String? streamDeviceId;
|
|
1194
1340
|
final bool busy;
|
|
1195
1341
|
final bool wakingUp;
|
|
1196
1342
|
final bool enabled;
|
|
1197
1343
|
final bool connectRequired;
|
|
1198
1344
|
final Future<void> Function(Offset point) onTapPoint;
|
|
1199
1345
|
final Future<void> Function(Offset start, Offset end) onSwipe;
|
|
1346
|
+
final void Function(Offset point)? onHover;
|
|
1200
1347
|
final Future<void> Function() onWakeRequested;
|
|
1201
1348
|
|
|
1202
1349
|
@override
|
|
@@ -1213,11 +1360,15 @@ class _InteractiveSurfacePreviewState
|
|
|
1213
1360
|
Object? _imageError;
|
|
1214
1361
|
Offset? _dragStart;
|
|
1215
1362
|
Offset? _dragEnd;
|
|
1363
|
+
bool _streamStarting = false;
|
|
1364
|
+
String? _activeStreamKey;
|
|
1365
|
+
String? _streamFailedKey;
|
|
1216
1366
|
|
|
1217
1367
|
@override
|
|
1218
1368
|
void initState() {
|
|
1219
1369
|
super.initState();
|
|
1220
1370
|
unawaited(_loadImage());
|
|
1371
|
+
unawaited(_syncStream());
|
|
1221
1372
|
}
|
|
1222
1373
|
|
|
1223
1374
|
@override
|
|
@@ -1226,14 +1377,95 @@ class _InteractiveSurfacePreviewState
|
|
|
1226
1377
|
if (oldWidget.screenshotPath != widget.screenshotPath) {
|
|
1227
1378
|
unawaited(_loadImage());
|
|
1228
1379
|
}
|
|
1380
|
+
if (oldWidget.streamPlatform != widget.streamPlatform ||
|
|
1381
|
+
oldWidget.streamDeviceId != widget.streamDeviceId ||
|
|
1382
|
+
oldWidget.controller.streamSocket != widget.controller.streamSocket) {
|
|
1383
|
+
unawaited(_syncStream());
|
|
1384
|
+
}
|
|
1229
1385
|
}
|
|
1230
1386
|
|
|
1231
1387
|
@override
|
|
1232
1388
|
void dispose() {
|
|
1389
|
+
unawaited(_stopActiveStream());
|
|
1233
1390
|
_detachImageListener();
|
|
1234
1391
|
super.dispose();
|
|
1235
1392
|
}
|
|
1236
1393
|
|
|
1394
|
+
String? get _requestedStreamKey {
|
|
1395
|
+
final platform = widget.streamPlatform?.trim();
|
|
1396
|
+
final deviceId = widget.streamDeviceId?.trim();
|
|
1397
|
+
if (platform == null ||
|
|
1398
|
+
platform.isEmpty ||
|
|
1399
|
+
deviceId == null ||
|
|
1400
|
+
deviceId.isEmpty ||
|
|
1401
|
+
widget.controller.streamSocket == null) {
|
|
1402
|
+
return null;
|
|
1403
|
+
}
|
|
1404
|
+
return '$platform:$deviceId';
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
Future<void> _syncStream() async {
|
|
1408
|
+
final requested = _requestedStreamKey;
|
|
1409
|
+
if (_activeStreamKey == requested || _streamStarting) {
|
|
1410
|
+
return;
|
|
1411
|
+
}
|
|
1412
|
+
if (requested != _streamFailedKey) {
|
|
1413
|
+
_streamFailedKey = null;
|
|
1414
|
+
}
|
|
1415
|
+
await _stopActiveStream();
|
|
1416
|
+
if (requested == null) {
|
|
1417
|
+
return;
|
|
1418
|
+
}
|
|
1419
|
+
final parts = requested.split(':');
|
|
1420
|
+
_streamStarting = true;
|
|
1421
|
+
try {
|
|
1422
|
+
await widget.controller.startStreamRuntime(
|
|
1423
|
+
platform: parts[0],
|
|
1424
|
+
deviceId: parts.sublist(1).join(':'),
|
|
1425
|
+
fps: 10,
|
|
1426
|
+
quality: 70,
|
|
1427
|
+
);
|
|
1428
|
+
if (mounted && _requestedStreamKey == requested) {
|
|
1429
|
+
_activeStreamKey = requested;
|
|
1430
|
+
_streamFailedKey = null;
|
|
1431
|
+
} else {
|
|
1432
|
+
await widget.controller.stopStreamRuntime(
|
|
1433
|
+
platform: parts[0],
|
|
1434
|
+
deviceId: parts.sublist(1).join(':'),
|
|
1435
|
+
);
|
|
1436
|
+
}
|
|
1437
|
+
} catch (_) {
|
|
1438
|
+
if (mounted) {
|
|
1439
|
+
_streamFailedKey = requested;
|
|
1440
|
+
}
|
|
1441
|
+
if (mounted && (widget.screenshotPath ?? '').isEmpty) {
|
|
1442
|
+
unawaited(_loadImage());
|
|
1443
|
+
}
|
|
1444
|
+
} finally {
|
|
1445
|
+
_streamStarting = false;
|
|
1446
|
+
if (mounted &&
|
|
1447
|
+
_activeStreamKey != _requestedStreamKey &&
|
|
1448
|
+
_streamFailedKey != _requestedStreamKey) {
|
|
1449
|
+
unawaited(_syncStream());
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
Future<void> _stopActiveStream() async {
|
|
1455
|
+
final active = _activeStreamKey;
|
|
1456
|
+
_activeStreamKey = null;
|
|
1457
|
+
if (active == null) {
|
|
1458
|
+
return;
|
|
1459
|
+
}
|
|
1460
|
+
final parts = active.split(':');
|
|
1461
|
+
try {
|
|
1462
|
+
await widget.controller.stopStreamRuntime(
|
|
1463
|
+
platform: parts[0],
|
|
1464
|
+
deviceId: parts.sublist(1).join(':'),
|
|
1465
|
+
);
|
|
1466
|
+
} catch (_) {}
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1237
1469
|
void _detachImageListener() {
|
|
1238
1470
|
if (_imageStream != null && _imageListener != null) {
|
|
1239
1471
|
_imageStream!.removeListener(_imageListener!);
|
|
@@ -1368,82 +1600,41 @@ class _InteractiveSurfacePreviewState
|
|
|
1368
1600
|
constraints.maxWidth,
|
|
1369
1601
|
constraints.maxHeight,
|
|
1370
1602
|
);
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
final imageBytes = _imageBytes;
|
|
1383
|
-
if (imageBytes == null) {
|
|
1384
|
-
return _EmptySurfaceState(
|
|
1385
|
-
surface: widget.surface,
|
|
1386
|
-
enabled: widget.enabled,
|
|
1387
|
-
busy: widget.busy,
|
|
1388
|
-
isLoadingPreview: _imageError == null,
|
|
1389
|
-
connectRequired: widget.connectRequired,
|
|
1390
|
-
errorMessage: _imageError?.toString(),
|
|
1391
|
-
onPressed: widget.onWakeRequested,
|
|
1392
|
-
);
|
|
1393
|
-
}
|
|
1394
|
-
return Semantics(
|
|
1395
|
-
button: true,
|
|
1396
|
-
label: 'Device surface preview — tap to interact, swipe to scroll',
|
|
1397
|
-
child: GestureDetector(
|
|
1398
|
-
onTapUp: widget.busy
|
|
1399
|
-
? null
|
|
1400
|
-
: (details) async {
|
|
1401
|
-
final point = _mapToPixels(
|
|
1402
|
-
details.localPosition,
|
|
1403
|
-
boxSize,
|
|
1404
|
-
);
|
|
1405
|
-
if (point != null) {
|
|
1406
|
-
await widget.onTapPoint(point);
|
|
1407
|
-
}
|
|
1408
|
-
},
|
|
1409
|
-
onPanStart: widget.busy
|
|
1410
|
-
? null
|
|
1411
|
-
: (details) {
|
|
1412
|
-
_dragStart = details.localPosition;
|
|
1413
|
-
_dragEnd = details.localPosition;
|
|
1414
|
-
},
|
|
1415
|
-
onPanUpdate: widget.busy
|
|
1416
|
-
? null
|
|
1417
|
-
: (details) {
|
|
1418
|
-
_dragEnd = details.localPosition;
|
|
1419
|
-
},
|
|
1420
|
-
onPanEnd: widget.busy
|
|
1421
|
-
? null
|
|
1422
|
-
: (_) async {
|
|
1423
|
-
final start = _dragStart;
|
|
1424
|
-
final end = _dragEnd;
|
|
1425
|
-
_dragStart = null;
|
|
1426
|
-
_dragEnd = null;
|
|
1427
|
-
if (start == null || end == null) {
|
|
1428
|
-
return;
|
|
1429
|
-
}
|
|
1430
|
-
if ((start - end).distance < 12) {
|
|
1431
|
-
return;
|
|
1432
|
-
}
|
|
1433
|
-
final mappedStart = _mapToPixels(start, boxSize);
|
|
1434
|
-
final mappedEnd = _mapToPixels(end, boxSize);
|
|
1435
|
-
if (mappedStart != null && mappedEnd != null) {
|
|
1436
|
-
await widget.onSwipe(mappedStart, mappedEnd);
|
|
1437
|
-
}
|
|
1438
|
-
},
|
|
1439
|
-
child: Stack(
|
|
1603
|
+
final socket = widget.controller.streamSocket;
|
|
1604
|
+
final streamPlatform = widget.streamPlatform;
|
|
1605
|
+
final streamDeviceId = widget.streamDeviceId;
|
|
1606
|
+
final streamKey = _requestedStreamKey;
|
|
1607
|
+
if (socket != null &&
|
|
1608
|
+
streamPlatform != null &&
|
|
1609
|
+
streamPlatform.isNotEmpty &&
|
|
1610
|
+
streamDeviceId != null &&
|
|
1611
|
+
streamDeviceId.isNotEmpty &&
|
|
1612
|
+
streamKey != _streamFailedKey) {
|
|
1613
|
+
return Stack(
|
|
1440
1614
|
fit: StackFit.expand,
|
|
1441
1615
|
children: <Widget>[
|
|
1442
1616
|
Container(color: _bgSecondary),
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1617
|
+
StreamRenderer(
|
|
1618
|
+
socket: socket,
|
|
1619
|
+
deviceId: streamDeviceId,
|
|
1620
|
+
platform: streamPlatform,
|
|
1621
|
+
remoteResolution: _pixelSize,
|
|
1622
|
+
onTap: widget.busy
|
|
1623
|
+
? null
|
|
1624
|
+
: (x, y) => unawaited(
|
|
1625
|
+
widget.onTapPoint(Offset(x, y)),
|
|
1626
|
+
),
|
|
1627
|
+
onSwipe: widget.busy
|
|
1628
|
+
? null
|
|
1629
|
+
: (x1, y1, x2, y2) => unawaited(
|
|
1630
|
+
widget.onSwipe(
|
|
1631
|
+
Offset(x1, y1),
|
|
1632
|
+
Offset(x2, y2),
|
|
1633
|
+
),
|
|
1634
|
+
),
|
|
1635
|
+
onHover: widget.busy
|
|
1636
|
+
? null
|
|
1637
|
+
: (x, y) => widget.onHover?.call(Offset(x, y)),
|
|
1447
1638
|
),
|
|
1448
1639
|
Positioned(
|
|
1449
1640
|
left: 12,
|
|
@@ -1469,8 +1660,115 @@ class _InteractiveSurfacePreviewState
|
|
|
1469
1660
|
if (widget.busy)
|
|
1470
1661
|
const Center(child: CircularProgressIndicator()),
|
|
1471
1662
|
],
|
|
1663
|
+
);
|
|
1664
|
+
}
|
|
1665
|
+
if (!hasImage) {
|
|
1666
|
+
return _EmptySurfaceState(
|
|
1667
|
+
surface: widget.surface,
|
|
1668
|
+
enabled: widget.enabled,
|
|
1669
|
+
busy: widget.busy,
|
|
1670
|
+
isLoadingPreview: widget.wakingUp,
|
|
1671
|
+
connectRequired: widget.connectRequired,
|
|
1672
|
+
errorMessage: _imageError?.toString(),
|
|
1673
|
+
onPressed: widget.onWakeRequested,
|
|
1674
|
+
);
|
|
1675
|
+
}
|
|
1676
|
+
final imageBytes = _imageBytes;
|
|
1677
|
+
if (imageBytes == null) {
|
|
1678
|
+
return _EmptySurfaceState(
|
|
1679
|
+
surface: widget.surface,
|
|
1680
|
+
enabled: widget.enabled,
|
|
1681
|
+
busy: widget.busy,
|
|
1682
|
+
isLoadingPreview: _imageError == null,
|
|
1683
|
+
connectRequired: widget.connectRequired,
|
|
1684
|
+
errorMessage: _imageError?.toString(),
|
|
1685
|
+
onPressed: widget.onWakeRequested,
|
|
1686
|
+
);
|
|
1687
|
+
}
|
|
1688
|
+
return Semantics(
|
|
1689
|
+
button: true,
|
|
1690
|
+
label:
|
|
1691
|
+
'Device surface preview — tap to interact, swipe to scroll',
|
|
1692
|
+
child: GestureDetector(
|
|
1693
|
+
onTapUp: widget.busy
|
|
1694
|
+
? null
|
|
1695
|
+
: (details) async {
|
|
1696
|
+
final point = _mapToPixels(
|
|
1697
|
+
details.localPosition,
|
|
1698
|
+
boxSize,
|
|
1699
|
+
);
|
|
1700
|
+
if (point != null) {
|
|
1701
|
+
await widget.onTapPoint(point);
|
|
1702
|
+
}
|
|
1703
|
+
},
|
|
1704
|
+
onPanStart: widget.busy
|
|
1705
|
+
? null
|
|
1706
|
+
: (details) {
|
|
1707
|
+
_dragStart = details.localPosition;
|
|
1708
|
+
_dragEnd = details.localPosition;
|
|
1709
|
+
},
|
|
1710
|
+
onPanUpdate: widget.busy
|
|
1711
|
+
? null
|
|
1712
|
+
: (details) {
|
|
1713
|
+
_dragEnd = details.localPosition;
|
|
1714
|
+
},
|
|
1715
|
+
onPanEnd: widget.busy
|
|
1716
|
+
? null
|
|
1717
|
+
: (_) async {
|
|
1718
|
+
final start = _dragStart;
|
|
1719
|
+
final end = _dragEnd;
|
|
1720
|
+
_dragStart = null;
|
|
1721
|
+
_dragEnd = null;
|
|
1722
|
+
if (start == null || end == null) {
|
|
1723
|
+
return;
|
|
1724
|
+
}
|
|
1725
|
+
if ((start - end).distance < 12) {
|
|
1726
|
+
return;
|
|
1727
|
+
}
|
|
1728
|
+
final mappedStart = _mapToPixels(
|
|
1729
|
+
start,
|
|
1730
|
+
boxSize,
|
|
1731
|
+
);
|
|
1732
|
+
final mappedEnd = _mapToPixels(end, boxSize);
|
|
1733
|
+
if (mappedStart != null && mappedEnd != null) {
|
|
1734
|
+
await widget.onSwipe(mappedStart, mappedEnd);
|
|
1735
|
+
}
|
|
1736
|
+
},
|
|
1737
|
+
child: Stack(
|
|
1738
|
+
fit: StackFit.expand,
|
|
1739
|
+
children: <Widget>[
|
|
1740
|
+
Container(color: _bgSecondary),
|
|
1741
|
+
Image.memory(
|
|
1742
|
+
imageBytes,
|
|
1743
|
+
fit: BoxFit.contain,
|
|
1744
|
+
gaplessPlayback: true,
|
|
1745
|
+
),
|
|
1746
|
+
Positioned(
|
|
1747
|
+
left: 12,
|
|
1748
|
+
right: 12,
|
|
1749
|
+
bottom: 12,
|
|
1750
|
+
child: Container(
|
|
1751
|
+
padding: const EdgeInsets.symmetric(
|
|
1752
|
+
horizontal: 12,
|
|
1753
|
+
vertical: 10,
|
|
1754
|
+
),
|
|
1755
|
+
decoration: BoxDecoration(
|
|
1756
|
+
color: const Color(0xB205080D),
|
|
1757
|
+
borderRadius: BorderRadius.circular(14),
|
|
1758
|
+
border: Border.all(color: _borderLight),
|
|
1759
|
+
),
|
|
1760
|
+
child: Text(
|
|
1761
|
+
widget.surface.helper,
|
|
1762
|
+
textAlign: TextAlign.center,
|
|
1763
|
+
style: TextStyle(color: _textPrimary),
|
|
1764
|
+
),
|
|
1765
|
+
),
|
|
1766
|
+
),
|
|
1767
|
+
if (widget.busy)
|
|
1768
|
+
const Center(child: CircularProgressIndicator()),
|
|
1769
|
+
],
|
|
1770
|
+
),
|
|
1472
1771
|
),
|
|
1473
|
-
),
|
|
1474
1772
|
);
|
|
1475
1773
|
},
|
|
1476
1774
|
),
|
|
@@ -1592,9 +1890,6 @@ class _EmptySurfaceState extends StatelessWidget {
|
|
|
1592
1890
|
}
|
|
1593
1891
|
}
|
|
1594
1892
|
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
1893
|
class _DeviceFieldRow extends StatelessWidget {
|
|
1599
1894
|
const _DeviceFieldRow({required this.children});
|
|
1600
1895
|
|
|
@@ -1741,7 +2036,9 @@ class _ExtensionStatusBar extends StatelessWidget {
|
|
|
1741
2036
|
child: Row(
|
|
1742
2037
|
children: <Widget>[
|
|
1743
2038
|
_DotStatus(
|
|
1744
|
-
label: connected
|
|
2039
|
+
label: connected
|
|
2040
|
+
? 'Extension connected'
|
|
2041
|
+
: 'Extension not connected',
|
|
1745
2042
|
color: connected ? _success : _warning,
|
|
1746
2043
|
),
|
|
1747
2044
|
const Spacer(),
|