neoagent 2.4.0 → 2.4.1-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +619 -21
- package/README.md +1 -1
- package/extensions/chrome-browser/background.mjs +19 -7
- package/extensions/chrome-browser/icons/icon128.png +0 -0
- package/extensions/chrome-browser/icons/icon16.png +0 -0
- package/extensions/chrome-browser/icons/icon48.png +0 -0
- package/extensions/chrome-browser/icons/logo.svg +12 -0
- package/extensions/chrome-browser/manifest.json +13 -2
- package/extensions/chrome-browser/popup.css +5 -0
- package/extensions/chrome-browser/popup.html +7 -5
- package/extensions/chrome-browser/popup.js +16 -7
- package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +721 -0
- package/flutter_app/lib/features/onboarding/onboarding_shell.dart +6 -0
- package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +1 -1
- package/flutter_app/lib/main.dart +1 -0
- package/flutter_app/lib/main_controller.dart +156 -3
- package/flutter_app/lib/main_devices.dart +485 -119
- package/flutter_app/lib/main_settings.dart +289 -30
- package/flutter_app/lib/src/backend_client.dart +89 -0
- package/flutter_app/lib/src/desktop_companion_actions.dart +153 -3
- package/flutter_app/lib/src/desktop_companion_io.dart +145 -4
- package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
- package/flutter_app/lib/src/stream_renderer.dart +286 -0
- package/flutter_app/macos/Runner/AppDelegate.swift +56 -1
- package/package.json +2 -2
- package/server/guest_agent.js +19 -1
- package/server/http/routes.js +191 -0
- package/server/http/socket.js +1 -1
- package/server/index.js +4 -1
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +75438 -74005
- package/server/routes/browser.js +14 -0
- package/server/routes/browser_extension.js +21 -4
- package/server/routes/desktop.js +10 -0
- package/server/routes/settings.js +4 -0
- package/server/routes/stream.js +187 -0
- package/server/services/ai/tools.js +40 -29
- package/server/services/android/controller.js +41 -2
- package/server/services/browser/controller.js +34 -0
- package/server/services/browser/extension/manifest.js +33 -0
- package/server/services/browser/extension/provider.js +12 -6
- package/server/services/browser/extension/registry.js +188 -18
- package/server/services/desktop/gateway.js +28 -3
- package/server/services/desktop/protocol.js +34 -0
- package/server/services/desktop/provider.js +25 -0
- package/server/services/desktop/registry.js +92 -10
- package/server/services/manager.js +19 -2
- package/server/services/runtime/backends/local-vm.js +6 -0
- package/server/services/runtime/docker-vm-manager.js +26 -3
- package/server/services/runtime/manager.js +36 -5
- package/server/services/runtime/settings.js +17 -0
- package/server/services/streaming/android-stream.js +298 -0
- package/server/services/streaming/browser-stream.js +87 -0
- package/server/services/streaming/stream-hub.js +231 -0
- package/server/services/websocket.js +73 -0
|
@@ -90,6 +90,29 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
90
90
|
bool get _androidStarting =>
|
|
91
91
|
widget.controller.androidRuntime['starting'] == true;
|
|
92
92
|
|
|
93
|
+
String? get _androidDeviceId {
|
|
94
|
+
final status = widget.controller.androidRuntime;
|
|
95
|
+
final direct = status['adbSerial']?.toString().trim();
|
|
96
|
+
if (direct != null && direct.isNotEmpty) {
|
|
97
|
+
return direct;
|
|
98
|
+
}
|
|
99
|
+
final devices = _jsonMapList(status['devices'], fallbackToMapValues: true);
|
|
100
|
+
for (final device in devices) {
|
|
101
|
+
if (device['status']?.toString() != 'device') {
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
final serial = device['serial']?.toString().trim();
|
|
105
|
+
if (serial != null && serial.isNotEmpty) {
|
|
106
|
+
return serial;
|
|
107
|
+
}
|
|
108
|
+
final id = device['deviceId']?.toString().trim();
|
|
109
|
+
if (id != null && id.isNotEmpty) {
|
|
110
|
+
return id;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
93
116
|
List<Map<String, dynamic>> get _onlineDesktopDevices => widget
|
|
94
117
|
.controller
|
|
95
118
|
.desktopDevices
|
|
@@ -103,8 +126,16 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
103
126
|
_onlineDesktopDevices.length > 1 &&
|
|
104
127
|
(widget.controller.selectedDesktopDeviceId ?? '').isEmpty;
|
|
105
128
|
|
|
129
|
+
bool get _extensionPreferredButOffline =>
|
|
130
|
+
widget.controller.browserBackend == 'extension' &&
|
|
131
|
+
!widget.controller.browserExtensionConnected;
|
|
132
|
+
|
|
133
|
+
List<Map<String, dynamic>> get _browserExtensionDevices =>
|
|
134
|
+
widget.controller.browserExtensionTokens;
|
|
135
|
+
|
|
106
136
|
String? get _activeScreenshotPath {
|
|
107
137
|
if (_isBrowser) {
|
|
138
|
+
if (_extensionPreferredButOffline) return null;
|
|
108
139
|
return widget.controller.browserScreenshotPath;
|
|
109
140
|
}
|
|
110
141
|
if (_isDesktop) {
|
|
@@ -121,6 +152,7 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
121
152
|
Future<void> _ensurePreview() async {
|
|
122
153
|
final controller = widget.controller;
|
|
123
154
|
if (_isBrowser) {
|
|
155
|
+
if (_extensionPreferredButOffline) return;
|
|
124
156
|
if (controller.browserRuntime['launched'] != true) {
|
|
125
157
|
return;
|
|
126
158
|
}
|
|
@@ -161,6 +193,7 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
161
193
|
return;
|
|
162
194
|
}
|
|
163
195
|
if (_isBrowser) {
|
|
196
|
+
if (_extensionPreferredButOffline) return;
|
|
164
197
|
await widget.controller.refreshBrowserFrameRuntime();
|
|
165
198
|
return;
|
|
166
199
|
}
|
|
@@ -289,6 +322,23 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
289
322
|
}
|
|
290
323
|
});
|
|
291
324
|
|
|
325
|
+
void _handleHover(Offset point) {
|
|
326
|
+
if (_isBrowser) {
|
|
327
|
+
unawaited(widget.controller.hoverBrowserPointRuntime(
|
|
328
|
+
x: point.dx.round(),
|
|
329
|
+
y: point.dy.round(),
|
|
330
|
+
));
|
|
331
|
+
} else if (_isDesktop) {
|
|
332
|
+
if (_desktopRequiresSelection) {
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
unawaited(widget.controller.hoverDesktopRuntime(
|
|
336
|
+
x: point.dx.round(),
|
|
337
|
+
y: point.dy.round(),
|
|
338
|
+
));
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
292
342
|
Future<void> _handleTap(Offset point) => _runOnSurface(() async {
|
|
293
343
|
if (_isBrowser) {
|
|
294
344
|
await widget.controller.clickBrowserPointRuntime(
|
|
@@ -317,36 +367,37 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
317
367
|
});
|
|
318
368
|
});
|
|
319
369
|
|
|
320
|
-
Future<void> _handleSwipe(Offset start, Offset end) =>
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
370
|
+
Future<void> _handleSwipe(Offset start, Offset end) =>
|
|
371
|
+
_runOnSurface(() async {
|
|
372
|
+
if (_isBrowser) {
|
|
373
|
+
await widget.controller.scrollBrowserRuntime(
|
|
374
|
+
deltaY: (start.dy - end.dy).round(),
|
|
375
|
+
);
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
if (_isDesktop) {
|
|
379
|
+
if (_desktopRequiresSelection) {
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
await widget.controller.dragDesktopRuntime(
|
|
383
|
+
x1: start.dx.round(),
|
|
384
|
+
y1: start.dy.round(),
|
|
385
|
+
x2: end.dx.round(),
|
|
386
|
+
y2: end.dy.round(),
|
|
387
|
+
);
|
|
388
|
+
return;
|
|
389
|
+
}
|
|
390
|
+
if (!_androidOnline) {
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
await widget.controller.swipeAndroidRuntime(<String, dynamic>{
|
|
394
|
+
'x1': start.dx.round(),
|
|
395
|
+
'y1': start.dy.round(),
|
|
396
|
+
'x2': end.dx.round(),
|
|
397
|
+
'y2': end.dy.round(),
|
|
398
|
+
'durationMs': 280,
|
|
399
|
+
});
|
|
400
|
+
});
|
|
350
401
|
|
|
351
402
|
Future<void> _runQuickAction(String action) => _runOnSurface(() async {
|
|
352
403
|
final controller = widget.controller;
|
|
@@ -391,6 +442,13 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
391
442
|
final prefersExtension = controller.browserBackend == 'extension';
|
|
392
443
|
final extensionConnected = controller.browserExtensionConnected;
|
|
393
444
|
final usingExtension = prefersExtension && extensionConnected;
|
|
445
|
+
final selectedBrowserExtension = controller.browserExtensionTokens
|
|
446
|
+
.where(
|
|
447
|
+
(device) =>
|
|
448
|
+
device['tokenId'] == controller.selectedBrowserExtensionTokenId,
|
|
449
|
+
)
|
|
450
|
+
.cast<Map<String, dynamic>?>()
|
|
451
|
+
.firstWhere((device) => device != null, orElse: () => null);
|
|
394
452
|
final browserFallbackLabel = 'cloud browser runtime';
|
|
395
453
|
final browserPageInfo = browserStatus['pageInfo'] is Map<dynamic, dynamic>
|
|
396
454
|
? Map<String, dynamic>.from(browserStatus['pageInfo'] as Map)
|
|
@@ -447,10 +505,69 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
447
505
|
desktopDevices: controller.desktopDevices,
|
|
448
506
|
selectedDesktopDeviceId:
|
|
449
507
|
controller.selectedDesktopDeviceId,
|
|
508
|
+
browserExtensionDevices:
|
|
509
|
+
controller.browserExtensionTokens,
|
|
510
|
+
selectedBrowserExtensionTokenId:
|
|
511
|
+
controller.selectedBrowserExtensionTokenId,
|
|
450
512
|
browserExtensionPreferred: prefersExtension,
|
|
451
513
|
browserExtensionActive: usingExtension,
|
|
452
514
|
browserFallbackLabel: browserFallbackLabel,
|
|
453
515
|
),
|
|
516
|
+
if (_isBrowser && prefersExtension) ...<Widget>[
|
|
517
|
+
const SizedBox(height: 14),
|
|
518
|
+
if (_browserExtensionDevices.isNotEmpty) ...<Widget>[
|
|
519
|
+
DropdownButtonFormField<String>(
|
|
520
|
+
isExpanded: true,
|
|
521
|
+
initialValue: selectedBrowserExtension?['tokenId']
|
|
522
|
+
?.toString(),
|
|
523
|
+
decoration: const InputDecoration(
|
|
524
|
+
labelText: 'Chrome extension device',
|
|
525
|
+
prefixIcon: Icon(Icons.extension_outlined),
|
|
526
|
+
),
|
|
527
|
+
hint: const Text('Select a paired extension'),
|
|
528
|
+
items: _browserExtensionDevices.map((device) {
|
|
529
|
+
final tokenId = device['tokenId']?.toString() ?? '';
|
|
530
|
+
final label =
|
|
531
|
+
device['name']?.toString().trim().isNotEmpty ==
|
|
532
|
+
true
|
|
533
|
+
? device['name'].toString()
|
|
534
|
+
: tokenId;
|
|
535
|
+
final state =
|
|
536
|
+
device['online'] == true ||
|
|
537
|
+
device['connected'] == true
|
|
538
|
+
? 'online'
|
|
539
|
+
: 'offline';
|
|
540
|
+
return DropdownMenuItem<String>(
|
|
541
|
+
value: tokenId,
|
|
542
|
+
child: Text(
|
|
543
|
+
'$label · $state',
|
|
544
|
+
maxLines: 1,
|
|
545
|
+
overflow: TextOverflow.ellipsis,
|
|
546
|
+
softWrap: false,
|
|
547
|
+
),
|
|
548
|
+
);
|
|
549
|
+
}).toList(),
|
|
550
|
+
onChanged: _isCurrentSurfaceBusy
|
|
551
|
+
? null
|
|
552
|
+
: (value) {
|
|
553
|
+
if (value == null || value.isEmpty) {
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
unawaited(
|
|
557
|
+
controller.selectBrowserExtensionRuntime(
|
|
558
|
+
value,
|
|
559
|
+
),
|
|
560
|
+
);
|
|
561
|
+
},
|
|
562
|
+
),
|
|
563
|
+
const SizedBox(height: 10),
|
|
564
|
+
],
|
|
565
|
+
_ExtensionStatusBar(
|
|
566
|
+
connected: extensionConnected,
|
|
567
|
+
onDownload: controller.downloadBrowserExtension,
|
|
568
|
+
onRefresh: controller.refreshBrowserExtensionStatus,
|
|
569
|
+
),
|
|
570
|
+
],
|
|
454
571
|
if (_isDesktop) ...<Widget>[
|
|
455
572
|
const SizedBox(height: 14),
|
|
456
573
|
DropdownButtonFormField<String>(
|
|
@@ -555,12 +672,30 @@ class _DevicesPanelState extends State<DevicesPanel> {
|
|
|
555
672
|
surface: _surface,
|
|
556
673
|
controller: controller,
|
|
557
674
|
screenshotPath: _activeScreenshotPath,
|
|
675
|
+
streamPlatform: _isBrowser && browserStatus['launched'] == true
|
|
676
|
+
? 'browser'
|
|
677
|
+
: (_isDesktop && _desktopOnline
|
|
678
|
+
? 'desktop'
|
|
679
|
+
: (!_isBrowser && !_isDesktop && _androidOnline
|
|
680
|
+
? 'android'
|
|
681
|
+
: null)),
|
|
682
|
+
streamDeviceId: _isBrowser && browserStatus['launched'] == true
|
|
683
|
+
? 'browser'
|
|
684
|
+
: (_isDesktop && _desktopOnline
|
|
685
|
+
? (widget.controller.selectedDesktopDeviceId ??
|
|
686
|
+
(_onlineDesktopDevices.isNotEmpty ? _onlineDesktopDevices.first['deviceId']?.toString() : null))
|
|
687
|
+
: (!_isBrowser && !_isDesktop && _androidOnline
|
|
688
|
+
? _androidDeviceId
|
|
689
|
+
: null)),
|
|
558
690
|
busy: _isCurrentSurfaceBusy,
|
|
559
691
|
wakingUp: !_isBrowser && !_isDesktop && _androidStarting,
|
|
560
692
|
enabled: _isBrowser || _isDesktop || _androidOnline,
|
|
561
|
-
connectRequired:
|
|
693
|
+
connectRequired: _isBrowser
|
|
694
|
+
? _extensionPreferredButOffline
|
|
695
|
+
: _desktopRequiresSelection,
|
|
562
696
|
onTapPoint: _handleTap,
|
|
563
697
|
onSwipe: _handleSwipe,
|
|
698
|
+
onHover: _handleHover,
|
|
564
699
|
onWakeRequested: _openPrimary,
|
|
565
700
|
),
|
|
566
701
|
if (!_isBrowser && !_isDesktop) ...<Widget>[
|
|
@@ -649,6 +784,8 @@ class _DeviceSurfaceHeader extends StatelessWidget {
|
|
|
649
784
|
required this.desktopRuntime,
|
|
650
785
|
required this.desktopDevices,
|
|
651
786
|
required this.selectedDesktopDeviceId,
|
|
787
|
+
required this.browserExtensionDevices,
|
|
788
|
+
required this.selectedBrowserExtensionTokenId,
|
|
652
789
|
required this.browserExtensionPreferred,
|
|
653
790
|
required this.browserExtensionActive,
|
|
654
791
|
required this.browserFallbackLabel,
|
|
@@ -662,6 +799,8 @@ class _DeviceSurfaceHeader extends StatelessWidget {
|
|
|
662
799
|
final Map<String, dynamic> desktopRuntime;
|
|
663
800
|
final List<Map<String, dynamic>> desktopDevices;
|
|
664
801
|
final String? selectedDesktopDeviceId;
|
|
802
|
+
final List<Map<String, dynamic>> browserExtensionDevices;
|
|
803
|
+
final String? selectedBrowserExtensionTokenId;
|
|
665
804
|
final bool browserExtensionPreferred;
|
|
666
805
|
final bool browserExtensionActive;
|
|
667
806
|
final String browserFallbackLabel;
|
|
@@ -677,12 +816,29 @@ class _DeviceSurfaceHeader extends StatelessWidget {
|
|
|
677
816
|
.where((device) => device['deviceId'] == selectedDesktopDeviceId)
|
|
678
817
|
.cast<Map<String, dynamic>?>()
|
|
679
818
|
.firstWhere((device) => device != null, orElse: () => null);
|
|
819
|
+
final selectedExtension = browserExtensionDevices
|
|
820
|
+
.where(
|
|
821
|
+
(device) => device['tokenId'] == selectedBrowserExtensionTokenId,
|
|
822
|
+
)
|
|
823
|
+
.cast<Map<String, dynamic>?>()
|
|
824
|
+
.firstWhere((device) => device != null, orElse: () => null);
|
|
825
|
+
final extensionOnlineCount = browserExtensionDevices
|
|
826
|
+
.where(
|
|
827
|
+
(device) =>
|
|
828
|
+
device['online'] == true || device['connected'] == true,
|
|
829
|
+
)
|
|
830
|
+
.length;
|
|
680
831
|
final desktopOnlineCount = desktopDevices
|
|
681
832
|
.where((device) => device['online'] == true)
|
|
682
833
|
.length;
|
|
683
834
|
final title = switch (surface) {
|
|
684
835
|
_DeviceSurface.browser =>
|
|
685
|
-
|
|
836
|
+
browserExtensionPreferred &&
|
|
837
|
+
selectedExtension?['name']?.toString().trim().isNotEmpty ==
|
|
838
|
+
true
|
|
839
|
+
? selectedExtension!['name'].toString()
|
|
840
|
+
: (browserPageInfo['title']?.toString().trim().isNotEmpty ??
|
|
841
|
+
false)
|
|
686
842
|
? browserPageInfo['title'].toString()
|
|
687
843
|
: 'Live Browser',
|
|
688
844
|
_DeviceSurface.android => 'Android Phone',
|
|
@@ -692,9 +848,13 @@ class _DeviceSurfaceHeader extends StatelessWidget {
|
|
|
692
848
|
: 'Desktop Companion',
|
|
693
849
|
};
|
|
694
850
|
final subtitle = switch (surface) {
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
?
|
|
851
|
+
_DeviceSurface.browser =>
|
|
852
|
+
browserExtensionPreferred && selectedExtension == null
|
|
853
|
+
? extensionOnlineCount > 1
|
|
854
|
+
? 'Multiple extension devices are online. Pick the browser you want to control.'
|
|
855
|
+
: 'No extension device is active. Using the $browserFallbackLabel.'
|
|
856
|
+
: browserExtensionPreferred && !browserExtensionActive
|
|
857
|
+
? 'Selected extension is offline. Using the $browserFallbackLabel.'
|
|
698
858
|
: (browserPageInfo['url']?.toString() ??
|
|
699
859
|
'Ready for navigation'),
|
|
700
860
|
_DeviceSurface.android =>
|
|
@@ -729,7 +889,9 @@ class _DeviceSurfaceHeader extends StatelessWidget {
|
|
|
729
889
|
: '${selectedDesktop['platform'] ?? 'desktop'} · ${selectedDesktop['hostname'] ?? 'unknown host'}',
|
|
730
890
|
};
|
|
731
891
|
final statusLabel = surface == _DeviceSurface.browser
|
|
732
|
-
? browserExtensionPreferred &&
|
|
892
|
+
? browserExtensionPreferred && selectedExtension == null
|
|
893
|
+
? (extensionOnlineCount > 0 ? 'Select Device' : 'Fallback')
|
|
894
|
+
: browserExtensionPreferred && !browserExtensionActive
|
|
733
895
|
? 'Fallback'
|
|
734
896
|
: browserExtensionActive
|
|
735
897
|
? 'Extension'
|
|
@@ -1103,10 +1265,7 @@ class _AndroidNavDock extends StatelessWidget {
|
|
|
1103
1265
|
}
|
|
1104
1266
|
|
|
1105
1267
|
class _SurfaceSwitcher extends StatelessWidget {
|
|
1106
|
-
const _SurfaceSwitcher({
|
|
1107
|
-
required this.surface,
|
|
1108
|
-
required this.onSelect,
|
|
1109
|
-
});
|
|
1268
|
+
const _SurfaceSwitcher({required this.surface, required this.onSelect});
|
|
1110
1269
|
|
|
1111
1270
|
final _DeviceSurface surface;
|
|
1112
1271
|
final Future<void> Function(_DeviceSurface) onSelect;
|
|
@@ -1140,8 +1299,7 @@ class _SurfaceSwitcher extends StatelessWidget {
|
|
|
1140
1299
|
),
|
|
1141
1300
|
labelStyle: TextStyle(
|
|
1142
1301
|
color: selected ? _textPrimary : _textSecondary,
|
|
1143
|
-
fontWeight:
|
|
1144
|
-
selected ? FontWeight.w700 : FontWeight.w500,
|
|
1302
|
+
fontWeight: selected ? FontWeight.w700 : FontWeight.w500,
|
|
1145
1303
|
),
|
|
1146
1304
|
);
|
|
1147
1305
|
}).toList(),
|
|
@@ -1162,24 +1320,30 @@ class _InteractiveSurfacePreview extends StatefulWidget {
|
|
|
1162
1320
|
required this.surface,
|
|
1163
1321
|
required this.controller,
|
|
1164
1322
|
required this.screenshotPath,
|
|
1323
|
+
required this.streamPlatform,
|
|
1324
|
+
required this.streamDeviceId,
|
|
1165
1325
|
required this.busy,
|
|
1166
1326
|
required this.wakingUp,
|
|
1167
1327
|
required this.enabled,
|
|
1168
1328
|
required this.connectRequired,
|
|
1169
1329
|
required this.onTapPoint,
|
|
1170
1330
|
required this.onSwipe,
|
|
1331
|
+
this.onHover,
|
|
1171
1332
|
required this.onWakeRequested,
|
|
1172
1333
|
});
|
|
1173
1334
|
|
|
1174
1335
|
final _DeviceSurface surface;
|
|
1175
1336
|
final NeoAgentController controller;
|
|
1176
1337
|
final String? screenshotPath;
|
|
1338
|
+
final String? streamPlatform;
|
|
1339
|
+
final String? streamDeviceId;
|
|
1177
1340
|
final bool busy;
|
|
1178
1341
|
final bool wakingUp;
|
|
1179
1342
|
final bool enabled;
|
|
1180
1343
|
final bool connectRequired;
|
|
1181
1344
|
final Future<void> Function(Offset point) onTapPoint;
|
|
1182
1345
|
final Future<void> Function(Offset start, Offset end) onSwipe;
|
|
1346
|
+
final void Function(Offset point)? onHover;
|
|
1183
1347
|
final Future<void> Function() onWakeRequested;
|
|
1184
1348
|
|
|
1185
1349
|
@override
|
|
@@ -1196,11 +1360,15 @@ class _InteractiveSurfacePreviewState
|
|
|
1196
1360
|
Object? _imageError;
|
|
1197
1361
|
Offset? _dragStart;
|
|
1198
1362
|
Offset? _dragEnd;
|
|
1363
|
+
bool _streamStarting = false;
|
|
1364
|
+
String? _activeStreamKey;
|
|
1365
|
+
String? _streamFailedKey;
|
|
1199
1366
|
|
|
1200
1367
|
@override
|
|
1201
1368
|
void initState() {
|
|
1202
1369
|
super.initState();
|
|
1203
1370
|
unawaited(_loadImage());
|
|
1371
|
+
unawaited(_syncStream());
|
|
1204
1372
|
}
|
|
1205
1373
|
|
|
1206
1374
|
@override
|
|
@@ -1209,14 +1377,95 @@ class _InteractiveSurfacePreviewState
|
|
|
1209
1377
|
if (oldWidget.screenshotPath != widget.screenshotPath) {
|
|
1210
1378
|
unawaited(_loadImage());
|
|
1211
1379
|
}
|
|
1380
|
+
if (oldWidget.streamPlatform != widget.streamPlatform ||
|
|
1381
|
+
oldWidget.streamDeviceId != widget.streamDeviceId ||
|
|
1382
|
+
oldWidget.controller.streamSocket != widget.controller.streamSocket) {
|
|
1383
|
+
unawaited(_syncStream());
|
|
1384
|
+
}
|
|
1212
1385
|
}
|
|
1213
1386
|
|
|
1214
1387
|
@override
|
|
1215
1388
|
void dispose() {
|
|
1389
|
+
unawaited(_stopActiveStream());
|
|
1216
1390
|
_detachImageListener();
|
|
1217
1391
|
super.dispose();
|
|
1218
1392
|
}
|
|
1219
1393
|
|
|
1394
|
+
String? get _requestedStreamKey {
|
|
1395
|
+
final platform = widget.streamPlatform?.trim();
|
|
1396
|
+
final deviceId = widget.streamDeviceId?.trim();
|
|
1397
|
+
if (platform == null ||
|
|
1398
|
+
platform.isEmpty ||
|
|
1399
|
+
deviceId == null ||
|
|
1400
|
+
deviceId.isEmpty ||
|
|
1401
|
+
widget.controller.streamSocket == null) {
|
|
1402
|
+
return null;
|
|
1403
|
+
}
|
|
1404
|
+
return '$platform:$deviceId';
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
Future<void> _syncStream() async {
|
|
1408
|
+
final requested = _requestedStreamKey;
|
|
1409
|
+
if (_activeStreamKey == requested || _streamStarting) {
|
|
1410
|
+
return;
|
|
1411
|
+
}
|
|
1412
|
+
if (requested != _streamFailedKey) {
|
|
1413
|
+
_streamFailedKey = null;
|
|
1414
|
+
}
|
|
1415
|
+
await _stopActiveStream();
|
|
1416
|
+
if (requested == null) {
|
|
1417
|
+
return;
|
|
1418
|
+
}
|
|
1419
|
+
final parts = requested.split(':');
|
|
1420
|
+
_streamStarting = true;
|
|
1421
|
+
try {
|
|
1422
|
+
await widget.controller.startStreamRuntime(
|
|
1423
|
+
platform: parts[0],
|
|
1424
|
+
deviceId: parts.sublist(1).join(':'),
|
|
1425
|
+
fps: 10,
|
|
1426
|
+
quality: 70,
|
|
1427
|
+
);
|
|
1428
|
+
if (mounted && _requestedStreamKey == requested) {
|
|
1429
|
+
_activeStreamKey = requested;
|
|
1430
|
+
_streamFailedKey = null;
|
|
1431
|
+
} else {
|
|
1432
|
+
await widget.controller.stopStreamRuntime(
|
|
1433
|
+
platform: parts[0],
|
|
1434
|
+
deviceId: parts.sublist(1).join(':'),
|
|
1435
|
+
);
|
|
1436
|
+
}
|
|
1437
|
+
} catch (_) {
|
|
1438
|
+
if (mounted) {
|
|
1439
|
+
_streamFailedKey = requested;
|
|
1440
|
+
}
|
|
1441
|
+
if (mounted && (widget.screenshotPath ?? '').isEmpty) {
|
|
1442
|
+
unawaited(_loadImage());
|
|
1443
|
+
}
|
|
1444
|
+
} finally {
|
|
1445
|
+
_streamStarting = false;
|
|
1446
|
+
if (mounted &&
|
|
1447
|
+
_activeStreamKey != _requestedStreamKey &&
|
|
1448
|
+
_streamFailedKey != _requestedStreamKey) {
|
|
1449
|
+
unawaited(_syncStream());
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
Future<void> _stopActiveStream() async {
|
|
1455
|
+
final active = _activeStreamKey;
|
|
1456
|
+
_activeStreamKey = null;
|
|
1457
|
+
if (active == null) {
|
|
1458
|
+
return;
|
|
1459
|
+
}
|
|
1460
|
+
final parts = active.split(':');
|
|
1461
|
+
try {
|
|
1462
|
+
await widget.controller.stopStreamRuntime(
|
|
1463
|
+
platform: parts[0],
|
|
1464
|
+
deviceId: parts.sublist(1).join(':'),
|
|
1465
|
+
);
|
|
1466
|
+
} catch (_) {}
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1220
1469
|
void _detachImageListener() {
|
|
1221
1470
|
if (_imageStream != null && _imageListener != null) {
|
|
1222
1471
|
_imageStream!.removeListener(_imageListener!);
|
|
@@ -1351,82 +1600,41 @@ class _InteractiveSurfacePreviewState
|
|
|
1351
1600
|
constraints.maxWidth,
|
|
1352
1601
|
constraints.maxHeight,
|
|
1353
1602
|
);
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
final imageBytes = _imageBytes;
|
|
1366
|
-
if (imageBytes == null) {
|
|
1367
|
-
return _EmptySurfaceState(
|
|
1368
|
-
surface: widget.surface,
|
|
1369
|
-
enabled: widget.enabled,
|
|
1370
|
-
busy: widget.busy,
|
|
1371
|
-
isLoadingPreview: _imageError == null,
|
|
1372
|
-
connectRequired: widget.connectRequired,
|
|
1373
|
-
errorMessage: _imageError?.toString(),
|
|
1374
|
-
onPressed: widget.onWakeRequested,
|
|
1375
|
-
);
|
|
1376
|
-
}
|
|
1377
|
-
return Semantics(
|
|
1378
|
-
button: true,
|
|
1379
|
-
label: 'Device surface preview — tap to interact, swipe to scroll',
|
|
1380
|
-
child: GestureDetector(
|
|
1381
|
-
onTapUp: widget.busy
|
|
1382
|
-
? null
|
|
1383
|
-
: (details) async {
|
|
1384
|
-
final point = _mapToPixels(
|
|
1385
|
-
details.localPosition,
|
|
1386
|
-
boxSize,
|
|
1387
|
-
);
|
|
1388
|
-
if (point != null) {
|
|
1389
|
-
await widget.onTapPoint(point);
|
|
1390
|
-
}
|
|
1391
|
-
},
|
|
1392
|
-
onPanStart: widget.busy
|
|
1393
|
-
? null
|
|
1394
|
-
: (details) {
|
|
1395
|
-
_dragStart = details.localPosition;
|
|
1396
|
-
_dragEnd = details.localPosition;
|
|
1397
|
-
},
|
|
1398
|
-
onPanUpdate: widget.busy
|
|
1399
|
-
? null
|
|
1400
|
-
: (details) {
|
|
1401
|
-
_dragEnd = details.localPosition;
|
|
1402
|
-
},
|
|
1403
|
-
onPanEnd: widget.busy
|
|
1404
|
-
? null
|
|
1405
|
-
: (_) async {
|
|
1406
|
-
final start = _dragStart;
|
|
1407
|
-
final end = _dragEnd;
|
|
1408
|
-
_dragStart = null;
|
|
1409
|
-
_dragEnd = null;
|
|
1410
|
-
if (start == null || end == null) {
|
|
1411
|
-
return;
|
|
1412
|
-
}
|
|
1413
|
-
if ((start - end).distance < 12) {
|
|
1414
|
-
return;
|
|
1415
|
-
}
|
|
1416
|
-
final mappedStart = _mapToPixels(start, boxSize);
|
|
1417
|
-
final mappedEnd = _mapToPixels(end, boxSize);
|
|
1418
|
-
if (mappedStart != null && mappedEnd != null) {
|
|
1419
|
-
await widget.onSwipe(mappedStart, mappedEnd);
|
|
1420
|
-
}
|
|
1421
|
-
},
|
|
1422
|
-
child: Stack(
|
|
1603
|
+
final socket = widget.controller.streamSocket;
|
|
1604
|
+
final streamPlatform = widget.streamPlatform;
|
|
1605
|
+
final streamDeviceId = widget.streamDeviceId;
|
|
1606
|
+
final streamKey = _requestedStreamKey;
|
|
1607
|
+
if (socket != null &&
|
|
1608
|
+
streamPlatform != null &&
|
|
1609
|
+
streamPlatform.isNotEmpty &&
|
|
1610
|
+
streamDeviceId != null &&
|
|
1611
|
+
streamDeviceId.isNotEmpty &&
|
|
1612
|
+
streamKey != _streamFailedKey) {
|
|
1613
|
+
return Stack(
|
|
1423
1614
|
fit: StackFit.expand,
|
|
1424
1615
|
children: <Widget>[
|
|
1425
1616
|
Container(color: _bgSecondary),
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1617
|
+
StreamRenderer(
|
|
1618
|
+
socket: socket,
|
|
1619
|
+
deviceId: streamDeviceId,
|
|
1620
|
+
platform: streamPlatform,
|
|
1621
|
+
remoteResolution: _pixelSize,
|
|
1622
|
+
onTap: widget.busy
|
|
1623
|
+
? null
|
|
1624
|
+
: (x, y) => unawaited(
|
|
1625
|
+
widget.onTapPoint(Offset(x, y)),
|
|
1626
|
+
),
|
|
1627
|
+
onSwipe: widget.busy
|
|
1628
|
+
? null
|
|
1629
|
+
: (x1, y1, x2, y2) => unawaited(
|
|
1630
|
+
widget.onSwipe(
|
|
1631
|
+
Offset(x1, y1),
|
|
1632
|
+
Offset(x2, y2),
|
|
1633
|
+
),
|
|
1634
|
+
),
|
|
1635
|
+
onHover: widget.busy
|
|
1636
|
+
? null
|
|
1637
|
+
: (x, y) => widget.onHover?.call(Offset(x, y)),
|
|
1430
1638
|
),
|
|
1431
1639
|
Positioned(
|
|
1432
1640
|
left: 12,
|
|
@@ -1452,8 +1660,115 @@ class _InteractiveSurfacePreviewState
|
|
|
1452
1660
|
if (widget.busy)
|
|
1453
1661
|
const Center(child: CircularProgressIndicator()),
|
|
1454
1662
|
],
|
|
1663
|
+
);
|
|
1664
|
+
}
|
|
1665
|
+
if (!hasImage) {
|
|
1666
|
+
return _EmptySurfaceState(
|
|
1667
|
+
surface: widget.surface,
|
|
1668
|
+
enabled: widget.enabled,
|
|
1669
|
+
busy: widget.busy,
|
|
1670
|
+
isLoadingPreview: widget.wakingUp,
|
|
1671
|
+
connectRequired: widget.connectRequired,
|
|
1672
|
+
errorMessage: _imageError?.toString(),
|
|
1673
|
+
onPressed: widget.onWakeRequested,
|
|
1674
|
+
);
|
|
1675
|
+
}
|
|
1676
|
+
final imageBytes = _imageBytes;
|
|
1677
|
+
if (imageBytes == null) {
|
|
1678
|
+
return _EmptySurfaceState(
|
|
1679
|
+
surface: widget.surface,
|
|
1680
|
+
enabled: widget.enabled,
|
|
1681
|
+
busy: widget.busy,
|
|
1682
|
+
isLoadingPreview: _imageError == null,
|
|
1683
|
+
connectRequired: widget.connectRequired,
|
|
1684
|
+
errorMessage: _imageError?.toString(),
|
|
1685
|
+
onPressed: widget.onWakeRequested,
|
|
1686
|
+
);
|
|
1687
|
+
}
|
|
1688
|
+
return Semantics(
|
|
1689
|
+
button: true,
|
|
1690
|
+
label:
|
|
1691
|
+
'Device surface preview — tap to interact, swipe to scroll',
|
|
1692
|
+
child: GestureDetector(
|
|
1693
|
+
onTapUp: widget.busy
|
|
1694
|
+
? null
|
|
1695
|
+
: (details) async {
|
|
1696
|
+
final point = _mapToPixels(
|
|
1697
|
+
details.localPosition,
|
|
1698
|
+
boxSize,
|
|
1699
|
+
);
|
|
1700
|
+
if (point != null) {
|
|
1701
|
+
await widget.onTapPoint(point);
|
|
1702
|
+
}
|
|
1703
|
+
},
|
|
1704
|
+
onPanStart: widget.busy
|
|
1705
|
+
? null
|
|
1706
|
+
: (details) {
|
|
1707
|
+
_dragStart = details.localPosition;
|
|
1708
|
+
_dragEnd = details.localPosition;
|
|
1709
|
+
},
|
|
1710
|
+
onPanUpdate: widget.busy
|
|
1711
|
+
? null
|
|
1712
|
+
: (details) {
|
|
1713
|
+
_dragEnd = details.localPosition;
|
|
1714
|
+
},
|
|
1715
|
+
onPanEnd: widget.busy
|
|
1716
|
+
? null
|
|
1717
|
+
: (_) async {
|
|
1718
|
+
final start = _dragStart;
|
|
1719
|
+
final end = _dragEnd;
|
|
1720
|
+
_dragStart = null;
|
|
1721
|
+
_dragEnd = null;
|
|
1722
|
+
if (start == null || end == null) {
|
|
1723
|
+
return;
|
|
1724
|
+
}
|
|
1725
|
+
if ((start - end).distance < 12) {
|
|
1726
|
+
return;
|
|
1727
|
+
}
|
|
1728
|
+
final mappedStart = _mapToPixels(
|
|
1729
|
+
start,
|
|
1730
|
+
boxSize,
|
|
1731
|
+
);
|
|
1732
|
+
final mappedEnd = _mapToPixels(end, boxSize);
|
|
1733
|
+
if (mappedStart != null && mappedEnd != null) {
|
|
1734
|
+
await widget.onSwipe(mappedStart, mappedEnd);
|
|
1735
|
+
}
|
|
1736
|
+
},
|
|
1737
|
+
child: Stack(
|
|
1738
|
+
fit: StackFit.expand,
|
|
1739
|
+
children: <Widget>[
|
|
1740
|
+
Container(color: _bgSecondary),
|
|
1741
|
+
Image.memory(
|
|
1742
|
+
imageBytes,
|
|
1743
|
+
fit: BoxFit.contain,
|
|
1744
|
+
gaplessPlayback: true,
|
|
1745
|
+
),
|
|
1746
|
+
Positioned(
|
|
1747
|
+
left: 12,
|
|
1748
|
+
right: 12,
|
|
1749
|
+
bottom: 12,
|
|
1750
|
+
child: Container(
|
|
1751
|
+
padding: const EdgeInsets.symmetric(
|
|
1752
|
+
horizontal: 12,
|
|
1753
|
+
vertical: 10,
|
|
1754
|
+
),
|
|
1755
|
+
decoration: BoxDecoration(
|
|
1756
|
+
color: const Color(0xB205080D),
|
|
1757
|
+
borderRadius: BorderRadius.circular(14),
|
|
1758
|
+
border: Border.all(color: _borderLight),
|
|
1759
|
+
),
|
|
1760
|
+
child: Text(
|
|
1761
|
+
widget.surface.helper,
|
|
1762
|
+
textAlign: TextAlign.center,
|
|
1763
|
+
style: TextStyle(color: _textPrimary),
|
|
1764
|
+
),
|
|
1765
|
+
),
|
|
1766
|
+
),
|
|
1767
|
+
if (widget.busy)
|
|
1768
|
+
const Center(child: CircularProgressIndicator()),
|
|
1769
|
+
],
|
|
1770
|
+
),
|
|
1455
1771
|
),
|
|
1456
|
-
),
|
|
1457
1772
|
);
|
|
1458
1773
|
},
|
|
1459
1774
|
),
|
|
@@ -1575,9 +1890,6 @@ class _EmptySurfaceState extends StatelessWidget {
|
|
|
1575
1890
|
}
|
|
1576
1891
|
}
|
|
1577
1892
|
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
1893
|
class _DeviceFieldRow extends StatelessWidget {
|
|
1582
1894
|
const _DeviceFieldRow({required this.children});
|
|
1583
1895
|
|
|
@@ -1701,6 +2013,60 @@ class _RuntimePreview extends StatelessWidget {
|
|
|
1701
2013
|
}
|
|
1702
2014
|
}
|
|
1703
2015
|
|
|
2016
|
+
class _ExtensionStatusBar extends StatelessWidget {
|
|
2017
|
+
const _ExtensionStatusBar({
|
|
2018
|
+
required this.connected,
|
|
2019
|
+
required this.onDownload,
|
|
2020
|
+
required this.onRefresh,
|
|
2021
|
+
});
|
|
2022
|
+
|
|
2023
|
+
final bool connected;
|
|
2024
|
+
final Future<void> Function() onDownload;
|
|
2025
|
+
final Future<void> Function() onRefresh;
|
|
2026
|
+
|
|
2027
|
+
@override
|
|
2028
|
+
Widget build(BuildContext context) {
|
|
2029
|
+
return Container(
|
|
2030
|
+
padding: const EdgeInsets.symmetric(horizontal: 14, vertical: 12),
|
|
2031
|
+
decoration: BoxDecoration(
|
|
2032
|
+
color: _bgSecondary,
|
|
2033
|
+
borderRadius: BorderRadius.circular(16),
|
|
2034
|
+
border: Border.all(color: _borderLight),
|
|
2035
|
+
),
|
|
2036
|
+
child: Row(
|
|
2037
|
+
children: <Widget>[
|
|
2038
|
+
_DotStatus(
|
|
2039
|
+
label: connected
|
|
2040
|
+
? 'Extension connected'
|
|
2041
|
+
: 'Extension not connected',
|
|
2042
|
+
color: connected ? _success : _warning,
|
|
2043
|
+
),
|
|
2044
|
+
const Spacer(),
|
|
2045
|
+
OutlinedButton.icon(
|
|
2046
|
+
onPressed: onDownload,
|
|
2047
|
+
icon: const Icon(Icons.download_outlined, size: 18),
|
|
2048
|
+
label: const Text('Download'),
|
|
2049
|
+
style: OutlinedButton.styleFrom(
|
|
2050
|
+
visualDensity: VisualDensity.compact,
|
|
2051
|
+
padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8),
|
|
2052
|
+
),
|
|
2053
|
+
),
|
|
2054
|
+
const SizedBox(width: 8),
|
|
2055
|
+
OutlinedButton.icon(
|
|
2056
|
+
onPressed: onRefresh,
|
|
2057
|
+
icon: const Icon(Icons.sync, size: 18),
|
|
2058
|
+
label: const Text('Refresh'),
|
|
2059
|
+
style: OutlinedButton.styleFrom(
|
|
2060
|
+
visualDensity: VisualDensity.compact,
|
|
2061
|
+
padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 8),
|
|
2062
|
+
),
|
|
2063
|
+
),
|
|
2064
|
+
],
|
|
2065
|
+
),
|
|
2066
|
+
);
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
|
|
1704
2070
|
class _ResultBlock extends StatelessWidget {
|
|
1705
2071
|
const _ResultBlock({required this.label, required this.value});
|
|
1706
2072
|
|