neoagent 3.1.1-beta.9 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/.env.example +2 -2
  2. package/README.md +5 -1
  3. package/docs/architecture.md +1 -1
  4. package/docs/benchmarking.md +68 -87
  5. package/docs/capabilities.md +1 -1
  6. package/docs/clients-and-devices.md +3 -4
  7. package/docs/configuration.md +1 -1
  8. package/docs/devices.md +1 -2
  9. package/docs/health.md +24 -0
  10. package/docs/index.md +2 -2
  11. package/docs/integrations-architecture.md +6 -0
  12. package/docs/integrations.md +15 -1
  13. package/docs/models.md +1 -1
  14. package/docs/persistence.md +1 -1
  15. package/docs/why-neoagent.md +1 -1
  16. package/flutter_app/README.md +0 -5
  17. package/flutter_app/android/app/src/main/AndroidManifest.xml +0 -14
  18. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +0 -98
  19. package/flutter_app/lib/main.dart +0 -6
  20. package/flutter_app/lib/main_app_shell.dart +0 -2
  21. package/flutter_app/lib/main_controller.dart +2 -867
  22. package/flutter_app/lib/main_devices.dart +0 -73
  23. package/flutter_app/lib/main_integrations.dart +339 -187
  24. package/flutter_app/lib/main_launcher.dart +2 -212
  25. package/flutter_app/lib/main_models.dart +0 -259
  26. package/flutter_app/lib/main_navigation.dart +2 -14
  27. package/flutter_app/lib/main_operations.dart +322 -364
  28. package/flutter_app/lib/main_runtime.dart +9 -208
  29. package/flutter_app/lib/main_settings.dart +17 -199
  30. package/flutter_app/lib/main_shared.dart +20 -591
  31. package/flutter_app/lib/main_timeline.dart +26 -122
  32. package/flutter_app/lib/main_voice_assistant.dart +0 -5
  33. package/flutter_app/lib/src/backend_client.dart +0 -102
  34. package/flutter_app/lib/src/desktop_companion_io.dart +1 -65
  35. package/flutter_app/lib/src/desktop_companion_stub.dart +0 -12
  36. package/flutter_app/linux/flutter/generated_plugin_registrant.cc +0 -4
  37. package/flutter_app/linux/flutter/generated_plugins.cmake +0 -1
  38. package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +0 -2
  39. package/flutter_app/macos/Podfile.lock +56 -5
  40. package/flutter_app/pubspec.lock +0 -7
  41. package/flutter_app/pubspec.yaml +0 -2
  42. package/flutter_app/web/index.html +174 -0
  43. package/flutter_app/windows/flutter/generated_plugin_registrant.cc +0 -3
  44. package/flutter_app/windows/flutter/generated_plugins.cmake +0 -1
  45. package/landing/index.html +0 -2
  46. package/lib/manager.js +1 -1
  47. package/lib/schema_migrations.js +75 -0
  48. package/package.json +1 -4
  49. package/server/db/database.js +1 -133
  50. package/server/http/middleware.js +2 -16
  51. package/server/http/routes.js +0 -2
  52. package/server/public/.last_build_id +1 -1
  53. package/server/public/assets/NOTICES +0 -25
  54. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  55. package/server/public/flutter_bootstrap.js +1 -1
  56. package/server/public/index.html +174 -0
  57. package/server/public/main.dart.js +88081 -90036
  58. package/server/routes/auth.js +11 -2
  59. package/server/routes/settings.js +0 -5
  60. package/server/routes/voice_assistant.js +1 -128
  61. package/server/services/account/erasure.js +0 -3
  62. package/server/services/ai/deliverables/artifact_helpers.js +1 -1
  63. package/server/services/ai/loop/conversation_loop.js +6 -2
  64. package/server/services/ai/loop/tool_dispatch.js +0 -3
  65. package/server/services/ai/messagingFallback.js +1 -1
  66. package/server/services/ai/settings.js +0 -6
  67. package/server/services/ai/toolEvidence.js +1 -4
  68. package/server/services/ai/tools.js +0 -234
  69. package/server/services/desktop/auth.js +0 -3
  70. package/server/services/desktop/registry.js +2 -50
  71. package/server/services/integrations/neoarchive/provider.js +616 -0
  72. package/server/services/integrations/neorecall/client.js +140 -0
  73. package/server/services/integrations/neorecall/constants.js +87 -0
  74. package/server/services/integrations/neorecall/provider.js +163 -0
  75. package/server/services/integrations/registry.js +4 -2
  76. package/server/services/manager.js +0 -22
  77. package/server/services/memory/manager.js +44 -19
  78. package/server/services/memory/retrieval_reasoning.js +36 -14
  79. package/server/services/messaging/whatsapp.js +18 -0
  80. package/server/services/security/tool_categories.js +0 -5
  81. package/server/services/social_video/service.js +2 -20
  82. package/server/services/tasks/adapters/index.js +1 -1
  83. package/server/services/tasks/integration_runtime.js +3 -67
  84. package/server/services/tasks/runtime.js +1 -6
  85. package/server/services/tasks/task_repository.js +6 -13
  86. package/server/services/timeline/service.js +0 -224
  87. package/server/services/voice/message.js +0 -24
  88. package/server/services/voice/providers.js +42 -1
  89. package/server/services/voice/runtimeManager.js +9 -5
  90. package/server/services/voice/turnRunner.js +0 -22
  91. package/server/services/wearable/service.js +0 -5
  92. package/docs/recordings-and-health.md +0 -41
  93. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +0 -21
  94. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +0 -586
  95. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +0 -78
  96. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +0 -104
  97. package/flutter_app/lib/main_recordings.dart +0 -920
  98. package/flutter_app/lib/src/desktop_ocr_bridge.dart +0 -2
  99. package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +0 -125
  100. package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +0 -30
  101. package/flutter_app/lib/src/desktop_passive_history.dart +0 -332
  102. package/flutter_app/lib/src/recording_bridge.dart +0 -232
  103. package/flutter_app/lib/src/recording_bridge_io.dart +0 -1027
  104. package/flutter_app/lib/src/recording_bridge_stub.dart +0 -120
  105. package/flutter_app/lib/src/recording_bridge_web.dart +0 -702
  106. package/flutter_app/lib/src/recording_chunk_queue.dart +0 -149
  107. package/flutter_app/lib/src/recording_chunk_queue_io.dart +0 -182
  108. package/flutter_app/lib/src/recording_payloads.dart +0 -95
  109. package/flutter_app/third_party/desktop_audio_capture/LICENSE +0 -21
  110. package/flutter_app/third_party/desktop_audio_capture/README.md +0 -262
  111. package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +0 -65
  112. package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +0 -153
  113. package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +0 -110
  114. package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +0 -461
  115. package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +0 -91
  116. package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +0 -106
  117. package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +0 -219
  118. package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +0 -336
  119. package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +0 -101
  120. package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +0 -692
  121. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +0 -35
  122. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +0 -36
  123. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +0 -32
  124. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +0 -32
  125. package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +0 -878
  126. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +0 -27
  127. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +0 -1172
  128. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +0 -655
  129. package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +0 -12
  130. package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +0 -30
  131. package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +0 -87
  132. package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +0 -105
  133. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +0 -80
  134. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +0 -31
  135. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +0 -12
  136. package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +0 -23
  137. package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +0 -25
  138. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +0 -1117
  139. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +0 -115
  140. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +0 -777
  141. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +0 -87
  142. package/server/routes/recordings.js +0 -335
  143. package/server/routes/screenHistory.js +0 -190
  144. package/server/services/ai/recordingInsights.js +0 -105
  145. package/server/services/integrations/neomail/provider.js +0 -993
  146. package/server/services/recordings/deepgram.js +0 -58
  147. package/server/services/recordings/manager.js +0 -1273
  148. package/server/services/tasks/adapters/neomail_email_received.js +0 -46
  149. package/server/services/voice/screenshotContext.js +0 -73
  150. package/server/services/voice/turnCoordinator.js +0 -51
@@ -1,1027 +0,0 @@
1
- import 'dart:async';
2
- import 'dart:io';
3
-
4
- import 'package:desktop_audio_capture/audio_capture.dart';
5
- import 'package:flutter/foundation.dart';
6
- import 'package:flutter/services.dart';
7
- import 'package:http/http.dart' as http;
8
- import 'package:path_provider/path_provider.dart';
9
-
10
- import 'diagnostics_logger.dart';
11
- import 'recording_bridge.dart';
12
- import 'recording_chunk_queue.dart';
13
- import 'recording_chunk_queue_io.dart';
14
-
15
- RecordingBridge createPlatformRecordingBridge() {
16
- if (!kIsWeb && defaultTargetPlatform == TargetPlatform.android) {
17
- return AndroidRecordingBridge();
18
- }
19
- if (!kIsWeb &&
20
- (defaultTargetPlatform == TargetPlatform.macOS ||
21
- defaultTargetPlatform == TargetPlatform.windows ||
22
- defaultTargetPlatform == TargetPlatform.linux)) {
23
- return DesktopRecordingBridge();
24
- }
25
- return UnsupportedIoRecordingBridge();
26
- }
27
-
28
- class UnsupportedIoRecordingBridge extends RecordingBridge {
29
- RecordingRuntimeStatus _status = const RecordingRuntimeStatus(
30
- supportsScreenAndMic: false,
31
- supportsBackgroundMic: false,
32
- platformLabel: 'Unsupported',
33
- );
34
-
35
- @override
36
- RecordingRuntimeStatus get status => _status;
37
-
38
- @override
39
- Future<void> refreshStatus() async {}
40
-
41
- @override
42
- Future<void> startWebRecording({
43
- required String baseUrl,
44
- required String sessionId,
45
- }) async {
46
- throw const RecordingBridgeException(
47
- 'Screen and microphone recording is available on web only.',
48
- );
49
- }
50
-
51
- @override
52
- Future<void> startWebMicrophoneRecording({
53
- required String baseUrl,
54
- required String sessionId,
55
- }) async {
56
- throw const RecordingBridgeException(
57
- 'Microphone-only browser recording is available on web only.',
58
- );
59
- }
60
-
61
- @override
62
- Future<void> startBackgroundRecording({
63
- required String baseUrl,
64
- required String sessionCookie,
65
- required String sessionId,
66
- }) async {
67
- throw const RecordingBridgeException(
68
- 'Background microphone recording is not supported on this platform.',
69
- );
70
- }
71
-
72
- @override
73
- Future<void> startDesktopAudioRecording({
74
- required String baseUrl,
75
- required String sessionCookie,
76
- required String sessionId,
77
- bool autoShowToolbar = true,
78
- }) async {
79
- throw const RecordingBridgeException(
80
- 'Desktop recording is not supported on this platform.',
81
- );
82
- }
83
-
84
- @override
85
- Future<void> pauseBackgroundRecording() async {
86
- throw const RecordingBridgeException(
87
- 'Recording is not supported on this platform.',
88
- );
89
- }
90
-
91
- @override
92
- Future<void> resumeBackgroundRecording() async {
93
- throw const RecordingBridgeException(
94
- 'Recording is not supported on this platform.',
95
- );
96
- }
97
-
98
- @override
99
- Future<void> pauseDesktopRecording() async {
100
- throw const RecordingBridgeException(
101
- 'Recording is not supported on this platform.',
102
- );
103
- }
104
-
105
- @override
106
- Future<void> resumeDesktopRecording() async {
107
- throw const RecordingBridgeException(
108
- 'Recording is not supported on this platform.',
109
- );
110
- }
111
-
112
- @override
113
- Future<void> showFloatingToolbar() async {}
114
-
115
- @override
116
- Future<void> hideFloatingToolbar() async {}
117
-
118
- @override
119
- Future<void> openMicrophoneSettings() async {
120
- throw const RecordingBridgeException(
121
- 'Microphone settings are not supported on this platform.',
122
- );
123
- }
124
-
125
- @override
126
- Future<void> openSystemAudioSettings() async {
127
- throw const RecordingBridgeException(
128
- 'System audio settings are not supported on this platform.',
129
- );
130
- }
131
-
132
- @override
133
- Future<void> stopActiveRecording({bool notifyEnded = false}) async {
134
- _status = _status.copyWith(
135
- active: false,
136
- paused: false,
137
- sessionId: null,
138
- errorMessage: null,
139
- floatingToolbarVisible: false,
140
- );
141
- notifyListeners();
142
- }
143
- }
144
-
145
- class AndroidRecordingBridge extends RecordingBridge {
146
- static const MethodChannel _channel = MethodChannel('neoagent/recordings');
147
-
148
- RecordingRuntimeStatus _status = RecordingRuntimeStatus(
149
- supportsScreenAndMic: false,
150
- supportsBackgroundMic: true,
151
- platformLabel: 'Android background recorder',
152
- backgroundRuntimeActive: true,
153
- );
154
-
155
- @override
156
- RecordingRuntimeStatus get status => _status;
157
-
158
- void _log(
159
- String event, {
160
- Map<String, Object?> data = const <String, Object?>{},
161
- Object? error,
162
- StackTrace? stackTrace,
163
- }) {
164
- AppDiagnostics.log(
165
- 'recording.bridge.android',
166
- event,
167
- data: data,
168
- error: error,
169
- stackTrace: stackTrace,
170
- );
171
- }
172
-
173
- @override
174
- Future<void> refreshStatus() async {
175
- _log('refresh_status.request');
176
- final result = await _channel.invokeMapMethod<String, dynamic>('status');
177
- _status = _status.copyWith(
178
- active: result?['active'] == true,
179
- paused: result?['paused'] == true,
180
- sessionId: result?['sessionId']?.toString(),
181
- errorMessage: result?['errorMessage']?.toString(),
182
- startedAt: _parseDate(result?['startedAt']),
183
- activeSources: result?['active'] == true
184
- ? const <String>['microphone']
185
- : const <String>[],
186
- );
187
- notifyListeners();
188
- }
189
-
190
- @override
191
- Future<void> startBackgroundRecording({
192
- required String baseUrl,
193
- required String sessionCookie,
194
- required String sessionId,
195
- }) async {
196
- _log('start_background.request', data: <String, Object?>{
197
- 'sessionId': sessionId,
198
- 'baseUrl': baseUrl,
199
- 'hasSessionCookie': sessionCookie.isNotEmpty,
200
- });
201
- await _channel.invokeMethod('startBackgroundRecording', <String, dynamic>{
202
- 'backendUrl': baseUrl,
203
- 'sessionCookie': sessionCookie,
204
- 'sessionId': sessionId,
205
- });
206
- await refreshStatus();
207
- }
208
-
209
- @override
210
- Future<void> pauseBackgroundRecording() async {
211
- await _channel.invokeMethod('pauseBackgroundRecording');
212
- await refreshStatus();
213
- }
214
-
215
- @override
216
- Future<void> resumeBackgroundRecording() async {
217
- await _channel.invokeMethod('resumeBackgroundRecording');
218
- await refreshStatus();
219
- }
220
-
221
- @override
222
- Future<void> stopActiveRecording({bool notifyEnded = false}) async {
223
- final sessionId = _status.sessionId;
224
- await _channel.invokeMethod('stopBackgroundRecording');
225
- await refreshStatus();
226
- if (notifyEnded && sessionId != null && onRecordingStopped != null) {
227
- await onRecordingStopped!(sessionId);
228
- }
229
- }
230
-
231
- @override
232
- Future<void> startDesktopAudioRecording({
233
- required String baseUrl,
234
- required String sessionCookie,
235
- required String sessionId,
236
- bool autoShowToolbar = true,
237
- }) async {
238
- throw const RecordingBridgeException(
239
- 'Native desktop recording is not available on Android.',
240
- );
241
- }
242
-
243
- @override
244
- Future<void> pauseDesktopRecording() async {
245
- throw const RecordingBridgeException(
246
- 'Desktop recording controls are not available on Android.',
247
- );
248
- }
249
-
250
- @override
251
- Future<void> resumeDesktopRecording() async {
252
- throw const RecordingBridgeException(
253
- 'Desktop recording controls are not available on Android.',
254
- );
255
- }
256
-
257
- @override
258
- Future<void> showFloatingToolbar() async {}
259
-
260
- @override
261
- Future<void> hideFloatingToolbar() async {}
262
-
263
- @override
264
- Future<void> openMicrophoneSettings() async {
265
- throw const RecordingBridgeException(
266
- 'Manage Android microphone permissions from system settings.',
267
- );
268
- }
269
-
270
- @override
271
- Future<void> openSystemAudioSettings() async {
272
- throw const RecordingBridgeException(
273
- 'System audio capture settings are only available on desktop.',
274
- );
275
- }
276
-
277
- @override
278
- Future<void> startWebRecording({
279
- required String baseUrl,
280
- required String sessionId,
281
- }) async {
282
- throw const RecordingBridgeException(
283
- 'Screen and microphone recording is available on web only.',
284
- );
285
- }
286
-
287
- @override
288
- Future<void> startWebMicrophoneRecording({
289
- required String baseUrl,
290
- required String sessionId,
291
- }) async {
292
- throw const RecordingBridgeException(
293
- 'Microphone-only browser recording is available on web only.',
294
- );
295
- }
296
-
297
- DateTime? _parseDate(Object? raw) {
298
- if (raw == null) {
299
- return null;
300
- }
301
- return DateTime.tryParse(raw.toString());
302
- }
303
- }
304
-
305
- class DesktopRecordingBridge extends RecordingBridge {
306
- DesktopRecordingBridge()
307
- : _micCapture = MicAudioCapture(
308
- config: MicAudioConfig(
309
- sampleRate: _sampleRate,
310
- channels: _channels,
311
- bitDepth: 16,
312
- gainBoost: 1.4,
313
- inputVolume: 1.0,
314
- ),
315
- ),
316
- _systemCapture = SystemAudioCapture(
317
- config: SystemAudioConfig(
318
- sampleRate: _sampleRate,
319
- channels: _channels,
320
- ),
321
- );
322
-
323
- static const int _sampleRate = 16000;
324
- static const int _channels = 1;
325
- static const int _bytesPerSample = 2;
326
- static const int _chunkDurationMs = 4000;
327
-
328
- final MicAudioCapture _micCapture;
329
- final SystemAudioCapture _systemCapture;
330
- http.Client _httpClient = http.Client();
331
-
332
- RecordingRuntimeStatus _status = RecordingRuntimeStatus(
333
- supportsScreenAndMic: false,
334
- supportsBackgroundMic: false,
335
- supportsSystemAudio: true,
336
- supportsDesktopBackgroundRuntime: true,
337
- supportsFloatingToolbar: true,
338
- supportsGlobalHotkeys: true,
339
- platformLabel: _desktopPlatformLabel(),
340
- backgroundRuntimeActive: true,
341
- );
342
-
343
- final Map<String, List<int>> _pcmBuffers = <String, List<int>>{
344
- 'microphone': <int>[],
345
- 'system': <int>[],
346
- };
347
- final Map<String, int> _nextSequenceBySource = <String, int>{
348
- 'microphone': 0,
349
- 'system': 0,
350
- };
351
- final Map<String, int> _lastEndMsBySource = <String, int>{
352
- 'microphone': 0,
353
- 'system': 0,
354
- };
355
- final Map<String, int> _bytesPerSecondBySource = <String, int>{
356
- 'microphone': _sampleRate * _channels * _bytesPerSample,
357
- 'system': _sampleRate * _channels * _bytesPerSample,
358
- };
359
-
360
- // Chunks are persisted to disk before upload and retried until confirmed, so a
361
- // transient network failure during a meeting never loses audio.
362
- DiskChunkUploadQueue? _uploadQueue;
363
- String? _systemPermissionErrorMessage;
364
-
365
- StreamSubscription<Uint8List>? _micAudioSub;
366
- StreamSubscription<Uint8List>? _systemAudioSub;
367
- StreamSubscription<MicAudioStatus>? _micStatusSub;
368
- StreamSubscription<SystemAudioStatus>? _systemStatusSub;
369
- StreamSubscription<DecibelData>? _micLevelSub;
370
- StreamSubscription<DecibelData>? _systemLevelSub;
371
-
372
- String? _baseUrl;
373
- String? _sessionCookie;
374
- String? _sessionId;
375
-
376
- @override
377
- RecordingRuntimeStatus get status => _status;
378
-
379
- @override
380
- Future<void> refreshStatus() async {
381
- final availableInputDevices = await _loadInputDevices();
382
- final selectedInput = availableInputDevices.cast<RecordingInputDevice?>()
383
- .firstWhere(
384
- (device) => device?.isDefault == true,
385
- orElse: () => availableInputDevices.isEmpty
386
- ? null
387
- : availableInputDevices.first,
388
- );
389
- _status = _status.copyWith(
390
- availableInputDevices: availableInputDevices,
391
- selectedInputDeviceId: selectedInput?.id,
392
- selectedInputDeviceName: selectedInput?.name,
393
- );
394
- notifyListeners();
395
- }
396
-
397
- @override
398
- Future<void> startDesktopAudioRecording({
399
- required String baseUrl,
400
- required String sessionCookie,
401
- required String sessionId,
402
- bool autoShowToolbar = true,
403
- }) async {
404
- if (_status.active) {
405
- throw const RecordingBridgeException(
406
- 'A desktop recording is already in progress.',
407
- );
408
- }
409
-
410
- _baseUrl = baseUrl;
411
- _sessionCookie = sessionCookie;
412
- _sessionId = sessionId;
413
- _resetBuffers();
414
- await refreshStatus();
415
-
416
- try {
417
- await _ensureDesktopPermissions();
418
- await _initUploadQueue(sessionId);
419
- await _startStreams();
420
- _status = _status.copyWith(
421
- active: true,
422
- paused: false,
423
- sessionId: sessionId,
424
- startedAt: DateTime.now(),
425
- errorMessage: null,
426
- activeSources: const <String>['microphone', 'system'],
427
- floatingToolbarVisible: autoShowToolbar,
428
- );
429
- notifyListeners();
430
- } catch (error, stackTrace) {
431
- _log('start_desktop.failed', error: error, stackTrace: stackTrace);
432
- _status = _status.copyWith(
433
- active: false,
434
- paused: false,
435
- sessionId: null,
436
- startedAt: null,
437
- activeSources: const <String>[],
438
- errorMessage: error.toString(),
439
- );
440
- notifyListeners();
441
- rethrow;
442
- }
443
- }
444
-
445
- @override
446
- Future<void> pauseDesktopRecording() async {
447
- if (!_status.active || _status.paused) {
448
- return;
449
- }
450
- await _stopStreams(flushPending: true);
451
- await _uploadQueue?.flush();
452
- _status = _status.copyWith(
453
- paused: true,
454
- activeSources: const <String>[],
455
- microphoneLevelDb: -120,
456
- systemAudioLevelDb: -120,
457
- );
458
- notifyListeners();
459
- }
460
-
461
- @override
462
- Future<void> resumeDesktopRecording() async {
463
- if (!_status.active || !_status.paused) {
464
- return;
465
- }
466
- await _ensureDesktopPermissions();
467
- await _startStreams();
468
- _status = _status.copyWith(
469
- paused: false,
470
- activeSources: const <String>['microphone', 'system'],
471
- errorMessage: null,
472
- );
473
- notifyListeners();
474
- }
475
-
476
- @override
477
- Future<void> showFloatingToolbar() async {
478
- _status = _status.copyWith(floatingToolbarVisible: true);
479
- notifyListeners();
480
- }
481
-
482
- @override
483
- Future<void> hideFloatingToolbar() async {
484
- _status = _status.copyWith(floatingToolbarVisible: false);
485
- notifyListeners();
486
- }
487
-
488
- @override
489
- Future<void> openMicrophoneSettings() async {
490
- await _openPlatformSettings(
491
- macUrl:
492
- 'x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone',
493
- windowsCommand: const <String>[
494
- 'cmd',
495
- '/c',
496
- 'start',
497
- 'ms-settings:privacy-microphone',
498
- ],
499
- linuxCommands: const <List<String>>[
500
- <String>['xdg-open', 'settings://privacy'],
501
- <String>['gnome-control-center', 'privacy'],
502
- <String>['gnome-control-center', 'sound'],
503
- <String>['pavucontrol'],
504
- ],
505
- failureMessage: 'Could not open microphone settings.',
506
- );
507
- }
508
-
509
- @override
510
- Future<void> openSystemAudioSettings() async {
511
- await _openPlatformSettings(
512
- macUrl:
513
- 'x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture',
514
- windowsCommand: const <String>[
515
- 'cmd',
516
- '/c',
517
- 'start',
518
- 'ms-settings:sound',
519
- ],
520
- linuxCommands: const <List<String>>[
521
- <String>['xdg-open', 'settings://sound'],
522
- <String>['gnome-control-center', 'sound'],
523
- <String>['pavucontrol'],
524
- ],
525
- failureMessage: 'Could not open system audio settings.',
526
- );
527
- }
528
-
529
- @override
530
- Future<void> stopActiveRecording({bool notifyEnded = false}) async {
531
- if (!_status.active) {
532
- return;
533
- }
534
- final sessionId = _sessionId;
535
- await _stopStreams(flushPending: true);
536
- await _uploadQueue?.flush();
537
- _status = _status.copyWith(
538
- active: false,
539
- paused: false,
540
- sessionId: null,
541
- startedAt: null,
542
- activeSources: const <String>[],
543
- microphoneLevelDb: -120,
544
- systemAudioLevelDb: -120,
545
- floatingToolbarVisible: false,
546
- );
547
- notifyListeners();
548
- if (notifyEnded && sessionId != null && onRecordingStopped != null) {
549
- await onRecordingStopped!(sessionId);
550
- }
551
- }
552
-
553
- @override
554
- Future<void> pauseBackgroundRecording() async {
555
- throw const RecordingBridgeException(
556
- 'Android background recording controls are not available on desktop.',
557
- );
558
- }
559
-
560
- @override
561
- Future<void> resumeBackgroundRecording() async {
562
- throw const RecordingBridgeException(
563
- 'Android background recording controls are not available on desktop.',
564
- );
565
- }
566
-
567
- @override
568
- Future<void> startBackgroundRecording({
569
- required String baseUrl,
570
- required String sessionCookie,
571
- required String sessionId,
572
- }) async {
573
- throw const RecordingBridgeException(
574
- 'Android background recording is not available on desktop.',
575
- );
576
- }
577
-
578
- @override
579
- Future<void> startWebRecording({
580
- required String baseUrl,
581
- required String sessionId,
582
- }) async {
583
- throw const RecordingBridgeException(
584
- 'Screen and microphone recording is available on web only.',
585
- );
586
- }
587
-
588
- @override
589
- Future<void> startWebMicrophoneRecording({
590
- required String baseUrl,
591
- required String sessionId,
592
- }) async {
593
- throw const RecordingBridgeException(
594
- 'Microphone-only browser recording is available on web only.',
595
- );
596
- }
597
-
598
- @override
599
- void dispose() {
600
- unawaited(_disposeDesktopBridge());
601
- super.dispose();
602
- }
603
-
604
- Future<void> _disposeDesktopBridge() async {
605
- try {
606
- await _stopStreams(
607
- flushPending: false,
608
- ).timeout(const Duration(seconds: 2));
609
- } on TimeoutException {
610
- _log('dispose.stop_streams.timeout');
611
- } catch (error, stackTrace) {
612
- _log('dispose.stop_streams.failed', error: error, stackTrace: stackTrace);
613
- }
614
-
615
- try {
616
- await _micCapture.stopCapture();
617
- } catch (_) {}
618
- try {
619
- await _systemCapture.stopCapture();
620
- } catch (_) {}
621
-
622
- await _uploadQueue?.dispose();
623
- _httpClient.close();
624
- }
625
-
626
- Future<void> _initUploadQueue(String sessionId) async {
627
- await _uploadQueue?.dispose();
628
- final support = await getApplicationSupportDirectory();
629
- final dir = Directory('${support.path}/recording-pending/$sessionId');
630
- _uploadQueue = DiskChunkUploadQueue(
631
- directory: dir,
632
- uploader: _performChunkUpload,
633
- logger: (event, {Map<String, Object?> data = const <String, Object?>{}, Object? error}) =>
634
- _log(event, data: data, error: error),
635
- );
636
- }
637
-
638
- Future<void> _ensureDesktopPermissions() async {
639
- final micGranted = await _requestMicPermission();
640
- final systemGranted = await _requestSystemPermission();
641
- if (!micGranted || !systemGranted) {
642
- throw RecordingBridgeException(
643
- _permissionFailureMessage(
644
- micGranted: micGranted,
645
- systemGranted: systemGranted,
646
- ),
647
- );
648
- }
649
- }
650
-
651
- Future<bool> _requestMicPermission() async {
652
- try {
653
- final granted = await _micCapture.requestPermissions();
654
- _status = _status.copyWith(
655
- microphonePermission: granted
656
- ? RecordingPermissionState.granted
657
- : RecordingPermissionState.denied,
658
- );
659
- return granted;
660
- } catch (error) {
661
- _status = _status.copyWith(
662
- microphonePermission: _permissionStateFromError(error),
663
- );
664
- return false;
665
- }
666
- }
667
-
668
- Future<bool> _requestSystemPermission() async {
669
- try {
670
- final granted = await _systemCapture.requestPermissions();
671
- _systemPermissionErrorMessage = null;
672
- _status = _status.copyWith(
673
- systemAudioPermission: granted
674
- ? RecordingPermissionState.granted
675
- : RecordingPermissionState.denied,
676
- );
677
- return granted;
678
- } on MissingPluginException catch (error, stackTrace) {
679
- // The system-audio plugin is only registered where capture is supported
680
- // (macOS 13+ via ScreenCaptureKit). Surface that instead of a misleading
681
- // "grant permission" prompt the user can never satisfy.
682
- _systemPermissionErrorMessage =
683
- 'System audio capture is unavailable on this OS version. On macOS it '
684
- 'requires macOS 13 (Ventura) or later.';
685
- _log('system_permission.unavailable', error: error, stackTrace: stackTrace);
686
- _status = _status.copyWith(
687
- systemAudioPermission: RecordingPermissionState.unknown,
688
- );
689
- return false;
690
- } catch (error) {
691
- _status = _status.copyWith(
692
- systemAudioPermission: _permissionStateFromError(error),
693
- );
694
- return false;
695
- }
696
- }
697
-
698
- Future<List<RecordingInputDevice>> _loadInputDevices() async {
699
- try {
700
- final devices = await _micCapture.getAvailableInputDevices();
701
- return devices
702
- .map(
703
- (device) => RecordingInputDevice(
704
- id: device.id,
705
- name: device.name,
706
- kind: device.type.toString(),
707
- channelCount: device.channelCount,
708
- isDefault: device.isDefault,
709
- ),
710
- )
711
- .toList();
712
- } catch (error, stackTrace) {
713
- _log('input_devices.failed', error: error, stackTrace: stackTrace);
714
- return const <RecordingInputDevice>[];
715
- }
716
- }
717
-
718
- Future<void> _startStreams() async {
719
- await _micCapture.startCapture();
720
- await _systemCapture.startCapture();
721
-
722
- _micAudioSub = _micCapture.audioStream?.listen(
723
- (bytes) => _handleAudioChunk('microphone', bytes),
724
- onError: (Object error, StackTrace stackTrace) {
725
- _handleRuntimeError('microphone.stream', error, stackTrace);
726
- },
727
- );
728
- _systemAudioSub = _systemCapture.audioStream?.listen(
729
- (bytes) => _handleAudioChunk('system', bytes),
730
- onError: (Object error, StackTrace stackTrace) {
731
- _handleRuntimeError('system.stream', error, stackTrace);
732
- },
733
- );
734
- _micStatusSub = _micCapture.statusStream?.listen((status) {
735
- _status = _status.copyWith(
736
- selectedInputDeviceName: status.deviceName,
737
- );
738
- notifyListeners();
739
- });
740
- _systemStatusSub = _systemCapture.statusStream?.listen((_) {});
741
- _micLevelSub = _micCapture.decibelStream?.listen((level) {
742
- _status = _status.copyWith(microphoneLevelDb: level.decibel);
743
- notifyListeners();
744
- });
745
- _systemLevelSub = _systemCapture.decibelStream?.listen((level) {
746
- _status = _status.copyWith(systemAudioLevelDb: level.decibel);
747
- notifyListeners();
748
- });
749
- }
750
-
751
- void _handleAudioChunk(String sourceKey, Uint8List bytes) {
752
- final buffer = _pcmBuffers[sourceKey]!;
753
- buffer.addAll(bytes);
754
- final chunkByteTarget =
755
- (_bytesPerSecondBySource[sourceKey]! * _chunkDurationMs) ~/ 1000;
756
- while (buffer.length >= chunkByteTarget) {
757
- final pcmBytes = Uint8List.fromList(buffer.sublist(0, chunkByteTarget));
758
- buffer.removeRange(0, chunkByteTarget);
759
- _queueUpload(
760
- sourceKey: sourceKey,
761
- pcmBytes: pcmBytes,
762
- byteLength: chunkByteTarget,
763
- );
764
- }
765
- }
766
-
767
- void _queueUpload({
768
- required String sourceKey,
769
- required Uint8List pcmBytes,
770
- required int byteLength,
771
- }) {
772
- final startMs = _lastEndMsBySource[sourceKey] ?? 0;
773
- final durationMs =
774
- (byteLength * 1000) ~/ _bytesPerSecondBySource[sourceKey]!;
775
- final endMs = startMs + durationMs;
776
- final sequence = _nextSequenceBySource[sourceKey] ?? 0;
777
- _nextSequenceBySource[sourceKey] = sequence + 1;
778
- _lastEndMsBySource[sourceKey] = endMs;
779
-
780
- final queue = _uploadQueue;
781
- if (queue == null) {
782
- return;
783
- }
784
- // Persist + enqueue. The queue uploads in order and retries until the chunk
785
- // is confirmed stored, so a dropped upload no longer loses that audio.
786
- unawaited(
787
- queue.enqueue(
788
- PendingChunk(
789
- sourceKey: sourceKey,
790
- sequence: sequence,
791
- startMs: startMs,
792
- endMs: endMs,
793
- mimeType: 'audio/wav',
794
- bytes: _wrapPcmAsWav(pcmBytes),
795
- ),
796
- ),
797
- );
798
- }
799
-
800
- Future<void> _performChunkUpload(PendingChunk chunk) async {
801
- final sessionId = _sessionId;
802
- final baseUrl = _baseUrl;
803
- if (sessionId == null || baseUrl == null || baseUrl.trim().isEmpty) {
804
- throw const RecordingBridgeException(
805
- 'Desktop recording session is not initialized.',
806
- );
807
- }
808
-
809
- final uri = Uri.parse(
810
- '${baseUrl.replaceFirst(RegExp(r'/$'), '')}/api/recordings/$sessionId/chunks',
811
- );
812
- final headers = <String, String>{
813
- 'Content-Type': chunk.mimeType,
814
- 'X-Recording-Source-Key': chunk.sourceKey,
815
- 'X-Recording-Sequence': '${chunk.sequence}',
816
- 'X-Recording-Start-Ms': '${chunk.startMs}',
817
- 'X-Recording-End-Ms': '${chunk.endMs}',
818
- if ((_sessionCookie ?? '').trim().isNotEmpty)
819
- 'Cookie': _sessionCookie!.trim(),
820
- };
821
-
822
- try {
823
- final response = await _httpClient
824
- .post(uri, headers: headers, body: chunk.bytes)
825
- .timeout(const Duration(seconds: 20));
826
- if (response.statusCode < 200 || response.statusCode >= 300) {
827
- throw RecordingBridgeException(
828
- 'Chunk upload failed with status ${response.statusCode}.',
829
- );
830
- }
831
- } on TimeoutException {
832
- // A stuck socket can wedge subsequent requests; rebuild the client so the
833
- // queue's retry gets a fresh connection.
834
- _httpClient.close();
835
- _httpClient = http.Client();
836
- rethrow;
837
- }
838
- }
839
-
840
- Future<void> _stopStreams({required bool flushPending}) async {
841
- await _micAudioSub?.cancel();
842
- await _systemAudioSub?.cancel();
843
- await _micStatusSub?.cancel();
844
- await _systemStatusSub?.cancel();
845
- await _micLevelSub?.cancel();
846
- await _systemLevelSub?.cancel();
847
- _micAudioSub = null;
848
- _systemAudioSub = null;
849
- _micStatusSub = null;
850
- _systemStatusSub = null;
851
- _micLevelSub = null;
852
- _systemLevelSub = null;
853
-
854
- if (_micCapture.isRecording) {
855
- await _micCapture.stopCapture();
856
- }
857
- if (_systemCapture.isRecording) {
858
- await _systemCapture.stopCapture();
859
- }
860
-
861
- if (flushPending) {
862
- for (final entry in _pcmBuffers.entries) {
863
- if (entry.value.isEmpty) {
864
- continue;
865
- }
866
- final remaining = Uint8List.fromList(entry.value);
867
- entry.value.clear();
868
- _queueUpload(
869
- sourceKey: entry.key,
870
- pcmBytes: remaining,
871
- byteLength: remaining.length,
872
- );
873
- }
874
- } else {
875
- _resetBuffers();
876
- }
877
- }
878
-
879
- void _resetBuffers() {
880
- for (final sourceKey in _pcmBuffers.keys) {
881
- _pcmBuffers[sourceKey] = <int>[];
882
- _nextSequenceBySource[sourceKey] = 0;
883
- _lastEndMsBySource[sourceKey] = 0;
884
- }
885
- }
886
-
887
- Uint8List _wrapPcmAsWav(Uint8List pcmBytes) {
888
- final totalLength = pcmBytes.length + 44;
889
- final data = ByteData(totalLength);
890
- void writeAscii(int offset, String value) {
891
- for (var index = 0; index < value.length; index += 1) {
892
- data.setUint8(offset + index, value.codeUnitAt(index));
893
- }
894
- }
895
-
896
- writeAscii(0, 'RIFF');
897
- data.setUint32(4, totalLength - 8, Endian.little);
898
- writeAscii(8, 'WAVE');
899
- writeAscii(12, 'fmt ');
900
- data.setUint32(16, 16, Endian.little);
901
- data.setUint16(20, 1, Endian.little);
902
- data.setUint16(22, _channels, Endian.little);
903
- data.setUint32(24, _sampleRate, Endian.little);
904
- data.setUint32(
905
- 28,
906
- _sampleRate * _channels * _bytesPerSample,
907
- Endian.little,
908
- );
909
- data.setUint16(32, _channels * _bytesPerSample, Endian.little);
910
- data.setUint16(34, _bytesPerSample * 8, Endian.little);
911
- writeAscii(36, 'data');
912
- data.setUint32(40, pcmBytes.length, Endian.little);
913
- for (var index = 0; index < pcmBytes.length; index += 1) {
914
- data.setUint8(44 + index, pcmBytes[index]);
915
- }
916
- return data.buffer.asUint8List();
917
- }
918
-
919
- RecordingPermissionState _permissionStateFromError(Object error) {
920
- final message = error.toString().toLowerCase();
921
- if (message.contains('restart')) {
922
- return RecordingPermissionState.needsRestart;
923
- }
924
- if (message.contains('permission')) {
925
- return RecordingPermissionState.denied;
926
- }
927
- return RecordingPermissionState.unknown;
928
- }
929
-
930
- String _permissionFailureMessage({
931
- required bool micGranted,
932
- required bool systemGranted,
933
- }) {
934
- if (!micGranted && !systemGranted) {
935
- return 'Grant microphone and system audio permissions before starting desktop recording.';
936
- }
937
- if (!micGranted) {
938
- return 'Grant microphone permission before starting desktop recording.';
939
- }
940
- return _systemPermissionErrorMessage ??
941
- 'Grant system audio permission before starting desktop recording.';
942
- }
943
-
944
- Future<void> _openPlatformSettings({
945
- required String macUrl,
946
- required List<String> windowsCommand,
947
- required List<List<String>> linuxCommands,
948
- required String failureMessage,
949
- }) async {
950
- try {
951
- ProcessResult result;
952
- if (Platform.isMacOS) {
953
- result = await Process.run('open', <String>[macUrl]);
954
- } else if (Platform.isWindows) {
955
- result = await Process.run(
956
- windowsCommand.first,
957
- windowsCommand.sublist(1),
958
- runInShell: true,
959
- );
960
- } else if (Platform.isLinux) {
961
- var success = false;
962
- for (final command in linuxCommands) {
963
- if (command.isEmpty) {
964
- continue;
965
- }
966
- result = await Process.run(command.first, command.sublist(1));
967
- if (result.exitCode == 0) {
968
- success = true;
969
- break;
970
- }
971
- }
972
- if (!success) {
973
- throw RecordingBridgeException(failureMessage);
974
- }
975
- return;
976
- } else {
977
- throw RecordingBridgeException(failureMessage);
978
- }
979
- if (result.exitCode != 0) {
980
- throw RecordingBridgeException(failureMessage);
981
- }
982
- } catch (error) {
983
- if (error is RecordingBridgeException) {
984
- rethrow;
985
- }
986
- throw RecordingBridgeException(failureMessage);
987
- }
988
- }
989
-
990
- void _handleRuntimeError(
991
- String event,
992
- Object error,
993
- StackTrace stackTrace,
994
- ) {
995
- _log(event, error: error, stackTrace: stackTrace);
996
- _status = _status.copyWith(errorMessage: error.toString());
997
- notifyListeners();
998
- }
999
-
1000
- void _log(
1001
- String event, {
1002
- Map<String, Object?> data = const <String, Object?>{},
1003
- Object? error,
1004
- StackTrace? stackTrace,
1005
- }) {
1006
- AppDiagnostics.log(
1007
- 'recording.bridge.desktop',
1008
- event,
1009
- data: data,
1010
- error: error,
1011
- stackTrace: stackTrace,
1012
- );
1013
- }
1014
-
1015
- static String _desktopPlatformLabel() {
1016
- switch (defaultTargetPlatform) {
1017
- case TargetPlatform.macOS:
1018
- return 'Desktop dual-source recorder (macOS)';
1019
- case TargetPlatform.windows:
1020
- return 'Desktop dual-source recorder (Windows)';
1021
- case TargetPlatform.linux:
1022
- return 'Desktop dual-source recorder (Linux)';
1023
- default:
1024
- return 'Desktop dual-source recorder';
1025
- }
1026
- }
1027
- }