neoagent 3.0.1-beta.8 → 3.1.0

package/flutter_app/lib/src/recording_bridge_io.dart

@@ -5,9 +5,12 @@ import 'package:desktop_audio_capture/audio_capture.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/services.dart';
 import 'package:http/http.dart' as http;
+import 'package:path_provider/path_provider.dart';
 
 import 'diagnostics_logger.dart';
 import 'recording_bridge.dart';
+import 'recording_chunk_queue.dart';
+import 'recording_chunk_queue_io.dart';
 
 RecordingBridge createPlatformRecordingBridge() {
   if (!kIsWeb && defaultTargetPlatform == TargetPlatform.android) {
@@ -349,15 +352,16 @@ class DesktopRecordingBridge extends RecordingBridge {
     'microphone': 0,
     'system': 0,
   };
-  final Map<String, Future<void>> _uploadQueueBySource = <String, Future<void>>{
-    'microphone': Future<void>.value(),
-    'system': Future<void>.value(),
-  };
   final Map<String, int> _bytesPerSecondBySource = <String, int>{
     'microphone': _sampleRate * _channels * _bytesPerSample,
     'system': _sampleRate * _channels * _bytesPerSample,
   };
 
+  // Chunks are persisted to disk before upload and retried until confirmed, so a
+  // transient network failure during a meeting never loses audio.
+  DiskChunkUploadQueue? _uploadQueue;
+  String? _systemPermissionErrorMessage;
+
   StreamSubscription<Uint8List>? _micAudioSub;
   StreamSubscription<Uint8List>? _systemAudioSub;
   StreamSubscription<MicAudioStatus>? _micStatusSub;
@@ -411,6 +415,7 @@ class DesktopRecordingBridge extends RecordingBridge {
 
     try {
       await _ensureDesktopPermissions();
+      await _initUploadQueue(sessionId);
       await _startStreams();
       _status = _status.copyWith(
         active: true,
@@ -443,7 +448,7 @@ class DesktopRecordingBridge extends RecordingBridge {
       return;
     }
     await _stopStreams(flushPending: true);
-    await Future.wait(_uploadQueueBySource.values);
+    await _uploadQueue?.flush();
     _status = _status.copyWith(
       paused: true,
       activeSources: const <String>[],
@@ -528,7 +533,7 @@ class DesktopRecordingBridge extends RecordingBridge {
     }
     final sessionId = _sessionId;
     await _stopStreams(flushPending: true);
-    await Future.wait(_uploadQueueBySource.values);
+    await _uploadQueue?.flush();
     _status = _status.copyWith(
       active: false,
       paused: false,
@@ -614,9 +619,22 @@ class DesktopRecordingBridge extends RecordingBridge {
       await _systemCapture.stopCapture();
     } catch (_) {}
 
+    await _uploadQueue?.dispose();
     _httpClient.close();
   }
 
+  Future<void> _initUploadQueue(String sessionId) async {
+    await _uploadQueue?.dispose();
+    final support = await getApplicationSupportDirectory();
+    final dir = Directory('${support.path}/recording-pending/$sessionId');
+    _uploadQueue = DiskChunkUploadQueue(
+      directory: dir,
+      uploader: _performChunkUpload,
+      logger: (event, {Map<String, Object?> data = const <String, Object?>{}, Object? error}) =>
+          _log(event, data: data, error: error),
+    );
+  }
+
   Future<void> _ensureDesktopPermissions() async {
     final micGranted = await _requestMicPermission();
     final systemGranted = await _requestSystemPermission();
@@ -650,12 +668,25 @@ class DesktopRecordingBridge extends RecordingBridge {
   Future<bool> _requestSystemPermission() async {
     try {
       final granted = await _systemCapture.requestPermissions();
+      _systemPermissionErrorMessage = null;
       _status = _status.copyWith(
         systemAudioPermission: granted
             ? RecordingPermissionState.granted
             : RecordingPermissionState.denied,
       );
       return granted;
+    } on MissingPluginException catch (error, stackTrace) {
+      // The system-audio plugin is only registered where capture is supported
+      // (macOS 13+ via ScreenCaptureKit). Surface that instead of a misleading
+      // "grant permission" prompt the user can never satisfy.
+      _systemPermissionErrorMessage =
+          'System audio capture is unavailable on this OS version. On macOS it '
+          'requires macOS 13 (Ventura) or later.';
+      _log('system_permission.unavailable', error: error, stackTrace: stackTrace);
+      _status = _status.copyWith(
+        systemAudioPermission: RecordingPermissionState.unknown,
+      );
+      return false;
     } catch (error) {
       _status = _status.copyWith(
         systemAudioPermission: _permissionStateFromError(error),
@@ -746,32 +777,27 @@ class DesktopRecordingBridge extends RecordingBridge {
     _nextSequenceBySource[sourceKey] = sequence + 1;
     _lastEndMsBySource[sourceKey] = endMs;
 
-    final previous = _uploadQueueBySource[sourceKey] ?? Future<void>.value();
-    _uploadQueueBySource[sourceKey] = previous
-        // Keep the queue moving even if a previous upload failed.
-        .catchError((Object _, StackTrace __) {})
-        .then((_) async {
-          try {
-            await _uploadChunk(
-              sourceKey: sourceKey,
-              sequence: sequence,
-              startMs: startMs,
-              endMs: endMs,
-              bytes: _wrapPcmAsWav(pcmBytes),
-            );
-          } catch (error, stackTrace) {
-            _handleRuntimeError('$sourceKey.upload', error, stackTrace);
-          }
-        });
+    final queue = _uploadQueue;
+    if (queue == null) {
+      return;
+    }
+    // Persist + enqueue. The queue uploads in order and retries until the chunk
+    // is confirmed stored, so a dropped upload no longer loses that audio.
+    unawaited(
+      queue.enqueue(
+        PendingChunk(
+          sourceKey: sourceKey,
+          sequence: sequence,
+          startMs: startMs,
+          endMs: endMs,
+          mimeType: 'audio/wav',
+          bytes: _wrapPcmAsWav(pcmBytes),
+        ),
+      ),
+    );
   }
 
-  Future<void> _uploadChunk({
-    required String sourceKey,
-    required int sequence,
-    required int startMs,
-    required int endMs,
-    required Uint8List bytes,
-  }) async {
+  Future<void> _performChunkUpload(PendingChunk chunk) async {
     final sessionId = _sessionId;
     final baseUrl = _baseUrl;
     if (sessionId == null || baseUrl == null || baseUrl.trim().isEmpty) {
@@ -783,49 +809,31 @@ class DesktopRecordingBridge extends RecordingBridge {
     final uri = Uri.parse(
       '${baseUrl.replaceFirst(RegExp(r'/$'), '')}/api/recordings/$sessionId/chunks',
     );
+    final headers = <String, String>{
+      'Content-Type': chunk.mimeType,
+      'X-Recording-Source-Key': chunk.sourceKey,
+      'X-Recording-Sequence': '${chunk.sequence}',
+      'X-Recording-Start-Ms': '${chunk.startMs}',
+      'X-Recording-End-Ms': '${chunk.endMs}',
+      if ((_sessionCookie ?? '').trim().isNotEmpty)
+        'Cookie': _sessionCookie!.trim(),
+    };
 
-    for (var attempt = 0; attempt < 3; attempt += 1) {
-      try {
-        final headers = <String, String>{
-          'Content-Type': 'audio/wav',
-          'X-Recording-Source-Key': sourceKey,
-          'X-Recording-Sequence': '$sequence',
-          'X-Recording-Start-Ms': '$startMs',
-          'X-Recording-End-Ms': '$endMs',
-          if ((_sessionCookie ?? '').trim().isNotEmpty)
-            'Cookie': _sessionCookie!.trim(),
-        };
-        final response = await _httpClient
-            .post(uri, headers: headers, body: bytes)
-            .timeout(const Duration(seconds: 20));
-        if (response.statusCode < 200 || response.statusCode >= 300) {
-          throw RecordingBridgeException(
-            'Chunk upload failed with status ${response.statusCode}.',
-          );
-        }
-        return;
-      } on TimeoutException catch (error, stackTrace) {
-        _log(
-          'upload.timeout',
-          data: <String, Object?>{
-            'sourceKey': sourceKey,
-            'sequence': sequence,
-            'attempt': attempt + 1,
-          },
-          error: error,
-          stackTrace: stackTrace,
+    try {
+      final response = await _httpClient
+          .post(uri, headers: headers, body: chunk.bytes)
+          .timeout(const Duration(seconds: 20));
+      if (response.statusCode < 200 || response.statusCode >= 300) {
+        throw RecordingBridgeException(
+          'Chunk upload failed with status ${response.statusCode}.',
         );
-        _httpClient.close();
-        _httpClient = http.Client();
-        if (attempt == 2) {
-          throw const RecordingBridgeException('Chunk upload timed out.');
-        }
-      } catch (error) {
-        if (attempt == 2) {
-          rethrow;
-        }
-        await Future<void>.delayed(Duration(milliseconds: 400 * (attempt + 1)));
       }
+    } on TimeoutException {
+      // A stuck socket can wedge subsequent requests; rebuild the client so the
+      // queue's retry gets a fresh connection.
+      _httpClient.close();
+      _httpClient = http.Client();
+      rethrow;
     }
   }
 
@@ -873,7 +881,6 @@ class DesktopRecordingBridge extends RecordingBridge {
       _pcmBuffers[sourceKey] = <int>[];
       _nextSequenceBySource[sourceKey] = 0;
       _lastEndMsBySource[sourceKey] = 0;
-      _uploadQueueBySource[sourceKey] = Future<void>.value();
     }
   }
 
@@ -930,7 +937,8 @@ class DesktopRecordingBridge extends RecordingBridge {
     if (!micGranted) {
       return 'Grant microphone permission before starting desktop recording.';
     }
-    return 'Grant system audio permission before starting desktop recording.';
+    return _systemPermissionErrorMessage ??
+        'Grant system audio permission before starting desktop recording.';
   }
 
   Future<void> _openPlatformSettings({

package/flutter_app/lib/src/recording_bridge_web.dart

@@ -7,6 +7,7 @@ import 'dart:typed_data';
 
 import 'diagnostics_logger.dart';
 import 'recording_bridge.dart';
+import 'recording_chunk_queue.dart';
 
 RecordingBridge createPlatformRecordingBridge() => WebRecordingBridge();
 
@@ -18,13 +19,18 @@ class WebRecordingBridge extends RecordingBridge {
   );
 
   html.MediaStream? _displayStream;
+  html.MediaStream? _systemAudioStream;
   html.MediaStream? _microphoneStream;
   html.MediaRecorder? _screenRecorder;
+  html.MediaRecorder? _systemAudioRecorder;
   html.MediaRecorder? _microphoneRecorder;
   final Map<String, int> _nextSequenceBySource = <String, int>{};
   final Map<String, int> _lastEndMsBySource = <String, int>{};
-  final Map<String, Future<void>> _uploadQueueBySource =
-      <String, Future<void>>{};
+  // One durable, retry-until-success queue per source so a failed upload is
+  // re-sent instead of silently dropped, and a stall in one source does not
+  // block the others.
+  final Map<String, MemoryChunkUploadQueue> _uploadQueues =
+      <String, MemoryChunkUploadQueue>{};
   Stopwatch? _stopwatch;
   String? _baseUrl;
   String? _sessionId;
@@ -119,22 +125,31 @@ class WebRecordingBridge extends RecordingBridge {
         );
       }
 
+      // Split the display capture: video-only feeds the screen-analysis source,
+      // while the shared system/tab audio gets its own transcribable source so
+      // remote participants are captured in the transcript (the mic alone does
+      // not reliably pick up other speakers, e.g. when wearing headphones).
+      final screenVideoStream = html.MediaStream();
+      for (final track in displayStream.getVideoTracks()) {
+        screenVideoStream.addTrack(track);
+      }
+      final systemAudioStream = html.MediaStream();
+      for (final track in displayStream.getAudioTracks()) {
+        systemAudioStream.addTrack(track);
+      }
+
       _baseUrl = baseUrl;
       _sessionId = sessionId;
       _displayStream = displayStream;
+      _systemAudioStream = systemAudioStream;
       _microphoneStream = microphoneStream;
       _nextSequenceBySource
         ..clear()
-        ..addAll(<String, int>{'screen': 0, 'microphone': 0});
+        ..addAll(<String, int>{'screen': 0, 'system': 0, 'microphone': 0});
       _lastEndMsBySource
         ..clear()
-        ..addAll(<String, int>{'screen': 0, 'microphone': 0});
-      _uploadQueueBySource
-        ..clear()
-        ..addAll(<String, Future<void>>{
-          'screen': Future<void>.value(),
-          'microphone': Future<void>.value(),
-        });
+        ..addAll(<String, int>{'screen': 0, 'system': 0, 'microphone': 0});
+      _initUploadQueues(<String>['screen', 'system', 'microphone']);
       _stopwatch = Stopwatch()..start();
 
       final screenMimeType = _pickMimeType(<String>[
@@ -142,13 +157,13 @@ class WebRecordingBridge extends RecordingBridge {
         'video/webm;codecs=vp8,opus',
         'video/webm',
       ]);
-      final micMimeType = _pickMimeType(<String>[
+      final audioMimeType = _pickMimeType(<String>[
         'audio/webm;codecs=opus',
         'audio/webm',
       ]);
 
       _screenRecorder = html.MediaRecorder(
-        displayStream,
+        screenVideoStream,
         screenMimeType == null
             ? null
             : <String, String>{'mimeType': screenMimeType},
@@ -165,17 +180,27 @@ class WebRecordingBridge extends RecordingBridge {
         sourceKey: 'screen',
         mimeType: screenMimeType ?? 'video/webm',
       );
+      _systemAudioRecorder = html.MediaRecorder(
+        systemAudioStream,
+        audioMimeType == null ? null : <String, String>{'mimeType': audioMimeType},
+      );
+      _bindRecorder(
+        recorder: _systemAudioRecorder!,
+        sourceKey: 'system',
+        mimeType: audioMimeType ?? 'audio/webm',
+      );
       _microphoneRecorder = html.MediaRecorder(
         microphoneStream,
-        micMimeType == null ? null : <String, String>{'mimeType': micMimeType},
+        audioMimeType == null ? null : <String, String>{'mimeType': audioMimeType},
       );
       _bindRecorder(
         recorder: _microphoneRecorder!,
         sourceKey: 'microphone',
-        mimeType: micMimeType ?? 'audio/webm',
+        mimeType: audioMimeType ?? 'audio/webm',
       );
 
       _screenRecorder!.start(4000);
+      _systemAudioRecorder!.start(4000);
       _microphoneRecorder!.start(4000);
       _status = _status.copyWith(
         active: true,
@@ -189,7 +214,8 @@ class WebRecordingBridge extends RecordingBridge {
         data: <String, Object?>{
           'sessionId': sessionId,
           'screenMimeType': screenMimeType ?? 'video/webm',
-          'micMimeType': micMimeType ?? 'audio/webm',
+          'systemMimeType': audioMimeType ?? 'audio/webm',
+          'micMimeType': audioMimeType ?? 'audio/webm',
         },
       );
       notifyListeners();
@@ -261,11 +287,7 @@ class WebRecordingBridge extends RecordingBridge {
       _lastEndMsBySource
         ..clear()
         ..addAll(<String, int>{'microphone': 0});
-      _uploadQueueBySource
-        ..clear()
-        ..addAll(<String, Future<void>>{
-          'microphone': Future<void>.value(),
-        });
+      _initUploadQueues(<String>['microphone']);
       _stopwatch = Stopwatch()..start();
 
       final micMimeType = _pickMimeType(<String>[
@@ -404,7 +426,9 @@ class WebRecordingBridge extends RecordingBridge {
     );
     try {
       await _stopRecorders();
-      await Future.wait(_uploadQueueBySource.values);
+      await Future.wait(
+        _uploadQueues.values.map((queue) => queue.flush()),
+      );
       await _disposeStreams();
       _status = _status.copyWith(
         active: false,
@@ -432,6 +456,30 @@ class WebRecordingBridge extends RecordingBridge {
     }
   }
 
+  void _initUploadQueues(List<String> sourceKeys) {
+    for (final queue in _uploadQueues.values) {
+      unawaited(queue.dispose());
+    }
+    _uploadQueues
+      ..clear()
+      ..addEntries(
+        sourceKeys.map(
+          (sourceKey) => MapEntry<String, MemoryChunkUploadQueue>(
+            sourceKey,
+            MemoryChunkUploadQueue(
+              uploader: _performChunkUpload,
+              logger:
+                  (
+                    event, {
+                    Map<String, Object?> data = const <String, Object?>{},
+                    Object? error,
+                  }) => _log(event, data: data, error: error),
+            ),
+          ),
+        ),
+      );
+  }
+
   void _bindRecorder({
     required html.MediaRecorder recorder,
     required String sourceKey,
@@ -447,49 +495,42 @@ class WebRecordingBridge extends RecordingBridge {
       _lastEndMsBySource[sourceKey] = endMs;
       final sequence = _nextSequenceBySource[sourceKey] ?? 0;
       _nextSequenceBySource[sourceKey] = sequence + 1;
-      final upload = (_uploadQueueBySource[sourceKey] ?? Future<void>.value())
-          .then(
-            (_) => _uploadChunk(
+      final queue = _uploadQueues[sourceKey];
+      if (queue == null) {
+        return;
+      }
+      // Read the blob bytes, then hand the chunk to the durable queue, which
+      // uploads and retries until the server confirms it (so a transient
+      // failure no longer drops the chunk or stalls the rest of the source).
+      unawaited(() async {
+        try {
+          final bytes = await _blobToBytes(blob);
+          queue.enqueue(
+            PendingChunk(
               sourceKey: sourceKey,
               sequence: sequence,
               startMs: startMs,
               endMs: endMs,
-              blob: blob,
               mimeType: mimeType,
+              bytes: bytes,
             ),
           );
-      _uploadQueueBySource[sourceKey] = upload.catchError((
-        Object error,
-        StackTrace stackTrace,
-      ) {
-        final stackText = stackTrace.toString();
-        _status = _status.copyWith(
-          errorMessage: stackText.isNotEmpty ? stackText : error.toString(),
-        );
-        _log(
-          'chunk.upload_queue.error',
-          data: <String, Object?>{
-            'sourceKey': sourceKey,
-            'sequence': sequence,
-            'stackTrace': stackText,
-          },
-          error: error,
-          stackTrace: stackTrace,
-        );
-        notifyListeners();
-        Error.throwWithStackTrace(error, stackTrace);
-      });
+        } catch (error, stackTrace) {
+          _log(
+            'chunk.read.failed',
+            data: <String, Object?>{
+              'sourceKey': sourceKey,
+              'sequence': sequence,
+            },
+            error: error,
+            stackTrace: stackTrace,
+          );
+        }
+      }());
     });
   }
 
-  Future<void> _uploadChunk({
-    required String sourceKey,
-    required int sequence,
-    required int startMs,
-    required int endMs,
-    required html.Blob blob,
-    required String mimeType,
-  }) async {
+  Future<void> _performChunkUpload(PendingChunk chunk) async {
     final sessionId = _sessionId;
     final baseUrl = _baseUrl;
     if (sessionId == null || baseUrl == null) {
@@ -497,38 +538,38 @@ class WebRecordingBridge extends RecordingBridge {
         'Recording session is not initialized.',
       );
     }
-    final bytes = await _blobToBytes(blob);
     final uri = _resolveUri(baseUrl, '/api/recordings/$sessionId/chunks');
+    final headers = <String, String>{
+      'Content-Type': chunk.mimeType,
+      'X-Recording-Source-Key': chunk.sourceKey,
+      'X-Recording-Sequence': '${chunk.sequence}',
+      'X-Recording-Start-Ms': '${chunk.startMs}',
+      'X-Recording-End-Ms': '${chunk.endMs}',
+    };
     _log(
-      'chunk.upload.request',
+      'chunk.upload.attempt',
       data: <String, Object?>{
         'sessionId': sessionId,
-        'sourceKey': sourceKey,
-        'sequence': sequence,
-        'startMs': startMs,
-        'endMs': endMs,
-        'size': bytes.length,
-        'mimeType': mimeType,
+        'sourceKey': chunk.sourceKey,
+        'sequence': chunk.sequence,
+        'size': chunk.bytes.length,
+        'headers': _filterHeadersForLog(headers),
       },
     );
-    await _requestWithRetry(
+    await html.HttpRequest.request(
       uri.toString(),
-      headers: <String, String>{
-        'Content-Type': mimeType,
-        'X-Recording-Source-Key': sourceKey,
-        'X-Recording-Sequence': '$sequence',
-        'X-Recording-Start-Ms': '$startMs',
-        'X-Recording-End-Ms': '$endMs',
-      },
-      body: bytes,
+      method: 'POST',
+      sendData: chunk.bytes,
+      requestHeaders: headers,
+      withCredentials: true,
     );
     _log(
       'chunk.upload.done',
       data: <String, Object?>{
         'sessionId': sessionId,
-        'sourceKey': sourceKey,
-        'sequence': sequence,
-        'size': bytes.length,
+        'sourceKey': chunk.sourceKey,
+        'sequence': chunk.sequence,
+        'size': chunk.bytes.length,
       },
     );
   }
@@ -551,6 +592,11 @@ class WebRecordingBridge extends RecordingBridge {
       futures.add(_waitForStop(_screenRecorder!));
       _screenRecorder!.stop();
     }
+    if (_systemAudioRecorder != null &&
+        _systemAudioRecorder!.state != 'inactive') {
+      futures.add(_waitForStop(_systemAudioRecorder!));
+      _systemAudioRecorder!.stop();
+    }
     if (_microphoneRecorder != null &&
         _microphoneRecorder!.state != 'inactive') {
       futures.add(_waitForStop(_microphoneRecorder!));
@@ -563,55 +609,22 @@ class WebRecordingBridge extends RecordingBridge {
     await _displayEndedSub?.cancel();
     _displayEndedSub = null;
     _displayStream?.getTracks().forEach((track) => track.stop());
+    _systemAudioStream?.getTracks().forEach((track) => track.stop());
     _microphoneStream?.getTracks().forEach((track) => track.stop());
     _displayStream = null;
+    _systemAudioStream = null;
     _microphoneStream = null;
     _screenRecorder = null;
+    _systemAudioRecorder = null;
     _microphoneRecorder = null;
+    for (final queue in _uploadQueues.values) {
+      await queue.dispose();
+    }
+    _uploadQueues.clear();
     _stopwatch?.stop();
     _stopwatch = null;
   }
 
-  Future<void> _requestWithRetry(
-    String url, {
-    required Map<String, String> headers,
-    required Uint8List body,
-  }) async {
-    Object? lastError;
-    for (var attempt = 0; attempt < 3; attempt += 1) {
-      try {
-        _log(
-          'chunk.upload.attempt',
-          data: <String, Object?>{
-            'url': url,
-            'attempt': attempt + 1,
-            'size': body.length,
-            'headers': _filterHeadersForLog(headers),
-          },
-        );
-        await html.HttpRequest.request(
-          url,
-          method: 'POST',
-          sendData: body,
-          requestHeaders: headers,
-          withCredentials: true,
-        );
-        return;
-      } catch (error) {
-        lastError = error;
-        _log(
-          'chunk.upload.attempt_failed',
-          data: <String, Object?>{'url': url, 'attempt': attempt + 1},
-          error: error,
-        );
-        await Future<void>.delayed(Duration(milliseconds: 400 * (attempt + 1)));
-      }
-    }
-    throw RecordingBridgeException(
-      'Could not upload a recording chunk: ${lastError ?? 'unknown error'}',
-    );
-  }
-
   Map<String, String> _filterHeadersForLog(Map<String, String> headers) {
     final filtered = <String, String>{};
     headers.forEach((key, value) {