neoagent 3.1.1-beta.9 → 3.2.0

package/flutter_app/lib/src/recording_bridge_web.dart

@@ -1,702 +0,0 @@
-// ignore_for_file: avoid_web_libraries_in_flutter, deprecated_member_use
-
-import 'dart:async';
-import 'dart:html' as html;
-import 'dart:js_util' as js_util;
-import 'dart:typed_data';
-
-import 'diagnostics_logger.dart';
-import 'recording_bridge.dart';
-import 'recording_chunk_queue.dart';
-
-RecordingBridge createPlatformRecordingBridge() => WebRecordingBridge();
-
-class WebRecordingBridge extends RecordingBridge {
-  RecordingRuntimeStatus _status = const RecordingRuntimeStatus(
-    supportsScreenAndMic: true,
-    supportsBackgroundMic: false,
-    platformLabel: 'Browser screen + microphone recorder',
-  );
-
-  html.MediaStream? _displayStream;
-  html.MediaStream? _systemAudioStream;
-  html.MediaStream? _microphoneStream;
-  html.MediaRecorder? _screenRecorder;
-  html.MediaRecorder? _systemAudioRecorder;
-  html.MediaRecorder? _microphoneRecorder;
-  final Map<String, int> _nextSequenceBySource = <String, int>{};
-  final Map<String, int> _lastEndMsBySource = <String, int>{};
-  // One durable, retry-until-success queue per source so a failed upload is
-  // re-sent instead of silently dropped, and a stall in one source does not
-  // block the others.
-  final Map<String, MemoryChunkUploadQueue> _uploadQueues =
-      <String, MemoryChunkUploadQueue>{};
-  Stopwatch? _stopwatch;
-  String? _baseUrl;
-  String? _sessionId;
-  bool _stopping = false;
-  StreamSubscription<html.Event>? _displayEndedSub;
-  static const Set<String> _safeUploadHeaderKeys = <String>{
-    'content-type',
-    'x-recording-sequence',
-    'x-recording-start-ms',
-    'x-recording-end-ms',
-  };
-
-  @override
-  RecordingRuntimeStatus get status => _status;
-
-  void _log(
-    String event, {
-    Map<String, Object?> data = const <String, Object?>{},
-    Object? error,
-    StackTrace? stackTrace,
-  }) {
-    AppDiagnostics.log(
-      'recording.bridge.web',
-      event,
-      data: data,
-      error: error,
-      stackTrace: stackTrace,
-    );
-  }
-
-  @override
-  Future<void> refreshStatus() async {}
-
-  @override
-  Future<void> startWebRecording({
-    required String baseUrl,
-    required String sessionId,
-  }) async {
-    _log(
-      'start_web.request',
-      data: <String, Object?>{
-        'sessionId': sessionId,
-        'baseUrl': baseUrl,
-        'alreadyActive': _status.active,
-      },
-    );
-    if (_status.active) {
-      throw const RecordingBridgeException(
-        'A recording is already in progress.',
-      );
-    }
-
-    try {
-      final mediaDevices = html.window.navigator.mediaDevices;
-      if (mediaDevices == null) {
-        throw const RecordingBridgeException(
-          'This browser does not expose media device APIs.',
-        );
-      }
-
-      final displayStream = await js_util.promiseToFuture<html.MediaStream>(
-        js_util.callMethod(mediaDevices, 'getDisplayMedia', <Object?>[
-          js_util.jsify(<String, Object>{'video': true, 'audio': true}),
-        ]),
-      );
-      if (displayStream.getVideoTracks().isEmpty) {
-        throw const RecordingBridgeException(
-          'Screen sharing was cancelled before capture started.',
-        );
-      }
-      if (displayStream.getAudioTracks().isEmpty) {
-        displayStream.getTracks().forEach((track) => track.stop());
-        throw const RecordingBridgeException(
-          'Screen share did not include audio. Share a tab/window and enable audio.',
-        );
-      }
-
-      final microphoneStream = await mediaDevices.getUserMedia(
-        <String, dynamic>{
-          'audio': <String, dynamic>{
-            'channelCount': 1,
-            'echoCancellation': false,
-            'noiseSuppression': false,
-            'autoGainControl': false,
-          },
-        },
-      );
-      if (microphoneStream.getAudioTracks().isEmpty) {
-        displayStream.getTracks().forEach((track) => track.stop());
-        throw const RecordingBridgeException(
-          'Microphone permission is required to start recording.',
-        );
-      }
-
-      // Split the display capture: video-only feeds the screen-analysis source,
-      // while the shared system/tab audio gets its own transcribable source so
-      // remote participants are captured in the transcript (the mic alone does
-      // not reliably pick up other speakers, e.g. when wearing headphones).
-      final screenVideoStream = html.MediaStream();
-      for (final track in displayStream.getVideoTracks()) {
-        screenVideoStream.addTrack(track);
-      }
-      final systemAudioStream = html.MediaStream();
-      for (final track in displayStream.getAudioTracks()) {
-        systemAudioStream.addTrack(track);
-      }
-
-      _baseUrl = baseUrl;
-      _sessionId = sessionId;
-      _displayStream = displayStream;
-      _systemAudioStream = systemAudioStream;
-      _microphoneStream = microphoneStream;
-      _nextSequenceBySource
-        ..clear()
-        ..addAll(<String, int>{'screen': 0, 'system': 0, 'microphone': 0});
-      _lastEndMsBySource
-        ..clear()
-        ..addAll(<String, int>{'screen': 0, 'system': 0, 'microphone': 0});
-      _initUploadQueues(<String>['screen', 'system', 'microphone']);
-      _stopwatch = Stopwatch()..start();
-
-      final screenMimeType = _pickMimeType(<String>[
-        'video/webm;codecs=vp9,opus',
-        'video/webm;codecs=vp8,opus',
-        'video/webm',
-      ]);
-      final audioMimeType = _pickMimeType(<String>[
-        'audio/webm;codecs=opus',
-        'audio/webm',
-      ]);
-
-      _screenRecorder = html.MediaRecorder(
-        screenVideoStream,
-        screenMimeType == null
-            ? null
-            : <String, String>{'mimeType': screenMimeType},
-      );
-
-      _displayEndedSub = displayStream.getVideoTracks().first.onEnded.listen((
-        _,
-      ) {
-        unawaited(_handleExternalStop());
-      });
-
-      _bindRecorder(
-        recorder: _screenRecorder!,
-        sourceKey: 'screen',
-        mimeType: screenMimeType ?? 'video/webm',
-      );
-      _systemAudioRecorder = html.MediaRecorder(
-        systemAudioStream,
-        audioMimeType == null ? null : <String, String>{'mimeType': audioMimeType},
-      );
-      _bindRecorder(
-        recorder: _systemAudioRecorder!,
-        sourceKey: 'system',
-        mimeType: audioMimeType ?? 'audio/webm',
-      );
-      _microphoneRecorder = html.MediaRecorder(
-        microphoneStream,
-        audioMimeType == null ? null : <String, String>{'mimeType': audioMimeType},
-      );
-      _bindRecorder(
-        recorder: _microphoneRecorder!,
-        sourceKey: 'microphone',
-        mimeType: audioMimeType ?? 'audio/webm',
-      );
-
-      _screenRecorder!.start(4000);
-      _systemAudioRecorder!.start(4000);
-      _microphoneRecorder!.start(4000);
-      _status = _status.copyWith(
-        active: true,
-        paused: false,
-        sessionId: sessionId,
-        startedAt: DateTime.now(),
-        errorMessage: null,
-      );
-      _log(
-        'start_web.done',
-        data: <String, Object?>{
-          'sessionId': sessionId,
-          'screenMimeType': screenMimeType ?? 'video/webm',
-          'systemMimeType': audioMimeType ?? 'audio/webm',
-          'micMimeType': audioMimeType ?? 'audio/webm',
-        },
-      );
-      notifyListeners();
-    } catch (error) {
-      await _disposeStreams();
-      _status = _status.copyWith(
-        active: false,
-        paused: false,
-        sessionId: null,
-        startedAt: null,
-        errorMessage: error.toString(),
-      );
-      _log('start_web.failed', error: error);
-      notifyListeners();
-      rethrow;
-    }
-  }
-
-  @override
-  Future<void> startWebMicrophoneRecording({
-    required String baseUrl,
-    required String sessionId,
-  }) async {
-    _log(
-      'start_web_mic_only.request',
-      data: <String, Object?>{
-        'sessionId': sessionId,
-        'baseUrl': baseUrl,
-        'alreadyActive': _status.active,
-      },
-    );
-    if (_status.active) {
-      throw const RecordingBridgeException(
-        'A recording is already in progress.',
-      );
-    }
-
-    try {
-      final mediaDevices = html.window.navigator.mediaDevices;
-      if (mediaDevices == null) {
-        throw const RecordingBridgeException(
-          'This browser does not expose media device APIs.',
-        );
-      }
-
-      final microphoneStream = await mediaDevices.getUserMedia(
-        <String, dynamic>{
-          'audio': <String, dynamic>{
-            'channelCount': 1,
-            'echoCancellation': false,
-            'noiseSuppression': false,
-            'autoGainControl': false,
-          },
-        },
-      );
-      if (microphoneStream.getAudioTracks().isEmpty) {
-        throw const RecordingBridgeException(
-          'Microphone permission is required to start recording.',
-        );
-      }
-
-      _baseUrl = baseUrl;
-      _sessionId = sessionId;
-      _displayStream = null;
-      _microphoneStream = microphoneStream;
-      _nextSequenceBySource
-        ..clear()
-        ..addAll(<String, int>{'microphone': 0});
-      _lastEndMsBySource
-        ..clear()
-        ..addAll(<String, int>{'microphone': 0});
-      _initUploadQueues(<String>['microphone']);
-      _stopwatch = Stopwatch()..start();
-
-      final micMimeType = _pickMimeType(<String>[
-        'audio/webm;codecs=opus',
-        'audio/webm',
-      ]);
-
-      _displayEndedSub = null;
-      _screenRecorder = null;
-      _microphoneRecorder = html.MediaRecorder(
-        microphoneStream,
-        micMimeType == null ? null : <String, String>{'mimeType': micMimeType},
-      );
-      _bindRecorder(
-        recorder: _microphoneRecorder!,
-        sourceKey: 'microphone',
-        mimeType: micMimeType ?? 'audio/webm',
-      );
-
-      _microphoneRecorder!.start(4000);
-      _status = _status.copyWith(
-        active: true,
-        paused: false,
-        sessionId: sessionId,
-        startedAt: DateTime.now(),
-        errorMessage: null,
-      );
-      _log(
-        'start_web_mic_only.done',
-        data: <String, Object?>{
-          'sessionId': sessionId,
-          'micMimeType': micMimeType ?? 'audio/webm',
-        },
-      );
-      notifyListeners();
-    } catch (error) {
-      await _disposeStreams();
-      _status = _status.copyWith(
-        active: false,
-        paused: false,
-        sessionId: null,
-        startedAt: null,
-        errorMessage: error.toString(),
-      );
-      _log('start_web_mic_only.failed', error: error);
-      notifyListeners();
-      rethrow;
-    }
-  }
-
-  @override
-  Future<void> startBackgroundRecording({
-    required String baseUrl,
-    required String sessionCookie,
-    required String sessionId,
-  }) async {
-    throw const RecordingBridgeException(
-      'Background microphone recording is available on Android only.',
-    );
-  }
-
-  @override
-  Future<void> startDesktopAudioRecording({
-    required String baseUrl,
-    required String sessionCookie,
-    required String sessionId,
-    bool autoShowToolbar = true,
-  }) async {
-    throw const RecordingBridgeException(
-      'Native desktop audio recording is not available in the browser.',
-    );
-  }
-
-  @override
-  Future<void> pauseBackgroundRecording() async {
-    throw const RecordingBridgeException(
-      'Pause is only available for Android background recording.',
-    );
-  }
-
-  @override
-  Future<void> pauseDesktopRecording() async {
-    throw const RecordingBridgeException(
-      'Desktop recording controls are not available in the browser.',
-    );
-  }
-
-  @override
-  Future<void> resumeBackgroundRecording() async {
-    throw const RecordingBridgeException(
-      'Resume is only available for Android background recording.',
-    );
-  }
-
-  @override
-  Future<void> resumeDesktopRecording() async {
-    throw const RecordingBridgeException(
-      'Desktop recording controls are not available in the browser.',
-    );
-  }
-
-  @override
-  Future<void> showFloatingToolbar() async {}
-
-  @override
-  Future<void> hideFloatingToolbar() async {}
-
-  @override
-  Future<void> openMicrophoneSettings() async {
-    throw const RecordingBridgeException(
-      'Open the browser site settings to manage microphone access.',
-    );
-  }
-
-  @override
-  Future<void> openSystemAudioSettings() async {
-    throw const RecordingBridgeException(
-      'Browser screen-share audio permissions are controlled by the browser.',
-    );
-  }
-
-  @override
-  Future<void> stopActiveRecording({bool notifyEnded = false}) async {
-    if (_stopping || !_status.active) {
-      return;
-    }
-    _stopping = true;
-    final sessionId = _sessionId;
-    _log(
-      'stop_active.request',
-      data: <String, Object?>{
-        'sessionId': sessionId,
-        'notifyEnded': notifyEnded,
-        'active': _status.active,
-      },
-    );
-    try {
-      await _stopRecorders();
-      await Future.wait(
-        _uploadQueues.values.map((queue) => queue.flush()),
-      );
-      await _disposeStreams();
-      _status = _status.copyWith(
-        active: false,
-        paused: false,
-        startedAt: null,
-        sessionId: sessionId,
-      );
-      _log(
-        'stop_active.done',
-        data: <String, Object?>{
-          'sessionId': sessionId,
-          'notifyEnded': notifyEnded,
-        },
-      );
-      notifyListeners();
-      if (notifyEnded && sessionId != null && onRecordingStopped != null) {
-        _log(
-          'stop_active.notify_ended',
-          data: <String, Object?>{'sessionId': sessionId},
-        );
-        await onRecordingStopped!(sessionId);
-      }
-    } finally {
-      _stopping = false;
-    }
-  }
-
-  void _initUploadQueues(List<String> sourceKeys) {
-    for (final queue in _uploadQueues.values) {
-      unawaited(queue.dispose());
-    }
-    _uploadQueues
-      ..clear()
-      ..addEntries(
-        sourceKeys.map(
-          (sourceKey) => MapEntry<String, MemoryChunkUploadQueue>(
-            sourceKey,
-            MemoryChunkUploadQueue(
-              uploader: _performChunkUpload,
-              logger:
-                  (
-                    event, {
-                    Map<String, Object?> data = const <String, Object?>{},
-                    Object? error,
-                  }) => _log(event, data: data, error: error),
-            ),
-          ),
-        ),
-      );
-  }
-
-  void _bindRecorder({
-    required html.MediaRecorder recorder,
-    required String sourceKey,
-    required String mimeType,
-  }) {
-    recorder.addEventListener('dataavailable', (html.Event event) {
-      final blob = js_util.getProperty<html.Blob?>(event, 'data');
-      if (blob == null || blob.size == 0) {
-        return;
-      }
-      final endMs = _stopwatch?.elapsedMilliseconds ?? 0;
-      final startMs = _lastEndMsBySource[sourceKey] ?? 0;
-      _lastEndMsBySource[sourceKey] = endMs;
-      final sequence = _nextSequenceBySource[sourceKey] ?? 0;
-      _nextSequenceBySource[sourceKey] = sequence + 1;
-      final queue = _uploadQueues[sourceKey];
-      if (queue == null) {
-        return;
-      }
-      // Read the blob bytes, then hand the chunk to the durable queue, which
-      // uploads and retries until the server confirms it (so a transient
-      // failure no longer drops the chunk or stalls the rest of the source).
-      unawaited(() async {
-        try {
-          final bytes = await _blobToBytes(blob);
-          queue.enqueue(
-            PendingChunk(
-              sourceKey: sourceKey,
-              sequence: sequence,
-              startMs: startMs,
-              endMs: endMs,
-              mimeType: mimeType,
-              bytes: bytes,
-            ),
-          );
-        } catch (error, stackTrace) {
-          _log(
-            'chunk.read.failed',
-            data: <String, Object?>{
-              'sourceKey': sourceKey,
-              'sequence': sequence,
-            },
-            error: error,
-            stackTrace: stackTrace,
-          );
-        }
-      }());
-    });
-  }
-
-  Future<void> _performChunkUpload(PendingChunk chunk) async {
-    final sessionId = _sessionId;
-    final baseUrl = _baseUrl;
-    if (sessionId == null || baseUrl == null) {
-      throw const RecordingBridgeException(
-        'Recording session is not initialized.',
-      );
-    }
-    final uri = _resolveUri(baseUrl, '/api/recordings/$sessionId/chunks');
-    final headers = <String, String>{
-      'Content-Type': chunk.mimeType,
-      'X-Recording-Source-Key': chunk.sourceKey,
-      'X-Recording-Sequence': '${chunk.sequence}',
-      'X-Recording-Start-Ms': '${chunk.startMs}',
-      'X-Recording-End-Ms': '${chunk.endMs}',
-    };
-    _log(
-      'chunk.upload.attempt',
-      data: <String, Object?>{
-        'sessionId': sessionId,
-        'sourceKey': chunk.sourceKey,
-        'sequence': chunk.sequence,
-        'size': chunk.bytes.length,
-        'headers': _filterHeadersForLog(headers),
-      },
-    );
-    await html.HttpRequest.request(
-      uri.toString(),
-      method: 'POST',
-      sendData: chunk.bytes,
-      requestHeaders: headers,
-      withCredentials: true,
-    );
-    _log(
-      'chunk.upload.done',
-      data: <String, Object?>{
-        'sessionId': sessionId,
-        'sourceKey': chunk.sourceKey,
-        'sequence': chunk.sequence,
-        'size': chunk.bytes.length,
-      },
-    );
-  }
-
-  Future<void> _handleExternalStop() async {
-    final sessionId = _sessionId;
-    if (sessionId == null) {
-      return;
-    }
-    _log(
-      'external_stop.detected',
-      data: <String, Object?>{'sessionId': sessionId},
-    );
-    await stopActiveRecording(notifyEnded: true);
-  }
-
-  Future<void> _stopRecorders() async {
-    final futures = <Future<void>>[];
-    if (_screenRecorder != null && _screenRecorder!.state != 'inactive') {
-      futures.add(_waitForStop(_screenRecorder!));
-      _screenRecorder!.stop();
-    }
-    if (_systemAudioRecorder != null &&
-        _systemAudioRecorder!.state != 'inactive') {
-      futures.add(_waitForStop(_systemAudioRecorder!));
-      _systemAudioRecorder!.stop();
-    }
-    if (_microphoneRecorder != null &&
-        _microphoneRecorder!.state != 'inactive') {
-      futures.add(_waitForStop(_microphoneRecorder!));
-      _microphoneRecorder!.stop();
-    }
-    await Future.wait(futures);
-  }
-
-  Future<void> _disposeStreams() async {
-    await _displayEndedSub?.cancel();
-    _displayEndedSub = null;
-    _displayStream?.getTracks().forEach((track) => track.stop());
-    _systemAudioStream?.getTracks().forEach((track) => track.stop());
-    _microphoneStream?.getTracks().forEach((track) => track.stop());
-    _displayStream = null;
-    _systemAudioStream = null;
-    _microphoneStream = null;
-    _screenRecorder = null;
-    _systemAudioRecorder = null;
-    _microphoneRecorder = null;
-    for (final queue in _uploadQueues.values) {
-      await queue.dispose();
-    }
-    _uploadQueues.clear();
-    _stopwatch?.stop();
-    _stopwatch = null;
-  }
-
-  Map<String, String> _filterHeadersForLog(Map<String, String> headers) {
-    final filtered = <String, String>{};
-    headers.forEach((key, value) {
-      if (_safeUploadHeaderKeys.contains(key.toLowerCase())) {
-        filtered[key] = value;
-      }
-    });
-    return filtered;
-  }
-
-  Future<Uint8List> _blobToBytes(html.Blob blob) {
-    final completer = Completer<Uint8List>();
-    final reader = html.FileReader();
-    reader.onError.first.then((_) {
-      if (!completer.isCompleted) {
-        completer.completeError(
-          const RecordingBridgeException(
-            'Could not read recorded browser chunk.',
-          ),
-        );
-      }
-    });
-    reader.onLoadEnd.first.then((_) {
-      if (completer.isCompleted) {
-        return;
-      }
-      final result = reader.result;
-      if (result is ByteBuffer) {
-        completer.complete(result.asUint8List());
-        return;
-      }
-      if (result is Uint8List) {
-        completer.complete(result);
-        return;
-      }
-      completer.completeError(
-        const RecordingBridgeException(
-          'Browser returned an unsupported chunk format.',
-        ),
-      );
-    });
-    reader.readAsArrayBuffer(blob);
-    return completer.future;
-  }
-
-  Future<void> _waitForStop(html.MediaRecorder recorder) {
-    final completer = Completer<void>();
-    late html.EventListener listener;
-    listener = (html.Event _) {
-      recorder.removeEventListener('stop', listener);
-      if (!completer.isCompleted) {
-        completer.complete();
-      }
-    };
-    recorder.addEventListener('stop', listener);
-    return completer.future;
-  }
-
-  String? _pickMimeType(List<String> candidates) {
-    for (final candidate in candidates) {
-      if (html.MediaRecorder.isTypeSupported(candidate)) {
-        return candidate;
-      }
-    }
-    return null;
-  }
-
-  Uri _resolveUri(String baseUrl, String path) {
-    final trimmed = baseUrl.trim();
-    if (trimmed.isEmpty) {
-      return Uri.parse(path);
-    }
-    return Uri.parse(trimmed.replaceFirst(RegExp(r'/$'), '') + path);
-  }
-}