neoagent 2.3.1-beta.2 → 2.3.1-beta.21

package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart

@@ -0,0 +1,106 @@
+/// Decibel data from audio capture.
+///
+/// This class represents a single decibel reading with its timestamp.
+/// Used by both microphone and system audio capture to provide volume level
+/// information.
+///
+/// Example:
+/// ```dart
+/// // From stream
+/// capture.decibelStream?.listen((data) {
+///   print('Decibel: ${data.decibel} dB');
+///   print('Time: ${DateTime.fromMillisecondsSinceEpoch((data.timestamp * 1000).toInt())}');
+/// });
+///
+/// // Create manually
+/// final data = DecibelData(
+///   decibel: -45.5,
+///   timestamp: DateTime.now().millisecondsSinceEpoch / 1000.0,
+/// );
+///
+/// // Convert to/from map
+/// final map = data.toMap();
+/// final restored = DecibelData.fromMap(map);
+/// ```
+class DecibelData {
+  /// Decibel value in dB, typically ranging from -120 to 0 dB.
+  ///
+  /// - -120 dB: silence or very quiet
+  /// - -60 dB: quiet background noise
+  /// - -40 dB: normal speech
+  /// - -20 dB: loud speech
+  /// - 0 dB: maximum level
+  final double decibel;
+
+  /// Unix timestamp in seconds (not milliseconds).
+  ///
+  /// This represents when the decibel reading was taken.
+  final double timestamp;
+
+  /// Creates a new [DecibelData] instance.
+  ///
+  /// [decibel] should be in the range -120 to 0 dB.
+  /// [timestamp] should be a Unix timestamp in seconds.
+  ///
+  /// Example:
+  /// ```dart
+  /// final data = DecibelData(
+  ///   decibel: -45.0,
+  ///   timestamp: DateTime.now().millisecondsSinceEpoch / 1000.0,
+  /// );
+  /// ```
+  const DecibelData({
+    required this.decibel,
+    required this.timestamp,
+  });
+
+  /// Creates a [DecibelData] instance from a map.
+  ///
+  /// The map should contain:
+  /// - `decibel`: num (will be converted to double)
+  /// - `timestamp`: num (will be converted to double)
+  ///
+  /// If values are missing, defaults to -120.0 dB and current timestamp.
+  ///
+  /// Example:
+  /// ```dart
+  /// final map = {
+  ///   'decibel': -45.5,
+  ///   'timestamp': 1234567890.0,
+  /// };
+  /// final data = DecibelData.fromMap(map);
+  /// ```
+  factory DecibelData.fromMap(Map<String, dynamic> map) {
+    return DecibelData(
+      decibel: (map['decibel'] as num?)?.toDouble() ?? -120.0,
+      timestamp: (map['timestamp'] as num?)?.toDouble() ??
+          DateTime.now().millisecondsSinceEpoch / 1000.0,
+    );
+  }
+
+  /// Converts this [DecibelData] instance to a map.
+  ///
+  /// Returns a map containing:
+  /// - `decibel`: double
+  /// - `timestamp`: double
+  ///
+  /// Example:
+  /// ```dart
+  /// final data = DecibelData(
+  ///   decibel: -45.0,
+  ///   timestamp: 1234567890.0,
+  /// );
+  /// final map = data.toMap();
+  /// // map = {'decibel': -45.0, 'timestamp': 1234567890.0}
+  /// ```
+  Map<String, dynamic> toMap() {
+    return {
+      'decibel': decibel,
+      'timestamp': timestamp,
+    };
+  }
+
+  @override
+  String toString() =>
+      'DecibelData(decibel: ${decibel.toStringAsFixed(1)} dB, timestamp: $timestamp)';
+}

package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart

@@ -0,0 +1,219 @@
+/// Enum representing the type of input device (microphone).
+///
+/// This enum categorizes input devices into three types: built-in, Bluetooth,
+/// or external (USB, etc.).
+///
+/// Example:
+/// ```dart
+/// final device = InputDevice(
+///   id: 'device1',
+///   name: 'Built-in Microphone',
+///   type: InputDeviceType.builtIn,
+///   channelCount: 1,
+///   isDefault: true,
+/// );
+///
+/// // Convert to/from string
+/// final typeString = device.type.toString(); // 'built-in'
+/// final type = InputDeviceType.fromString('bluetooth'); // InputDeviceType.bluetooth
+/// ```
+enum InputDeviceType {
+  /// Built-in device (e.g., laptop microphone)
+  builtIn,
+
+  /// Bluetooth device (wireless microphone)
+  bluetooth,
+
+  /// External device (USB microphone, etc.)
+  external;
+
+  /// Creates an [InputDeviceType] from a string.
+  ///
+  /// Accepts: 'built-in', 'bluetooth', 'external' (case-insensitive).
+  /// Returns [InputDeviceType.external] for unknown values.
+  ///
+  /// Example:
+  /// ```dart
+  /// final type1 = InputDeviceType.fromString('built-in'); // InputDeviceType.builtIn
+  /// final type2 = InputDeviceType.fromString('BLUETOOTH'); // InputDeviceType.bluetooth
+  /// final type3 = InputDeviceType.fromString('unknown'); // InputDeviceType.external (default)
+  /// ```
+  static InputDeviceType fromString(String type) {
+    switch (type.toLowerCase()) {
+      case 'built-in':
+        return InputDeviceType.builtIn;
+      case 'bluetooth':
+        return InputDeviceType.bluetooth;
+      case 'external':
+        return InputDeviceType.external;
+      default:
+        return InputDeviceType.external;
+    }
+  }
+
+  /// Converts this [InputDeviceType] to a string representation.
+  ///
+  /// Returns: 'built-in', 'bluetooth', or 'external'.
+  ///
+  /// Example:
+  /// ```dart
+  /// InputDeviceType.builtIn.toString(); // 'built-in'
+  /// InputDeviceType.bluetooth.toString(); // 'bluetooth'
+  /// InputDeviceType.external.toString(); // 'external'
+  /// ```
+  @override
+  String toString() {
+    switch (this) {
+      case InputDeviceType.builtIn:
+        return 'built-in';
+      case InputDeviceType.bluetooth:
+        return 'bluetooth';
+      case InputDeviceType.external:
+        return 'external';
+    }
+  }
+}
+
+/// Class representing information about an input device (microphone).
+///
+/// This class contains all relevant information about a microphone device,
+/// including its ID, name, type, channel count, and whether it's the default device.
+///
+/// Example:
+/// ```dart
+/// // Get available devices
+/// final devices = await micCapture.getAvailableInputDevices();
+///
+/// // Find a specific device
+/// final usbMic = devices.firstWhere(
+///   (device) => device.name.contains('USB'),
+/// );
+///
+/// print('Device: ${usbMic.name}');
+/// print('Type: ${usbMic.type}');
+/// print('Channels: ${usbMic.channelCount}');
+/// print('Default: ${usbMic.isDefault}');
+///
+/// // Convert to/from map
+/// final map = usbMic.toMap();
+/// final restored = InputDevice.fromMap(map);
+/// ```
+class InputDevice {
+  /// Unique identifier of the device.
+  ///
+  /// This ID can be used to identify and select a specific device.
+  final String id;
+
+  /// Human-readable name of the device.
+  ///
+  /// Examples: "Built-in Microphone", "USB Microphone", "AirPods Pro"
+  final String name;
+
+  /// Type of the device (built-in, Bluetooth, or external).
+  final InputDeviceType type;
+
+  /// Number of audio channels supported by the device.
+  ///
+  /// Typically 1 for mono, 2 for stereo.
+  final int channelCount;
+
+  /// Whether this device is the system default input device.
+  final bool isDefault;
+
+  /// Creates a new [InputDevice] instance.
+  ///
+  /// All parameters are required.
+  ///
+  /// Example:
+  /// ```dart
+  /// final device = InputDevice(
+  ///   id: 'device-123',
+  ///   name: 'Built-in Microphone',
+  ///   type: InputDeviceType.builtIn,
+  ///   channelCount: 1,
+  ///   isDefault: true,
+  /// );
+  /// ```
+  const InputDevice({
+    required this.id,
+    required this.name,
+    required this.type,
+    required this.channelCount,
+    required this.isDefault,
+  });
+
+  /// Creates an [InputDevice] instance from a map.
+  ///
+  /// The map should contain:
+  /// - `id`: String (defaults to empty string if missing)
+  /// - `name`: String (defaults to empty string if missing)
+  /// - `type`: String (converted via [InputDeviceType.fromString], defaults to external)
+  /// - `channelCount`: int (defaults to 0 if missing)
+  /// - `isDefault`: bool (defaults to false if missing)
+  ///
+  /// Example:
+  /// ```dart
+  /// final map = {
+  ///   'id': 'device-123',
+  ///   'name': 'USB Microphone',
+  ///   'type': 'external',
+  ///   'channelCount': 2,
+  ///   'isDefault': false,
+  /// };
+  /// final device = InputDevice.fromMap(map);
+  /// ```
+  factory InputDevice.fromMap(Map<String, dynamic> map) {
+    return InputDevice(
+      id: map['id'] as String? ?? '',
+      name: map['name'] as String? ?? '',
+      type: InputDeviceType.fromString(map['type'] as String? ?? 'external'),
+      channelCount: map['channelCount'] as int? ?? 0,
+      isDefault: map['isDefault'] as bool? ?? false,
+    );
+  }
+
+  /// Converts this [InputDevice] instance to a map.
+  ///
+  /// Returns a map containing all device information:
+  /// - `id`: String
+  /// - `name`: String
+  /// - `type`: String (from [InputDeviceType.toString])
+  /// - `channelCount`: int
+  /// - `isDefault`: bool
+  ///
+  /// Example:
+  /// ```dart
+  /// final device = InputDevice(
+  ///   id: 'device-123',
+  ///   name: 'USB Microphone',
+  ///   type: InputDeviceType.external,
+  ///   channelCount: 2,
+  ///   isDefault: false,
+  /// );
+  /// final map = device.toMap();
+  /// ```
+  Map<String, dynamic> toMap() {
+    return {
+      'id': id,
+      'name': name,
+      'type': type.toString(),
+      'channelCount': channelCount,
+      'isDefault': isDefault,
+    };
+  }
+
+  @override
+  String toString() {
+    return 'InputDevice(id: $id, name: $name, type: ${type.toString()}, '
+        'channelCount: $channelCount, isDefault: $isDefault)';
+  }
+
+  @override
+  bool operator ==(Object other) {
+    if (identical(this, other)) return true;
+    return other is InputDevice && other.id == id;
+  }
+
+  @override
+  int get hashCode => id.hashCode;
+}

package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart

@@ -0,0 +1,336 @@
+import 'dart:async';
+
+import 'package:desktop_audio_capture/audio_capture.dart';
+import 'package:flutter/services.dart';
+
+export 'package:desktop_audio_capture/config/system_adudio_config.dart';
+
+enum _SystemAudioMethod {
+  startCapture,
+  stopCapture,
+  requestPermissions,
+}
+
+/// Class for capturing system audio (audio output from the device).
+///
+/// This class allows you to capture audio that is being played by the system,
+/// such as music, videos, or other applications. It requires screen recording
+/// permissions on macOS.
+///
+/// Example:
+/// ```dart
+/// final systemCapture = SystemAudioCapture(
+///   config: SystemAudioConfig(
+///     sampleRate: 44100,
+///     channels: 2,
+///   ),
+/// );
+///
+/// await systemCapture.startCapture();
+///
+/// // Listen to audio stream
+/// systemCapture.audioStream?.listen((audioData) {
+///   // Process audio bytes
+///   print('Received ${audioData.length} bytes');
+/// });
+///
+/// // Listen to decibel readings
+/// systemCapture.decibelStream?.listen((data) {
+///   print('Decibel: ${data.decibel} dB');
+/// });
+///
+/// // Stop when done
+/// await systemCapture.stopCapture();
+/// ```
+class SystemAudioCapture extends AudioCapture {
+  static const MethodChannel _channel = MethodChannel(
+    'com.system_audio_transcriber/audio_capture',
+  );
+  static const EventChannel _audioStreamChannel = EventChannel(
+    'com.system_audio_transcriber/audio_stream',
+  );
+  static const EventChannel _statusStreamChannel = EventChannel(
+    'com.system_audio_transcriber/audio_status',
+  );
+  static const EventChannel _decibelStreamChannel = EventChannel(
+    'com.system_audio_transcriber/audio_decibel',
+  );
+
+  Stream<Uint8List>? _audioStream;
+  Stream<SystemAudioStatus>? _statusStream;
+  Stream<DecibelData>? _decibelStream;
+  bool _isRecording = false;
+
+  /// Stream of raw audio data bytes from system audio capture.
+  ///
+  /// Returns a [Stream<Uint8List>] containing the captured audio data.
+  /// The stream is only available after [startCapture] has been called.
+  ///
+  /// Example:
+  /// ```dart
+  /// await systemCapture.startCapture();
+  ///
+  /// systemCapture.audioStream?.listen((audioData) {
+  ///   // Process audio bytes
+  ///   final audioBuffer = audioData.buffer.asUint8List();
+  ///   // Use audio buffer for processing, saving, etc.
+  /// });
+  /// ```
+  Stream<Uint8List>? get audioStream => _audioStream;
+
+  /// Stream of system audio capture status updates.
+  ///
+  /// Returns a [Stream<SystemAudioStatus>] containing status information:
+  /// - [SystemAudioStatus.isActive]: bool - whether system audio capture is currently active
+  ///
+  /// Example:
+  /// ```dart
+  /// systemCapture.statusStream?.listen((status) {
+  ///   if (status.isActive) {
+  ///     print('System audio capture is active');
+  ///   } else {
+  ///     print('System audio capture is inactive');
+  ///   }
+  /// });
+  /// ```
+  Stream<SystemAudioStatus>? get statusStream {
+    // Create status stream if not already created
+    _statusStream ??= _statusStreamChannel.receiveBroadcastStream().map((
+      dynamic event,
+    ) {
+      if (event is Map) {
+        return SystemAudioStatus.fromJson(Map<String, dynamic>.from(event));
+      }
+      return SystemAudioStatus(isActive: false);
+    });
+    return _statusStream;
+  }
+
+  /// Stream of system audio decibel (dB) readings.
+  ///
+  /// Returns a [Stream<DecibelData>] containing:
+  /// - `decibel`: double - decibel value (-120 to 0 dB)
+  /// - `timestamp`: double - Unix timestamp
+  ///
+  /// The stream is only available while recording is active.
+  ///
+  /// Example:
+  /// ```dart
+  /// await systemCapture.startCapture();
+  ///
+  /// systemCapture.decibelStream?.listen((data) {
+  ///   print('System audio level: ${data.decibel.toStringAsFixed(1)} dB');
+  ///   print('Timestamp: ${DateTime.fromMillisecondsSinceEpoch((data.timestamp * 1000).toInt())}');
+  /// });
+  /// ```
+  Stream<DecibelData>? get decibelStream {
+    if (!_isRecording) {
+      return null;
+    }
+    // Create decibel stream if not already created
+    _decibelStream ??= _decibelStreamChannel.receiveBroadcastStream().map((
+      dynamic event,
+    ) {
+      if (event is Map) {
+        return DecibelData.fromMap(Map<String, dynamic>.from(event));
+      }
+      return DecibelData(
+          decibel: -120.0,
+          timestamp: DateTime.now().millisecondsSinceEpoch / 1000.0);
+    });
+    return _decibelStream;
+  }
+
+  SystemAudioConfig _config = SystemAudioConfig();
+
+  /// Creates a new [SystemAudioCapture] instance.
+  ///
+  /// [config] is optional. If not provided, default configuration will be used
+  /// (sampleRate: 16000, channels: 1).
+  ///
+  /// Example:
+  /// ```dart
+  /// final capture = SystemAudioCapture(
+  ///   config: SystemAudioConfig(
+  ///     sampleRate: 44100,
+  ///     channels: 2,
+  ///   ),
+  /// );
+  /// ```
+  SystemAudioCapture({SystemAudioConfig? config}) {
+    _config = config ?? SystemAudioConfig();
+  }
+
+  /// Updates the audio capture configuration.
+  ///
+  /// This method allows you to change the configuration after the instance
+  /// has been created. The new configuration will be applied on the next
+  /// [startCapture] call.
+  ///
+  /// Example:
+  /// ```dart
+  /// final capture = SystemAudioCapture();
+  ///
+  /// // Update config before starting
+  /// capture.updateConfig(SystemAudioConfig(
+  ///   sampleRate: 48000,
+  ///   channels: 2,
+  /// ));
+  ///
+  /// await capture.startCapture();
+  /// ```
+  void updateConfig(SystemAudioConfig config) {
+    _config = config;
+  }
+
+  /// Starts capturing system audio.
+  ///
+  /// This method will request necessary permissions (screen recording on macOS)
+  /// and begin capturing audio from the system output.
+  ///
+  /// [config] is optional. If provided, it will update the current configuration
+  /// before starting capture.
+  ///
+  /// Throws an [Exception] if:
+  /// - Permissions are not granted
+  /// - Capture fails to start
+  ///
+  /// Example:
+  /// ```dart
+  /// final capture = SystemAudioCapture();
+  ///
+  /// try {
+  ///   await capture.startCapture(
+  ///     config: SystemAudioConfig(
+  ///       sampleRate: 44100,
+  ///       channels: 2,
+  ///     ),
+  ///   );
+  ///   print('System audio capture started');
+  /// } catch (e) {
+  ///   print('Failed to start: $e');
+  /// }
+  /// ```
+  Future<void> startCapture({SystemAudioConfig? config}) async {
+    if (_isRecording) {
+      return;
+    }
+
+    if (config != null) {
+      updateConfig(config);
+    }
+
+    try {
+      await requestPermissions();
+
+      final started = await _channel.invokeMethod<bool>(
+        _SystemAudioMethod.startCapture.name,
+        _config.toMap(),
+      );
+
+      if (started != true) {
+        throw Exception('Failed to start system audio capture');
+      }
+
+      // Listen to audio stream
+      _audioStream = _audioStreamChannel.receiveBroadcastStream().map((
+        dynamic event,
+      ) {
+        if (event is Uint8List) {
+          return event;
+        } else if (event is List<int>) {
+          return Uint8List.fromList(event);
+        }
+        throw Exception('Unexpected audio data type: ${event.runtimeType}');
+      });
+
+      // Status stream is created lazily via getter, no need to recreate here
+
+      _isRecording = true;
+    } catch (e) {
+      rethrow;
+    }
+  }
+
+  /// Stops capturing system audio.
+  ///
+  /// This method will stop the active capture and close all associated streams.
+  /// If capture is not active, this method does nothing.
+  ///
+  /// Throws an [Exception] if stopping fails.
+  ///
+  /// Example:
+  /// ```dart
+  /// await systemCapture.startCapture();
+  ///
+  /// // ... use audio stream ...
+  ///
+  /// await systemCapture.stopCapture();
+  /// print('System audio capture stopped');
+  /// ```
+  Future<void> stopCapture() async {
+    if (!_isRecording) return;
+
+    try {
+      final stopped = await _channel.invokeMethod<bool>(
+        _SystemAudioMethod.stopCapture.name,
+      );
+
+      if (stopped != true) {
+        throw Exception("Failed to stop system audio capture");
+      }
+
+      _isRecording = false;
+      _audioStream = null;
+      _statusStream = null;
+      _decibelStream = null;
+    } catch (e) {
+      rethrow;
+    }
+  }
+
+  /// Whether system audio capture is currently recording.
+  ///
+  /// Returns `true` if capture is active, `false` otherwise.
+  ///
+  /// Example:
+  /// ```dart
+  /// if (systemCapture.isRecording) {
+  ///   print('System audio is being captured');
+  /// } else {
+  ///   print('System audio capture is not active');
+  /// }
+  /// ```
+  @override
+  bool get isRecording => _isRecording;
+
+  /// Requests necessary permissions for system audio capture.
+  ///
+  /// On macOS, this requests screen recording permission which is required
+  /// to capture system audio.
+  ///
+  /// Returns `true` if permissions are granted.
+  ///
+  /// Throws an [Exception] if permissions are not granted.
+  ///
+  /// Example:
+  /// ```dart
+  /// try {
+  ///   final hasPermission = await systemCapture.requestPermissions();
+  ///   if (hasPermission) {
+  ///     await systemCapture.startCapture();
+  ///   }
+  /// } catch (e) {
+  ///   print('Permission denied: $e');
+  /// }
+  /// ```
+  Future<bool> requestPermissions() async {
+    final hasPermission = await _channel.invokeMethod<bool>(
+      _SystemAudioMethod.requestPermissions.name,
+    );
+    if (hasPermission != true) {
+      throw Exception('Screen recording permission not granted');
+    }
+    return true;
+  }
+}