neoagent 3.1.1-beta.9 → 3.2.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -2
- package/README.md +5 -1
- package/docs/architecture.md +1 -1
- package/docs/benchmarking.md +68 -87
- package/docs/capabilities.md +1 -1
- package/docs/clients-and-devices.md +3 -4
- package/docs/configuration.md +1 -1
- package/docs/devices.md +1 -2
- package/docs/health.md +24 -0
- package/docs/index.md +2 -2
- package/docs/integrations-architecture.md +6 -0
- package/docs/integrations.md +15 -1
- package/docs/models.md +1 -1
- package/docs/persistence.md +1 -1
- package/docs/why-neoagent.md +1 -1
- package/flutter_app/README.md +0 -5
- package/flutter_app/android/app/src/main/AndroidManifest.xml +0 -14
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +0 -98
- package/flutter_app/lib/main.dart +0 -6
- package/flutter_app/lib/main_app_shell.dart +0 -2
- package/flutter_app/lib/main_controller.dart +2 -867
- package/flutter_app/lib/main_devices.dart +0 -73
- package/flutter_app/lib/main_integrations.dart +339 -187
- package/flutter_app/lib/main_launcher.dart +2 -212
- package/flutter_app/lib/main_models.dart +0 -259
- package/flutter_app/lib/main_navigation.dart +2 -14
- package/flutter_app/lib/main_operations.dart +875 -383
- package/flutter_app/lib/main_runtime.dart +9 -208
- package/flutter_app/lib/main_settings.dart +17 -199
- package/flutter_app/lib/main_shared.dart +20 -591
- package/flutter_app/lib/main_timeline.dart +26 -122
- package/flutter_app/lib/main_voice_assistant.dart +0 -5
- package/flutter_app/lib/src/backend_client.dart +0 -102
- package/flutter_app/lib/src/desktop_companion_io.dart +1 -65
- package/flutter_app/lib/src/desktop_companion_stub.dart +0 -12
- package/flutter_app/linux/flutter/generated_plugin_registrant.cc +0 -4
- package/flutter_app/linux/flutter/generated_plugins.cmake +0 -1
- package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +0 -2
- package/flutter_app/macos/Podfile.lock +56 -5
- package/flutter_app/pubspec.lock +0 -7
- package/flutter_app/pubspec.yaml +0 -2
- package/flutter_app/web/index.html +174 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.cc +0 -3
- package/flutter_app/windows/flutter/generated_plugins.cmake +0 -1
- package/landing/index.html +0 -2
- package/lib/manager.js +1 -1
- package/lib/schema_migrations.js +75 -0
- package/package.json +1 -4
- package/server/db/database.js +1 -133
- package/server/http/middleware.js +2 -16
- package/server/http/routes.js +0 -2
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/NOTICES +0 -25
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +2 -2
- package/server/public/index.html +174 -0
- package/server/public/main.dart.js +92036 -91145
- package/server/routes/auth.js +11 -2
- package/server/routes/settings.js +0 -5
- package/server/routes/voice_assistant.js +1 -128
- package/server/services/account/erasure.js +0 -3
- package/server/services/ai/deliverables/artifact_helpers.js +1 -1
- package/server/services/ai/loop/conversation_loop.js +6 -2
- package/server/services/ai/loop/tool_dispatch.js +0 -3
- package/server/services/ai/messagingFallback.js +1 -1
- package/server/services/ai/settings.js +0 -6
- package/server/services/ai/toolEvidence.js +1 -4
- package/server/services/ai/tools.js +0 -234
- package/server/services/desktop/auth.js +0 -3
- package/server/services/desktop/registry.js +2 -50
- package/server/services/integrations/neoarchive/provider.js +616 -0
- package/server/services/integrations/neorecall/client.js +140 -0
- package/server/services/integrations/neorecall/constants.js +87 -0
- package/server/services/integrations/neorecall/provider.js +163 -0
- package/server/services/integrations/registry.js +4 -2
- package/server/services/manager.js +0 -22
- package/server/services/memory/manager.js +44 -19
- package/server/services/memory/retrieval_reasoning.js +36 -14
- package/server/services/messaging/formatting_guides.js +1 -0
- package/server/services/messaging/whatsapp.js +18 -0
- package/server/services/security/tool_categories.js +0 -5
- package/server/services/social_video/service.js +2 -20
- package/server/services/tasks/adapters/index.js +1 -1
- package/server/services/tasks/integration_runtime.js +3 -67
- package/server/services/tasks/runtime.js +1 -6
- package/server/services/tasks/task_repository.js +6 -13
- package/server/services/timeline/service.js +0 -224
- package/server/services/voice/message.js +0 -24
- package/server/services/voice/providers.js +42 -1
- package/server/services/voice/runtimeManager.js +9 -5
- package/server/services/voice/turnRunner.js +0 -22
- package/server/services/wearable/service.js +0 -5
- package/docs/recordings-and-health.md +0 -41
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +0 -21
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +0 -586
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +0 -78
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +0 -104
- package/flutter_app/lib/main_recordings.dart +0 -920
- package/flutter_app/lib/src/desktop_ocr_bridge.dart +0 -2
- package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +0 -125
- package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +0 -30
- package/flutter_app/lib/src/desktop_passive_history.dart +0 -332
- package/flutter_app/lib/src/recording_bridge.dart +0 -232
- package/flutter_app/lib/src/recording_bridge_io.dart +0 -1027
- package/flutter_app/lib/src/recording_bridge_stub.dart +0 -120
- package/flutter_app/lib/src/recording_bridge_web.dart +0 -702
- package/flutter_app/lib/src/recording_chunk_queue.dart +0 -149
- package/flutter_app/lib/src/recording_chunk_queue_io.dart +0 -182
- package/flutter_app/lib/src/recording_payloads.dart +0 -95
- package/flutter_app/third_party/desktop_audio_capture/LICENSE +0 -21
- package/flutter_app/third_party/desktop_audio_capture/README.md +0 -262
- package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +0 -65
- package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +0 -153
- package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +0 -110
- package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +0 -461
- package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +0 -91
- package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +0 -106
- package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +0 -219
- package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +0 -336
- package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +0 -101
- package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +0 -692
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +0 -35
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +0 -36
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +0 -32
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +0 -32
- package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +0 -878
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +0 -27
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +0 -1172
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +0 -655
- package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +0 -12
- package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +0 -30
- package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +0 -87
- package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +0 -105
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +0 -80
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +0 -31
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +0 -12
- package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +0 -23
- package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +0 -25
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +0 -1117
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +0 -115
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +0 -777
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +0 -87
- package/server/routes/recordings.js +0 -335
- package/server/routes/screenHistory.js +0 -190
- package/server/services/ai/recordingInsights.js +0 -105
- package/server/services/integrations/neomail/provider.js +0 -993
- package/server/services/recordings/deepgram.js +0 -58
- package/server/services/recordings/manager.js +0 -1273
- package/server/services/tasks/adapters/neomail_email_received.js +0 -46
- package/server/services/voice/screenshotContext.js +0 -73
- package/server/services/voice/turnCoordinator.js +0 -51
|
@@ -1,219 +0,0 @@
|
|
|
1
|
-
/// Enum representing the type of input device (microphone).
|
|
2
|
-
///
|
|
3
|
-
/// This enum categorizes input devices into three types: built-in, Bluetooth,
|
|
4
|
-
/// or external (USB, etc.).
|
|
5
|
-
///
|
|
6
|
-
/// Example:
|
|
7
|
-
/// ```dart
|
|
8
|
-
/// final device = InputDevice(
|
|
9
|
-
/// id: 'device1',
|
|
10
|
-
/// name: 'Built-in Microphone',
|
|
11
|
-
/// type: InputDeviceType.builtIn,
|
|
12
|
-
/// channelCount: 1,
|
|
13
|
-
/// isDefault: true,
|
|
14
|
-
/// );
|
|
15
|
-
///
|
|
16
|
-
/// // Convert to/from string
|
|
17
|
-
/// final typeString = device.type.toString(); // 'built-in'
|
|
18
|
-
/// final type = InputDeviceType.fromString('bluetooth'); // InputDeviceType.bluetooth
|
|
19
|
-
/// ```
|
|
20
|
-
enum InputDeviceType {
|
|
21
|
-
/// Built-in device (e.g., laptop microphone)
|
|
22
|
-
builtIn,
|
|
23
|
-
|
|
24
|
-
/// Bluetooth device (wireless microphone)
|
|
25
|
-
bluetooth,
|
|
26
|
-
|
|
27
|
-
/// External device (USB microphone, etc.)
|
|
28
|
-
external;
|
|
29
|
-
|
|
30
|
-
/// Creates an [InputDeviceType] from a string.
|
|
31
|
-
///
|
|
32
|
-
/// Accepts: 'built-in', 'bluetooth', 'external' (case-insensitive).
|
|
33
|
-
/// Returns [InputDeviceType.external] for unknown values.
|
|
34
|
-
///
|
|
35
|
-
/// Example:
|
|
36
|
-
/// ```dart
|
|
37
|
-
/// final type1 = InputDeviceType.fromString('built-in'); // InputDeviceType.builtIn
|
|
38
|
-
/// final type2 = InputDeviceType.fromString('BLUETOOTH'); // InputDeviceType.bluetooth
|
|
39
|
-
/// final type3 = InputDeviceType.fromString('unknown'); // InputDeviceType.external (default)
|
|
40
|
-
/// ```
|
|
41
|
-
static InputDeviceType fromString(String type) {
|
|
42
|
-
switch (type.toLowerCase()) {
|
|
43
|
-
case 'built-in':
|
|
44
|
-
return InputDeviceType.builtIn;
|
|
45
|
-
case 'bluetooth':
|
|
46
|
-
return InputDeviceType.bluetooth;
|
|
47
|
-
case 'external':
|
|
48
|
-
return InputDeviceType.external;
|
|
49
|
-
default:
|
|
50
|
-
return InputDeviceType.external;
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/// Converts this [InputDeviceType] to a string representation.
|
|
55
|
-
///
|
|
56
|
-
/// Returns: 'built-in', 'bluetooth', or 'external'.
|
|
57
|
-
///
|
|
58
|
-
/// Example:
|
|
59
|
-
/// ```dart
|
|
60
|
-
/// InputDeviceType.builtIn.toString(); // 'built-in'
|
|
61
|
-
/// InputDeviceType.bluetooth.toString(); // 'bluetooth'
|
|
62
|
-
/// InputDeviceType.external.toString(); // 'external'
|
|
63
|
-
/// ```
|
|
64
|
-
@override
|
|
65
|
-
String toString() {
|
|
66
|
-
switch (this) {
|
|
67
|
-
case InputDeviceType.builtIn:
|
|
68
|
-
return 'built-in';
|
|
69
|
-
case InputDeviceType.bluetooth:
|
|
70
|
-
return 'bluetooth';
|
|
71
|
-
case InputDeviceType.external:
|
|
72
|
-
return 'external';
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
/// Class representing information about an input device (microphone).
|
|
78
|
-
///
|
|
79
|
-
/// This class contains all relevant information about a microphone device,
|
|
80
|
-
/// including its ID, name, type, channel count, and whether it's the default device.
|
|
81
|
-
///
|
|
82
|
-
/// Example:
|
|
83
|
-
/// ```dart
|
|
84
|
-
/// // Get available devices
|
|
85
|
-
/// final devices = await micCapture.getAvailableInputDevices();
|
|
86
|
-
///
|
|
87
|
-
/// // Find a specific device
|
|
88
|
-
/// final usbMic = devices.firstWhere(
|
|
89
|
-
/// (device) => device.name.contains('USB'),
|
|
90
|
-
/// );
|
|
91
|
-
///
|
|
92
|
-
/// print('Device: ${usbMic.name}');
|
|
93
|
-
/// print('Type: ${usbMic.type}');
|
|
94
|
-
/// print('Channels: ${usbMic.channelCount}');
|
|
95
|
-
/// print('Default: ${usbMic.isDefault}');
|
|
96
|
-
///
|
|
97
|
-
/// // Convert to/from map
|
|
98
|
-
/// final map = usbMic.toMap();
|
|
99
|
-
/// final restored = InputDevice.fromMap(map);
|
|
100
|
-
/// ```
|
|
101
|
-
class InputDevice {
|
|
102
|
-
/// Unique identifier of the device.
|
|
103
|
-
///
|
|
104
|
-
/// This ID can be used to identify and select a specific device.
|
|
105
|
-
final String id;
|
|
106
|
-
|
|
107
|
-
/// Human-readable name of the device.
|
|
108
|
-
///
|
|
109
|
-
/// Examples: "Built-in Microphone", "USB Microphone", "AirPods Pro"
|
|
110
|
-
final String name;
|
|
111
|
-
|
|
112
|
-
/// Type of the device (built-in, Bluetooth, or external).
|
|
113
|
-
final InputDeviceType type;
|
|
114
|
-
|
|
115
|
-
/// Number of audio channels supported by the device.
|
|
116
|
-
///
|
|
117
|
-
/// Typically 1 for mono, 2 for stereo.
|
|
118
|
-
final int channelCount;
|
|
119
|
-
|
|
120
|
-
/// Whether this device is the system default input device.
|
|
121
|
-
final bool isDefault;
|
|
122
|
-
|
|
123
|
-
/// Creates a new [InputDevice] instance.
|
|
124
|
-
///
|
|
125
|
-
/// All parameters are required.
|
|
126
|
-
///
|
|
127
|
-
/// Example:
|
|
128
|
-
/// ```dart
|
|
129
|
-
/// final device = InputDevice(
|
|
130
|
-
/// id: 'device-123',
|
|
131
|
-
/// name: 'Built-in Microphone',
|
|
132
|
-
/// type: InputDeviceType.builtIn,
|
|
133
|
-
/// channelCount: 1,
|
|
134
|
-
/// isDefault: true,
|
|
135
|
-
/// );
|
|
136
|
-
/// ```
|
|
137
|
-
const InputDevice({
|
|
138
|
-
required this.id,
|
|
139
|
-
required this.name,
|
|
140
|
-
required this.type,
|
|
141
|
-
required this.channelCount,
|
|
142
|
-
required this.isDefault,
|
|
143
|
-
});
|
|
144
|
-
|
|
145
|
-
/// Creates an [InputDevice] instance from a map.
|
|
146
|
-
///
|
|
147
|
-
/// The map should contain:
|
|
148
|
-
/// - `id`: String (defaults to empty string if missing)
|
|
149
|
-
/// - `name`: String (defaults to empty string if missing)
|
|
150
|
-
/// - `type`: String (converted via [InputDeviceType.fromString], defaults to external)
|
|
151
|
-
/// - `channelCount`: int (defaults to 0 if missing)
|
|
152
|
-
/// - `isDefault`: bool (defaults to false if missing)
|
|
153
|
-
///
|
|
154
|
-
/// Example:
|
|
155
|
-
/// ```dart
|
|
156
|
-
/// final map = {
|
|
157
|
-
/// 'id': 'device-123',
|
|
158
|
-
/// 'name': 'USB Microphone',
|
|
159
|
-
/// 'type': 'external',
|
|
160
|
-
/// 'channelCount': 2,
|
|
161
|
-
/// 'isDefault': false,
|
|
162
|
-
/// };
|
|
163
|
-
/// final device = InputDevice.fromMap(map);
|
|
164
|
-
/// ```
|
|
165
|
-
factory InputDevice.fromMap(Map<String, dynamic> map) {
|
|
166
|
-
return InputDevice(
|
|
167
|
-
id: map['id'] as String? ?? '',
|
|
168
|
-
name: map['name'] as String? ?? '',
|
|
169
|
-
type: InputDeviceType.fromString(map['type'] as String? ?? 'external'),
|
|
170
|
-
channelCount: map['channelCount'] as int? ?? 0,
|
|
171
|
-
isDefault: map['isDefault'] as bool? ?? false,
|
|
172
|
-
);
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
/// Converts this [InputDevice] instance to a map.
|
|
176
|
-
///
|
|
177
|
-
/// Returns a map containing all device information:
|
|
178
|
-
/// - `id`: String
|
|
179
|
-
/// - `name`: String
|
|
180
|
-
/// - `type`: String (from [InputDeviceType.toString])
|
|
181
|
-
/// - `channelCount`: int
|
|
182
|
-
/// - `isDefault`: bool
|
|
183
|
-
///
|
|
184
|
-
/// Example:
|
|
185
|
-
/// ```dart
|
|
186
|
-
/// final device = InputDevice(
|
|
187
|
-
/// id: 'device-123',
|
|
188
|
-
/// name: 'USB Microphone',
|
|
189
|
-
/// type: InputDeviceType.external,
|
|
190
|
-
/// channelCount: 2,
|
|
191
|
-
/// isDefault: false,
|
|
192
|
-
/// );
|
|
193
|
-
/// final map = device.toMap();
|
|
194
|
-
/// ```
|
|
195
|
-
Map<String, dynamic> toMap() {
|
|
196
|
-
return {
|
|
197
|
-
'id': id,
|
|
198
|
-
'name': name,
|
|
199
|
-
'type': type.toString(),
|
|
200
|
-
'channelCount': channelCount,
|
|
201
|
-
'isDefault': isDefault,
|
|
202
|
-
};
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
@override
|
|
206
|
-
String toString() {
|
|
207
|
-
return 'InputDevice(id: $id, name: $name, type: ${type.toString()}, '
|
|
208
|
-
'channelCount: $channelCount, isDefault: $isDefault)';
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
@override
|
|
212
|
-
bool operator ==(Object other) {
|
|
213
|
-
if (identical(this, other)) return true;
|
|
214
|
-
return other is InputDevice && other.id == id;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
@override
|
|
218
|
-
int get hashCode => id.hashCode;
|
|
219
|
-
}
|
|
@@ -1,336 +0,0 @@
|
|
|
1
|
-
import 'dart:async';
|
|
2
|
-
|
|
3
|
-
import 'package:desktop_audio_capture/audio_capture.dart';
|
|
4
|
-
import 'package:flutter/services.dart';
|
|
5
|
-
|
|
6
|
-
export 'package:desktop_audio_capture/config/system_adudio_config.dart';
|
|
7
|
-
|
|
8
|
-
enum _SystemAudioMethod {
|
|
9
|
-
startCapture,
|
|
10
|
-
stopCapture,
|
|
11
|
-
requestPermissions,
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
/// Class for capturing system audio (audio output from the device).
|
|
15
|
-
///
|
|
16
|
-
/// This class allows you to capture audio that is being played by the system,
|
|
17
|
-
/// such as music, videos, or other applications. It requires screen recording
|
|
18
|
-
/// permissions on macOS.
|
|
19
|
-
///
|
|
20
|
-
/// Example:
|
|
21
|
-
/// ```dart
|
|
22
|
-
/// final systemCapture = SystemAudioCapture(
|
|
23
|
-
/// config: SystemAudioConfig(
|
|
24
|
-
/// sampleRate: 44100,
|
|
25
|
-
/// channels: 2,
|
|
26
|
-
/// ),
|
|
27
|
-
/// );
|
|
28
|
-
///
|
|
29
|
-
/// await systemCapture.startCapture();
|
|
30
|
-
///
|
|
31
|
-
/// // Listen to audio stream
|
|
32
|
-
/// systemCapture.audioStream?.listen((audioData) {
|
|
33
|
-
/// // Process audio bytes
|
|
34
|
-
/// print('Received ${audioData.length} bytes');
|
|
35
|
-
/// });
|
|
36
|
-
///
|
|
37
|
-
/// // Listen to decibel readings
|
|
38
|
-
/// systemCapture.decibelStream?.listen((data) {
|
|
39
|
-
/// print('Decibel: ${data.decibel} dB');
|
|
40
|
-
/// });
|
|
41
|
-
///
|
|
42
|
-
/// // Stop when done
|
|
43
|
-
/// await systemCapture.stopCapture();
|
|
44
|
-
/// ```
|
|
45
|
-
class SystemAudioCapture extends AudioCapture {
|
|
46
|
-
static const MethodChannel _channel = MethodChannel(
|
|
47
|
-
'com.system_audio_transcriber/audio_capture',
|
|
48
|
-
);
|
|
49
|
-
static const EventChannel _audioStreamChannel = EventChannel(
|
|
50
|
-
'com.system_audio_transcriber/audio_stream',
|
|
51
|
-
);
|
|
52
|
-
static const EventChannel _statusStreamChannel = EventChannel(
|
|
53
|
-
'com.system_audio_transcriber/audio_status',
|
|
54
|
-
);
|
|
55
|
-
static const EventChannel _decibelStreamChannel = EventChannel(
|
|
56
|
-
'com.system_audio_transcriber/audio_decibel',
|
|
57
|
-
);
|
|
58
|
-
|
|
59
|
-
Stream<Uint8List>? _audioStream;
|
|
60
|
-
Stream<SystemAudioStatus>? _statusStream;
|
|
61
|
-
Stream<DecibelData>? _decibelStream;
|
|
62
|
-
bool _isRecording = false;
|
|
63
|
-
|
|
64
|
-
/// Stream of raw audio data bytes from system audio capture.
|
|
65
|
-
///
|
|
66
|
-
/// Returns a [Stream<Uint8List>] containing the captured audio data.
|
|
67
|
-
/// The stream is only available after [startCapture] has been called.
|
|
68
|
-
///
|
|
69
|
-
/// Example:
|
|
70
|
-
/// ```dart
|
|
71
|
-
/// await systemCapture.startCapture();
|
|
72
|
-
///
|
|
73
|
-
/// systemCapture.audioStream?.listen((audioData) {
|
|
74
|
-
/// // Process audio bytes
|
|
75
|
-
/// final audioBuffer = audioData.buffer.asUint8List();
|
|
76
|
-
/// // Use audio buffer for processing, saving, etc.
|
|
77
|
-
/// });
|
|
78
|
-
/// ```
|
|
79
|
-
Stream<Uint8List>? get audioStream => _audioStream;
|
|
80
|
-
|
|
81
|
-
/// Stream of system audio capture status updates.
|
|
82
|
-
///
|
|
83
|
-
/// Returns a [Stream<SystemAudioStatus>] containing status information:
|
|
84
|
-
/// - [SystemAudioStatus.isActive]: bool - whether system audio capture is currently active
|
|
85
|
-
///
|
|
86
|
-
/// Example:
|
|
87
|
-
/// ```dart
|
|
88
|
-
/// systemCapture.statusStream?.listen((status) {
|
|
89
|
-
/// if (status.isActive) {
|
|
90
|
-
/// print('System audio capture is active');
|
|
91
|
-
/// } else {
|
|
92
|
-
/// print('System audio capture is inactive');
|
|
93
|
-
/// }
|
|
94
|
-
/// });
|
|
95
|
-
/// ```
|
|
96
|
-
Stream<SystemAudioStatus>? get statusStream {
|
|
97
|
-
// Create status stream if not already created
|
|
98
|
-
_statusStream ??= _statusStreamChannel.receiveBroadcastStream().map((
|
|
99
|
-
dynamic event,
|
|
100
|
-
) {
|
|
101
|
-
if (event is Map) {
|
|
102
|
-
return SystemAudioStatus.fromJson(Map<String, dynamic>.from(event));
|
|
103
|
-
}
|
|
104
|
-
return SystemAudioStatus(isActive: false);
|
|
105
|
-
});
|
|
106
|
-
return _statusStream;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/// Stream of system audio decibel (dB) readings.
|
|
110
|
-
///
|
|
111
|
-
/// Returns a [Stream<DecibelData>] containing:
|
|
112
|
-
/// - `decibel`: double - decibel value (-120 to 0 dB)
|
|
113
|
-
/// - `timestamp`: double - Unix timestamp
|
|
114
|
-
///
|
|
115
|
-
/// The stream is only available while recording is active.
|
|
116
|
-
///
|
|
117
|
-
/// Example:
|
|
118
|
-
/// ```dart
|
|
119
|
-
/// await systemCapture.startCapture();
|
|
120
|
-
///
|
|
121
|
-
/// systemCapture.decibelStream?.listen((data) {
|
|
122
|
-
/// print('System audio level: ${data.decibel.toStringAsFixed(1)} dB');
|
|
123
|
-
/// print('Timestamp: ${DateTime.fromMillisecondsSinceEpoch((data.timestamp * 1000).toInt())}');
|
|
124
|
-
/// });
|
|
125
|
-
/// ```
|
|
126
|
-
Stream<DecibelData>? get decibelStream {
|
|
127
|
-
if (!_isRecording) {
|
|
128
|
-
return null;
|
|
129
|
-
}
|
|
130
|
-
// Create decibel stream if not already created
|
|
131
|
-
_decibelStream ??= _decibelStreamChannel.receiveBroadcastStream().map((
|
|
132
|
-
dynamic event,
|
|
133
|
-
) {
|
|
134
|
-
if (event is Map) {
|
|
135
|
-
return DecibelData.fromMap(Map<String, dynamic>.from(event));
|
|
136
|
-
}
|
|
137
|
-
return DecibelData(
|
|
138
|
-
decibel: -120.0,
|
|
139
|
-
timestamp: DateTime.now().millisecondsSinceEpoch / 1000.0);
|
|
140
|
-
});
|
|
141
|
-
return _decibelStream;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
SystemAudioConfig _config = SystemAudioConfig();
|
|
145
|
-
|
|
146
|
-
/// Creates a new [SystemAudioCapture] instance.
|
|
147
|
-
///
|
|
148
|
-
/// [config] is optional. If not provided, default configuration will be used
|
|
149
|
-
/// (sampleRate: 16000, channels: 1).
|
|
150
|
-
///
|
|
151
|
-
/// Example:
|
|
152
|
-
/// ```dart
|
|
153
|
-
/// final capture = SystemAudioCapture(
|
|
154
|
-
/// config: SystemAudioConfig(
|
|
155
|
-
/// sampleRate: 44100,
|
|
156
|
-
/// channels: 2,
|
|
157
|
-
/// ),
|
|
158
|
-
/// );
|
|
159
|
-
/// ```
|
|
160
|
-
SystemAudioCapture({SystemAudioConfig? config}) {
|
|
161
|
-
_config = config ?? SystemAudioConfig();
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
/// Updates the audio capture configuration.
|
|
165
|
-
///
|
|
166
|
-
/// This method allows you to change the configuration after the instance
|
|
167
|
-
/// has been created. The new configuration will be applied on the next
|
|
168
|
-
/// [startCapture] call.
|
|
169
|
-
///
|
|
170
|
-
/// Example:
|
|
171
|
-
/// ```dart
|
|
172
|
-
/// final capture = SystemAudioCapture();
|
|
173
|
-
///
|
|
174
|
-
/// // Update config before starting
|
|
175
|
-
/// capture.updateConfig(SystemAudioConfig(
|
|
176
|
-
/// sampleRate: 48000,
|
|
177
|
-
/// channels: 2,
|
|
178
|
-
/// ));
|
|
179
|
-
///
|
|
180
|
-
/// await capture.startCapture();
|
|
181
|
-
/// ```
|
|
182
|
-
void updateConfig(SystemAudioConfig config) {
|
|
183
|
-
_config = config;
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
/// Starts capturing system audio.
|
|
187
|
-
///
|
|
188
|
-
/// This method will request necessary permissions (screen recording on macOS)
|
|
189
|
-
/// and begin capturing audio from the system output.
|
|
190
|
-
///
|
|
191
|
-
/// [config] is optional. If provided, it will update the current configuration
|
|
192
|
-
/// before starting capture.
|
|
193
|
-
///
|
|
194
|
-
/// Throws an [Exception] if:
|
|
195
|
-
/// - Permissions are not granted
|
|
196
|
-
/// - Capture fails to start
|
|
197
|
-
///
|
|
198
|
-
/// Example:
|
|
199
|
-
/// ```dart
|
|
200
|
-
/// final capture = SystemAudioCapture();
|
|
201
|
-
///
|
|
202
|
-
/// try {
|
|
203
|
-
/// await capture.startCapture(
|
|
204
|
-
/// config: SystemAudioConfig(
|
|
205
|
-
/// sampleRate: 44100,
|
|
206
|
-
/// channels: 2,
|
|
207
|
-
/// ),
|
|
208
|
-
/// );
|
|
209
|
-
/// print('System audio capture started');
|
|
210
|
-
/// } catch (e) {
|
|
211
|
-
/// print('Failed to start: $e');
|
|
212
|
-
/// }
|
|
213
|
-
/// ```
|
|
214
|
-
Future<void> startCapture({SystemAudioConfig? config}) async {
|
|
215
|
-
if (_isRecording) {
|
|
216
|
-
return;
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
if (config != null) {
|
|
220
|
-
updateConfig(config);
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
try {
|
|
224
|
-
await requestPermissions();
|
|
225
|
-
|
|
226
|
-
final started = await _channel.invokeMethod<bool>(
|
|
227
|
-
_SystemAudioMethod.startCapture.name,
|
|
228
|
-
_config.toMap(),
|
|
229
|
-
);
|
|
230
|
-
|
|
231
|
-
if (started != true) {
|
|
232
|
-
throw Exception('Failed to start system audio capture');
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
// Listen to audio stream
|
|
236
|
-
_audioStream = _audioStreamChannel.receiveBroadcastStream().map((
|
|
237
|
-
dynamic event,
|
|
238
|
-
) {
|
|
239
|
-
if (event is Uint8List) {
|
|
240
|
-
return event;
|
|
241
|
-
} else if (event is List<int>) {
|
|
242
|
-
return Uint8List.fromList(event);
|
|
243
|
-
}
|
|
244
|
-
throw Exception('Unexpected audio data type: ${event.runtimeType}');
|
|
245
|
-
});
|
|
246
|
-
|
|
247
|
-
// Status stream is created lazily via getter, no need to recreate here
|
|
248
|
-
|
|
249
|
-
_isRecording = true;
|
|
250
|
-
} catch (e) {
|
|
251
|
-
rethrow;
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
/// Stops capturing system audio.
|
|
256
|
-
///
|
|
257
|
-
/// This method will stop the active capture and close all associated streams.
|
|
258
|
-
/// If capture is not active, this method does nothing.
|
|
259
|
-
///
|
|
260
|
-
/// Throws an [Exception] if stopping fails.
|
|
261
|
-
///
|
|
262
|
-
/// Example:
|
|
263
|
-
/// ```dart
|
|
264
|
-
/// await systemCapture.startCapture();
|
|
265
|
-
///
|
|
266
|
-
/// // ... use audio stream ...
|
|
267
|
-
///
|
|
268
|
-
/// await systemCapture.stopCapture();
|
|
269
|
-
/// print('System audio capture stopped');
|
|
270
|
-
/// ```
|
|
271
|
-
Future<void> stopCapture() async {
|
|
272
|
-
if (!_isRecording) return;
|
|
273
|
-
|
|
274
|
-
try {
|
|
275
|
-
final stopped = await _channel.invokeMethod<bool>(
|
|
276
|
-
_SystemAudioMethod.stopCapture.name,
|
|
277
|
-
);
|
|
278
|
-
|
|
279
|
-
if (stopped != true) {
|
|
280
|
-
throw Exception("Failed to stop system audio capture");
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
_isRecording = false;
|
|
284
|
-
_audioStream = null;
|
|
285
|
-
_statusStream = null;
|
|
286
|
-
_decibelStream = null;
|
|
287
|
-
} catch (e) {
|
|
288
|
-
rethrow;
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
/// Whether system audio capture is currently recording.
|
|
293
|
-
///
|
|
294
|
-
/// Returns `true` if capture is active, `false` otherwise.
|
|
295
|
-
///
|
|
296
|
-
/// Example:
|
|
297
|
-
/// ```dart
|
|
298
|
-
/// if (systemCapture.isRecording) {
|
|
299
|
-
/// print('System audio is being captured');
|
|
300
|
-
/// } else {
|
|
301
|
-
/// print('System audio capture is not active');
|
|
302
|
-
/// }
|
|
303
|
-
/// ```
|
|
304
|
-
@override
|
|
305
|
-
bool get isRecording => _isRecording;
|
|
306
|
-
|
|
307
|
-
/// Requests necessary permissions for system audio capture.
|
|
308
|
-
///
|
|
309
|
-
/// On macOS, this requests screen recording permission which is required
|
|
310
|
-
/// to capture system audio.
|
|
311
|
-
///
|
|
312
|
-
/// Returns `true` if permissions are granted.
|
|
313
|
-
///
|
|
314
|
-
/// Throws an [Exception] if permissions are not granted.
|
|
315
|
-
///
|
|
316
|
-
/// Example:
|
|
317
|
-
/// ```dart
|
|
318
|
-
/// try {
|
|
319
|
-
/// final hasPermission = await systemCapture.requestPermissions();
|
|
320
|
-
/// if (hasPermission) {
|
|
321
|
-
/// await systemCapture.startCapture();
|
|
322
|
-
/// }
|
|
323
|
-
/// } catch (e) {
|
|
324
|
-
/// print('Permission denied: $e');
|
|
325
|
-
/// }
|
|
326
|
-
/// ```
|
|
327
|
-
Future<bool> requestPermissions() async {
|
|
328
|
-
final hasPermission = await _channel.invokeMethod<bool>(
|
|
329
|
-
_SystemAudioMethod.requestPermissions.name,
|
|
330
|
-
);
|
|
331
|
-
if (hasPermission != true) {
|
|
332
|
-
throw Exception('Screen recording permission not granted');
|
|
333
|
-
}
|
|
334
|
-
return true;
|
|
335
|
-
}
|
|
336
|
-
}
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
# The Flutter tooling requires that developers have CMake 3.10 or later
|
|
2
|
-
# installed. You should not increase this version, as doing so will cause
|
|
3
|
-
# the plugin to fail to compile for some customers of the plugin.
|
|
4
|
-
cmake_minimum_required(VERSION 3.10)
|
|
5
|
-
|
|
6
|
-
# Project-level configuration.
|
|
7
|
-
set(PROJECT_NAME "desktop_audio_capture")
|
|
8
|
-
project(${PROJECT_NAME} LANGUAGES CXX)
|
|
9
|
-
|
|
10
|
-
# This value is used when generating builds using this plugin, so it must
|
|
11
|
-
# not be changed.
|
|
12
|
-
set(PLUGIN_NAME "desktop_audio_capture_plugin")
|
|
13
|
-
|
|
14
|
-
# System-level dependencies.
|
|
15
|
-
find_package(PkgConfig REQUIRED)
|
|
16
|
-
pkg_check_modules(PULSEAUDIO REQUIRED IMPORTED_TARGET libpulse-simple)
|
|
17
|
-
|
|
18
|
-
# Any new source files that you add to the plugin should be added here.
|
|
19
|
-
list(APPEND PLUGIN_SOURCES
|
|
20
|
-
"audio_capture_plugin.cc"
|
|
21
|
-
"mic_capture_plugin.cc"
|
|
22
|
-
)
|
|
23
|
-
|
|
24
|
-
# Define the plugin library target. Its name must not be changed (see comment
|
|
25
|
-
# on PLUGIN_NAME above).
|
|
26
|
-
add_library(${PLUGIN_NAME} SHARED
|
|
27
|
-
${PLUGIN_SOURCES}
|
|
28
|
-
)
|
|
29
|
-
|
|
30
|
-
# Apply a standard set of build settings that are configured in the
|
|
31
|
-
# application-level CMakeLists.txt. This can be removed for plugins that want
|
|
32
|
-
# full control over build settings.
|
|
33
|
-
apply_standard_settings(${PLUGIN_NAME})
|
|
34
|
-
|
|
35
|
-
# Symbols are hidden by default to reduce the chance of accidental conflicts
|
|
36
|
-
# between plugins. This should not be removed; any symbols that should be
|
|
37
|
-
# exported should be explicitly exported with the FLUTTER_PLUGIN_EXPORT macro.
|
|
38
|
-
set_target_properties(${PLUGIN_NAME} PROPERTIES
|
|
39
|
-
CXX_VISIBILITY_PRESET hidden)
|
|
40
|
-
target_compile_definitions(${PLUGIN_NAME} PRIVATE FLUTTER_PLUGIN_IMPL)
|
|
41
|
-
|
|
42
|
-
# Source include directories and library dependencies. Add any plugin-specific
|
|
43
|
-
# dependencies here.
|
|
44
|
-
target_include_directories(${PLUGIN_NAME} INTERFACE
|
|
45
|
-
"${CMAKE_CURRENT_SOURCE_DIR}/include")
|
|
46
|
-
target_link_libraries(${PLUGIN_NAME} PRIVATE flutter)
|
|
47
|
-
target_link_libraries(${PLUGIN_NAME} PRIVATE PkgConfig::GTK)
|
|
48
|
-
target_link_libraries(${PLUGIN_NAME} PRIVATE PkgConfig::PULSEAUDIO)
|
|
49
|
-
|
|
50
|
-
# List of absolute paths to libraries that should be bundled with the plugin.
|
|
51
|
-
# This list could contain prebuilt libraries, or libraries created by an
|
|
52
|
-
# external build triggered from this build file.
|
|
53
|
-
set(desktop_audio_capture_bundled_libraries
|
|
54
|
-
""
|
|
55
|
-
PARENT_SCOPE
|
|
56
|
-
)
|
|
57
|
-
|
|
58
|
-
# === Tests ===
|
|
59
|
-
# These unit tests can be run from a terminal after building the example.
|
|
60
|
-
|
|
61
|
-
# Only enable test builds when building the example (which sets this variable)
|
|
62
|
-
# so that plugin clients aren't building the tests.
|
|
63
|
-
if (${include_${PROJECT_NAME}_tests})
|
|
64
|
-
if(${CMAKE_VERSION} VERSION_LESS "3.11.0")
|
|
65
|
-
message("Unit tests require CMake 3.11.0 or later")
|
|
66
|
-
else()
|
|
67
|
-
set(TEST_RUNNER "${PROJECT_NAME}_test")
|
|
68
|
-
enable_testing()
|
|
69
|
-
|
|
70
|
-
# Add the Google Test dependency.
|
|
71
|
-
include(FetchContent)
|
|
72
|
-
FetchContent_Declare(
|
|
73
|
-
googletest
|
|
74
|
-
URL https://github.com/google/googletest/archive/release-1.11.0.zip
|
|
75
|
-
)
|
|
76
|
-
# Prevent overriding the parent project's compiler/linker settings
|
|
77
|
-
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
|
|
78
|
-
# Disable install commands for gtest so it doesn't end up in the bundle.
|
|
79
|
-
set(INSTALL_GTEST OFF CACHE BOOL "Disable installation of googletest" FORCE)
|
|
80
|
-
|
|
81
|
-
FetchContent_MakeAvailable(googletest)
|
|
82
|
-
|
|
83
|
-
# The plugin's exported API is not very useful for unit testing, so build the
|
|
84
|
-
# sources directly into the test binary rather than using the shared library.
|
|
85
|
-
add_executable(${TEST_RUNNER}
|
|
86
|
-
test/audio_capture_plugin_test.cc
|
|
87
|
-
${PLUGIN_SOURCES}
|
|
88
|
-
)
|
|
89
|
-
apply_standard_settings(${TEST_RUNNER})
|
|
90
|
-
target_include_directories(${TEST_RUNNER} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
|
|
91
|
-
target_link_libraries(${TEST_RUNNER} PRIVATE flutter)
|
|
92
|
-
target_link_libraries(${TEST_RUNNER} PRIVATE PkgConfig::GTK)
|
|
93
|
-
target_link_libraries(${TEST_RUNNER} PRIVATE PkgConfig::PULSEAUDIO)
|
|
94
|
-
target_link_libraries(${TEST_RUNNER} PRIVATE gtest_main gmock)
|
|
95
|
-
|
|
96
|
-
# Enable automatic test discovery.
|
|
97
|
-
include(GoogleTest)
|
|
98
|
-
gtest_discover_tests(${TEST_RUNNER})
|
|
99
|
-
|
|
100
|
-
endif() # CMake version check
|
|
101
|
-
endif() # include_${PROJECT_NAME}_tests
|