neoagent 3.1.1-beta.9 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/.env.example +2 -2
  2. package/README.md +5 -1
  3. package/docs/architecture.md +1 -1
  4. package/docs/benchmarking.md +68 -87
  5. package/docs/capabilities.md +1 -1
  6. package/docs/clients-and-devices.md +3 -4
  7. package/docs/configuration.md +1 -1
  8. package/docs/devices.md +1 -2
  9. package/docs/health.md +24 -0
  10. package/docs/index.md +2 -2
  11. package/docs/integrations-architecture.md +6 -0
  12. package/docs/integrations.md +15 -1
  13. package/docs/models.md +1 -1
  14. package/docs/persistence.md +1 -1
  15. package/docs/why-neoagent.md +1 -1
  16. package/flutter_app/README.md +0 -5
  17. package/flutter_app/android/app/src/main/AndroidManifest.xml +0 -14
  18. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +0 -98
  19. package/flutter_app/lib/main.dart +0 -6
  20. package/flutter_app/lib/main_app_shell.dart +0 -2
  21. package/flutter_app/lib/main_controller.dart +2 -867
  22. package/flutter_app/lib/main_devices.dart +0 -73
  23. package/flutter_app/lib/main_integrations.dart +339 -187
  24. package/flutter_app/lib/main_launcher.dart +2 -212
  25. package/flutter_app/lib/main_models.dart +0 -259
  26. package/flutter_app/lib/main_navigation.dart +2 -14
  27. package/flutter_app/lib/main_operations.dart +322 -364
  28. package/flutter_app/lib/main_runtime.dart +9 -208
  29. package/flutter_app/lib/main_settings.dart +17 -199
  30. package/flutter_app/lib/main_shared.dart +20 -591
  31. package/flutter_app/lib/main_timeline.dart +26 -122
  32. package/flutter_app/lib/main_voice_assistant.dart +0 -5
  33. package/flutter_app/lib/src/backend_client.dart +0 -102
  34. package/flutter_app/lib/src/desktop_companion_io.dart +1 -65
  35. package/flutter_app/lib/src/desktop_companion_stub.dart +0 -12
  36. package/flutter_app/linux/flutter/generated_plugin_registrant.cc +0 -4
  37. package/flutter_app/linux/flutter/generated_plugins.cmake +0 -1
  38. package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +0 -2
  39. package/flutter_app/macos/Podfile.lock +56 -5
  40. package/flutter_app/pubspec.lock +0 -7
  41. package/flutter_app/pubspec.yaml +0 -2
  42. package/flutter_app/web/index.html +174 -0
  43. package/flutter_app/windows/flutter/generated_plugin_registrant.cc +0 -3
  44. package/flutter_app/windows/flutter/generated_plugins.cmake +0 -1
  45. package/landing/index.html +0 -2
  46. package/lib/manager.js +1 -1
  47. package/lib/schema_migrations.js +75 -0
  48. package/package.json +1 -4
  49. package/server/db/database.js +1 -133
  50. package/server/http/middleware.js +2 -16
  51. package/server/http/routes.js +0 -2
  52. package/server/public/.last_build_id +1 -1
  53. package/server/public/assets/NOTICES +0 -25
  54. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  55. package/server/public/flutter_bootstrap.js +1 -1
  56. package/server/public/index.html +174 -0
  57. package/server/public/main.dart.js +88081 -90036
  58. package/server/routes/auth.js +11 -2
  59. package/server/routes/settings.js +0 -5
  60. package/server/routes/voice_assistant.js +1 -128
  61. package/server/services/account/erasure.js +0 -3
  62. package/server/services/ai/deliverables/artifact_helpers.js +1 -1
  63. package/server/services/ai/loop/conversation_loop.js +6 -2
  64. package/server/services/ai/loop/tool_dispatch.js +0 -3
  65. package/server/services/ai/messagingFallback.js +1 -1
  66. package/server/services/ai/settings.js +0 -6
  67. package/server/services/ai/toolEvidence.js +1 -4
  68. package/server/services/ai/tools.js +0 -234
  69. package/server/services/desktop/auth.js +0 -3
  70. package/server/services/desktop/registry.js +2 -50
  71. package/server/services/integrations/neoarchive/provider.js +616 -0
  72. package/server/services/integrations/neorecall/client.js +140 -0
  73. package/server/services/integrations/neorecall/constants.js +87 -0
  74. package/server/services/integrations/neorecall/provider.js +163 -0
  75. package/server/services/integrations/registry.js +4 -2
  76. package/server/services/manager.js +0 -22
  77. package/server/services/memory/manager.js +44 -19
  78. package/server/services/memory/retrieval_reasoning.js +36 -14
  79. package/server/services/messaging/whatsapp.js +18 -0
  80. package/server/services/security/tool_categories.js +0 -5
  81. package/server/services/social_video/service.js +2 -20
  82. package/server/services/tasks/adapters/index.js +1 -1
  83. package/server/services/tasks/integration_runtime.js +3 -67
  84. package/server/services/tasks/runtime.js +1 -6
  85. package/server/services/tasks/task_repository.js +6 -13
  86. package/server/services/timeline/service.js +0 -224
  87. package/server/services/voice/message.js +0 -24
  88. package/server/services/voice/providers.js +42 -1
  89. package/server/services/voice/runtimeManager.js +9 -5
  90. package/server/services/voice/turnRunner.js +0 -22
  91. package/server/services/wearable/service.js +0 -5
  92. package/docs/recordings-and-health.md +0 -41
  93. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +0 -21
  94. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +0 -586
  95. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +0 -78
  96. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +0 -104
  97. package/flutter_app/lib/main_recordings.dart +0 -920
  98. package/flutter_app/lib/src/desktop_ocr_bridge.dart +0 -2
  99. package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +0 -125
  100. package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +0 -30
  101. package/flutter_app/lib/src/desktop_passive_history.dart +0 -332
  102. package/flutter_app/lib/src/recording_bridge.dart +0 -232
  103. package/flutter_app/lib/src/recording_bridge_io.dart +0 -1027
  104. package/flutter_app/lib/src/recording_bridge_stub.dart +0 -120
  105. package/flutter_app/lib/src/recording_bridge_web.dart +0 -702
  106. package/flutter_app/lib/src/recording_chunk_queue.dart +0 -149
  107. package/flutter_app/lib/src/recording_chunk_queue_io.dart +0 -182
  108. package/flutter_app/lib/src/recording_payloads.dart +0 -95
  109. package/flutter_app/third_party/desktop_audio_capture/LICENSE +0 -21
  110. package/flutter_app/third_party/desktop_audio_capture/README.md +0 -262
  111. package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +0 -65
  112. package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +0 -153
  113. package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +0 -110
  114. package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +0 -461
  115. package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +0 -91
  116. package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +0 -106
  117. package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +0 -219
  118. package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +0 -336
  119. package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +0 -101
  120. package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +0 -692
  121. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +0 -35
  122. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +0 -36
  123. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +0 -32
  124. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +0 -32
  125. package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +0 -878
  126. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +0 -27
  127. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +0 -1172
  128. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +0 -655
  129. package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +0 -12
  130. package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +0 -30
  131. package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +0 -87
  132. package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +0 -105
  133. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +0 -80
  134. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +0 -31
  135. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +0 -12
  136. package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +0 -23
  137. package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +0 -25
  138. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +0 -1117
  139. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +0 -115
  140. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +0 -777
  141. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +0 -87
  142. package/server/routes/recordings.js +0 -335
  143. package/server/routes/screenHistory.js +0 -190
  144. package/server/services/ai/recordingInsights.js +0 -105
  145. package/server/services/integrations/neomail/provider.js +0 -993
  146. package/server/services/recordings/deepgram.js +0 -58
  147. package/server/services/recordings/manager.js +0 -1273
  148. package/server/services/tasks/adapters/neomail_email_received.js +0 -46
  149. package/server/services/voice/screenshotContext.js +0 -73
  150. package/server/services/voice/turnCoordinator.js +0 -51
@@ -1,692 +0,0 @@
1
- #include "include/audio_capture/audio_capture_plugin.h"
2
- #include "include/audio_capture/mic_capture_plugin.h"
3
-
4
- #include <flutter_linux/flutter_linux.h>
5
- #include <glib-object.h>
6
- #include <glib.h>
7
- #include <pulse/error.h>
8
- #include <pulse/simple.h>
9
-
10
- #include <algorithm>
11
- #include <cmath>
12
- #include <cstdint>
13
- #include <memory>
14
- #include <string>
15
- #include <vector>
16
-
17
- namespace {
18
-
19
- constexpr char kMethodChannelName[] = "com.system_audio_transcriber/audio_capture";
20
- constexpr char kEventChannelName[] = "com.system_audio_transcriber/audio_stream";
21
- constexpr char kStatusEventChannelName[] = "com.system_audio_transcriber/audio_status";
22
- constexpr char kDecibelEventChannelName[] = "com.system_audio_transcriber/audio_decibel";
23
-
24
- constexpr int kDefaultSampleRate = 16000;
25
- constexpr int kDefaultChannels = 1;
26
- constexpr int kDefaultBitsPerSample = 16;
27
- constexpr int kDefaultChunkDurationMs = 1000;
28
- constexpr float kDefaultGainBoost = 2.5f;
29
- constexpr float kDefaultInputVolume = 1.0f;
30
-
31
- struct AudioChunkPayload {
32
- AudioChunkPayload(AudioCapturePlugin* plugin, GBytes* bytes, double decibel)
33
- : plugin(plugin), bytes(bytes), decibel(decibel) {}
34
-
35
- AudioCapturePlugin* plugin;
36
- GBytes* bytes;
37
- double decibel;
38
- };
39
-
40
- struct CaptureThreadContext {
41
- AudioCapturePlugin* plugin;
42
- pa_simple* stream;
43
- size_t chunk_size;
44
- int sample_rate;
45
- int channels;
46
- int bits_per_sample;
47
- float gain_boost;
48
- float input_volume;
49
- };
50
-
51
- gboolean EmitAudioOnMainThread(gpointer user_data);
52
- gpointer CaptureThread(gpointer user_data);
53
- double CalculateDecibel(const int16_t* samples, size_t sample_count);
54
-
55
- } // namespace
56
-
57
- struct _AudioCapturePlugin {
58
- GObject parent_instance;
59
-
60
- FlMethodChannel* method_channel;
61
- FlEventChannel* event_channel;
62
- FlEventChannel* status_event_channel;
63
- FlEventChannel* decibel_event_channel;
64
- GMainContext* main_context;
65
-
66
- GMutex lock;
67
- gint should_stop;
68
- gboolean is_capturing;
69
- gboolean has_listener;
70
- gboolean has_status_listener;
71
- gboolean has_decibel_listener;
72
-
73
- GThread* capture_thread;
74
- };
75
-
76
- G_DEFINE_TYPE(AudioCapturePlugin, audio_capture_plugin, G_TYPE_OBJECT)
77
-
78
- namespace {
79
-
80
- bool OpenPulseStream(int sample_rate, int channels, int bits_per_sample,
81
- size_t chunk_size, pa_simple** out_stream, std::string* error_message) {
82
- pa_sample_spec spec;
83
- spec.rate = sample_rate;
84
- spec.channels = static_cast<uint8_t>(channels);
85
- if (bits_per_sample == 16) {
86
- spec.format = PA_SAMPLE_S16LE;
87
- } else {
88
- spec.format = PA_SAMPLE_S16LE;
89
- }
90
-
91
- pa_buffer_attr attr;
92
- attr.maxlength = static_cast<uint32_t>(chunk_size * 4);
93
- attr.tlength = (uint32_t)-1;
94
- attr.prebuf = (uint32_t)-1;
95
- attr.minreq = (uint32_t)-1;
96
- attr.fragsize = static_cast<uint32_t>(chunk_size);
97
-
98
- int error = 0;
99
-
100
- pa_simple* stream =
101
- pa_simple_new(nullptr, "Voxa", PA_STREAM_RECORD, "@DEFAULT_MONITOR@",
102
- "System Capture", &spec, nullptr, &attr, &error);
103
-
104
- if (stream == nullptr) {
105
- // Fallback to default source (microphone) if monitor is unavailable.
106
- stream = pa_simple_new(nullptr, "Voxa", PA_STREAM_RECORD, nullptr,
107
- "Default Capture", &spec, nullptr, &attr, &error);
108
- }
109
-
110
- if (stream == nullptr) {
111
- if (error_message != nullptr) {
112
- *error_message = pa_strerror(error);
113
- }
114
- return false;
115
- }
116
-
117
- *out_stream = stream;
118
- return true;
119
- }
120
-
121
- size_t CalculateChunkSize(int sample_rate, int channels, int bits_per_sample,
122
- int chunk_duration_ms) {
123
- const int bytes_per_sample = std::max(bits_per_sample / 8, 1);
124
- const size_t bytes_per_second =
125
- static_cast<size_t>(sample_rate) * static_cast<size_t>(channels) *
126
- static_cast<size_t>(bytes_per_sample);
127
- size_t chunk_size =
128
- (bytes_per_second * static_cast<size_t>(chunk_duration_ms)) / 1000;
129
- if (chunk_size == 0) {
130
- chunk_size = bytes_per_second / 20; // 50 ms fallback
131
- }
132
- const size_t frame_size = static_cast<size_t>(channels) * bytes_per_sample;
133
- chunk_size = std::max(chunk_size, frame_size);
134
- return chunk_size;
135
- }
136
-
137
- void ApplyGainBoostAndConvertToMono(const int16_t* input, int16_t* output,
138
- size_t frame_count, int input_channels,
139
- float gain_boost) {
140
- const float max_value = 32767.0f;
141
- const float min_value = -32768.0f;
142
-
143
- if (input_channels == 1) {
144
- // Mono: just apply gain boost
145
- for (size_t i = 0; i < frame_count; ++i) {
146
- float sample = static_cast<float>(input[i]) * gain_boost;
147
- sample = std::max(min_value, std::min(max_value, sample));
148
- output[i] = static_cast<int16_t>(sample);
149
- }
150
- } else {
151
- // Stereo: convert to mono and apply gain boost
152
- for (size_t i = 0; i < frame_count; ++i) {
153
- float left = static_cast<float>(input[i * 2]);
154
- float right = static_cast<float>(input[i * 2 + 1]);
155
- float mono = (left + right) / 2.0f * gain_boost;
156
- mono = std::max(min_value, std::min(max_value, mono));
157
- output[i] = static_cast<int16_t>(mono);
158
- }
159
- }
160
- }
161
-
162
- double CalculateDecibel(const int16_t* samples, size_t sample_count) {
163
- if (sample_count == 0) {
164
- return -120.0;
165
- }
166
-
167
- // Calculate RMS (Root Mean Square)
168
- double sum_of_squares = 0.0;
169
- for (size_t i = 0; i < sample_count; ++i) {
170
- double value = static_cast<double>(samples[i]);
171
- sum_of_squares += value * value;
172
- }
173
- double mean_square = sum_of_squares / static_cast<double>(sample_count);
174
- double rms = sqrt(mean_square);
175
-
176
- // Calculate decibel: dB = 20 * log10(RMS / max_value)
177
- // For Int16, max_value is 32767.0
178
- const double max_value = 32767.0;
179
- if (rms <= 0.0) {
180
- return -120.0; // Avoid log(0)
181
- }
182
-
183
- double decibel = 20.0 * log10(rms / max_value);
184
-
185
- // Clamp to reasonable range (-120 dB to 0 dB)
186
- return std::max(-120.0, std::min(0.0, decibel));
187
- }
188
-
189
- gboolean EmitAudioOnMainThread(gpointer user_data) {
190
- std::unique_ptr<AudioChunkPayload> payload(
191
- static_cast<AudioChunkPayload*>(user_data));
192
- AudioCapturePlugin* plugin = payload->plugin;
193
-
194
- gsize length = 0;
195
- const guint8* data =
196
- static_cast<const guint8*>(g_bytes_get_data(payload->bytes, &length));
197
-
198
- g_mutex_lock(&plugin->lock);
199
- const gboolean can_emit =
200
- plugin->event_channel != nullptr && plugin->has_listener;
201
- const gboolean can_emit_decibel =
202
- plugin->decibel_event_channel != nullptr && plugin->has_decibel_listener;
203
- g_mutex_unlock(&plugin->lock);
204
-
205
- if (can_emit && length > 0) {
206
- g_autoptr(FlValue) value = fl_value_new_uint8_list(data, length);
207
- g_autoptr(GError) error = nullptr;
208
-
209
- if (!fl_event_channel_send(plugin->event_channel, value, nullptr, &error)) {
210
- g_warning("Failed to send audio chunk: %s",
211
- error != nullptr ? error->message : "unknown error");
212
- }
213
- }
214
-
215
- // Send decibel data
216
- if (can_emit_decibel) {
217
- g_autoptr(FlValue) decibel_map = fl_value_new_map();
218
- fl_value_set_string_take(decibel_map, "decibel", fl_value_new_float(payload->decibel));
219
- fl_value_set_string_take(decibel_map, "timestamp", fl_value_new_float(g_get_real_time() / 1000000.0));
220
-
221
- g_autoptr(GError) error = nullptr;
222
- if (!fl_event_channel_send(plugin->decibel_event_channel, decibel_map, nullptr, &error)) {
223
- g_warning("Failed to send decibel data: %s",
224
- error != nullptr ? error->message : "unknown error");
225
- }
226
- }
227
-
228
- g_bytes_unref(payload->bytes);
229
- g_object_unref(plugin);
230
-
231
- return G_SOURCE_REMOVE;
232
- }
233
-
234
- gpointer CaptureThread(gpointer user_data) {
235
- std::unique_ptr<CaptureThreadContext> context(
236
- static_cast<CaptureThreadContext*>(user_data));
237
- AudioCapturePlugin* plugin = context->plugin;
238
-
239
- // Read raw audio from PulseAudio
240
- std::vector<uint8_t> raw_buffer(context->chunk_size);
241
-
242
- // Output buffer for processed audio (mono)
243
- const size_t output_frame_count = context->chunk_size / (sizeof(int16_t) * context->channels);
244
- std::vector<int16_t> output_buffer(output_frame_count);
245
-
246
- while (!g_atomic_int_get(&plugin->should_stop)) {
247
- int error = 0;
248
- if (pa_simple_read(context->stream, raw_buffer.data(), raw_buffer.size(),
249
- &error) < 0) {
250
- g_warning("PulseAudio read error: %s", pa_strerror(error));
251
- break;
252
- }
253
-
254
- if (g_atomic_int_get(&plugin->should_stop)) {
255
- break;
256
- }
257
-
258
- // Apply input volume
259
- if (context->input_volume < 1.0f) {
260
- int16_t* samples = reinterpret_cast<int16_t*>(raw_buffer.data());
261
- const size_t sample_count = raw_buffer.size() / sizeof(int16_t);
262
- for (size_t i = 0; i < sample_count; ++i) {
263
- samples[i] = static_cast<int16_t>(
264
- static_cast<float>(samples[i]) * context->input_volume);
265
- }
266
- }
267
-
268
- // Process audio: convert to mono and apply gain boost
269
- const int16_t* input_samples =
270
- reinterpret_cast<const int16_t*>(raw_buffer.data());
271
- const size_t input_frame_count =
272
- raw_buffer.size() / (sizeof(int16_t) * context->channels);
273
- const size_t frames_to_process =
274
- std::min(input_frame_count, output_frame_count);
275
-
276
- // Always process to ensure mono output and gain boost application
277
- ApplyGainBoostAndConvertToMono(input_samples, output_buffer.data(),
278
- frames_to_process, context->channels,
279
- context->gain_boost);
280
-
281
- // Create output bytes (mono)
282
- const size_t output_bytes = frames_to_process * sizeof(int16_t);
283
-
284
- // Calculate decibel from output buffer
285
- double decibel = CalculateDecibel(output_buffer.data(), frames_to_process);
286
-
287
- GBytes* bytes = g_bytes_new(output_buffer.data(), output_bytes);
288
- auto* payload = new AudioChunkPayload(plugin, bytes, decibel);
289
- g_object_ref(plugin);
290
- g_main_context_invoke_full(plugin->main_context, G_PRIORITY_DEFAULT,
291
- EmitAudioOnMainThread, payload, nullptr);
292
- }
293
-
294
- pa_simple_free(context->stream);
295
-
296
- g_mutex_lock(&plugin->lock);
297
- plugin->is_capturing = FALSE;
298
- plugin->capture_thread = nullptr;
299
- g_mutex_unlock(&plugin->lock);
300
-
301
- // Send status update
302
- g_mutex_lock(&plugin->lock);
303
- const gboolean has_status_listener = plugin->has_status_listener;
304
- g_mutex_unlock(&plugin->lock);
305
-
306
- if (has_status_listener && plugin->status_event_channel != nullptr) {
307
- g_autoptr(FlValue) status_map = fl_value_new_map();
308
- fl_value_set_string_take(status_map, "isActive", fl_value_new_bool(FALSE));
309
- fl_value_set_string_take(status_map, "timestamp", fl_value_new_float(g_get_real_time() / 1000000.0));
310
-
311
- g_autoptr(GError) error = nullptr;
312
- fl_event_channel_send(plugin->status_event_channel, status_map, nullptr, &error);
313
- }
314
-
315
- g_object_unref(plugin);
316
- return nullptr;
317
- }
318
-
319
- static FlMethodErrorResponse* OnListenHandler(FlEventChannel* channel,
320
- FlValue* arguments,
321
- gpointer user_data) {
322
- AudioCapturePlugin* plugin = AUDIO_CAPTURE_PLUGIN(user_data);
323
- (void)channel;
324
- (void)arguments;
325
- g_mutex_lock(&plugin->lock);
326
- plugin->has_listener = TRUE;
327
- g_mutex_unlock(&plugin->lock);
328
- return nullptr;
329
- }
330
-
331
- static FlMethodErrorResponse* OnCancelHandler(FlEventChannel* channel,
332
- FlValue* arguments,
333
- gpointer user_data) {
334
- AudioCapturePlugin* plugin = AUDIO_CAPTURE_PLUGIN(user_data);
335
- (void)channel;
336
- (void)arguments;
337
- g_mutex_lock(&plugin->lock);
338
- plugin->has_listener = FALSE;
339
- g_mutex_unlock(&plugin->lock);
340
- return nullptr;
341
- }
342
-
343
- static FlMethodErrorResponse* OnStatusListenHandler(FlEventChannel* channel,
344
- FlValue* arguments,
345
- gpointer user_data) {
346
- AudioCapturePlugin* plugin = AUDIO_CAPTURE_PLUGIN(user_data);
347
- (void)channel;
348
- (void)arguments;
349
- g_mutex_lock(&plugin->lock);
350
- plugin->has_status_listener = TRUE;
351
- const gboolean is_active = plugin->is_capturing;
352
- g_mutex_unlock(&plugin->lock);
353
-
354
- // Send current status immediately
355
- g_autoptr(FlValue) status_map = fl_value_new_map();
356
- fl_value_set_string_take(status_map, "isActive", fl_value_new_bool(is_active));
357
- fl_value_set_string_take(status_map, "timestamp", fl_value_new_float(g_get_real_time() / 1000000.0));
358
-
359
- g_autoptr(GError) error = nullptr;
360
- fl_event_channel_send(plugin->status_event_channel, status_map, nullptr, &error);
361
-
362
- return nullptr;
363
- }
364
-
365
- static FlMethodErrorResponse* OnStatusCancelHandler(FlEventChannel* channel,
366
- FlValue* arguments,
367
- gpointer user_data) {
368
- AudioCapturePlugin* plugin = AUDIO_CAPTURE_PLUGIN(user_data);
369
- (void)channel;
370
- (void)arguments;
371
- g_mutex_lock(&plugin->lock);
372
- plugin->has_status_listener = FALSE;
373
- g_mutex_unlock(&plugin->lock);
374
- return nullptr;
375
- }
376
-
377
- static FlMethodErrorResponse* OnDecibelListenHandler(FlEventChannel* channel,
378
- FlValue* arguments,
379
- gpointer user_data) {
380
- AudioCapturePlugin* plugin = AUDIO_CAPTURE_PLUGIN(user_data);
381
- (void)channel;
382
- (void)arguments;
383
- g_mutex_lock(&plugin->lock);
384
- plugin->has_decibel_listener = TRUE;
385
- g_mutex_unlock(&plugin->lock);
386
- return nullptr;
387
- }
388
-
389
- static FlMethodErrorResponse* OnDecibelCancelHandler(FlEventChannel* channel,
390
- FlValue* arguments,
391
- gpointer user_data) {
392
- AudioCapturePlugin* plugin = AUDIO_CAPTURE_PLUGIN(user_data);
393
- (void)channel;
394
- (void)arguments;
395
- g_mutex_lock(&plugin->lock);
396
- plugin->has_decibel_listener = FALSE;
397
- g_mutex_unlock(&plugin->lock);
398
- return nullptr;
399
- }
400
-
401
- bool StartCapture(AudioCapturePlugin* plugin, FlValue* args) {
402
- int sample_rate = kDefaultSampleRate;
403
- int channels = kDefaultChannels;
404
- int bits_per_sample = kDefaultBitsPerSample;
405
- int chunk_duration_ms = kDefaultChunkDurationMs;
406
- float gain_boost = kDefaultGainBoost;
407
- float input_volume = kDefaultInputVolume;
408
-
409
- if (args != nullptr && fl_value_get_type(args) == FL_VALUE_TYPE_MAP) {
410
- FlValue* value = nullptr;
411
-
412
- value = fl_value_lookup_string(args, "sampleRate");
413
- if (value != nullptr && fl_value_get_type(value) == FL_VALUE_TYPE_INT) {
414
- sample_rate = fl_value_get_int(value);
415
- }
416
-
417
- value = fl_value_lookup_string(args, "channels");
418
- if (value != nullptr && fl_value_get_type(value) == FL_VALUE_TYPE_INT) {
419
- channels = fl_value_get_int(value);
420
- }
421
-
422
- value = fl_value_lookup_string(args, "bitsPerSample");
423
- if (value != nullptr && fl_value_get_type(value) == FL_VALUE_TYPE_INT) {
424
- bits_per_sample = fl_value_get_int(value);
425
- }
426
-
427
- value = fl_value_lookup_string(args, "chunkDurationMs");
428
- if (value != nullptr && fl_value_get_type(value) == FL_VALUE_TYPE_INT) {
429
- chunk_duration_ms = fl_value_get_int(value);
430
- }
431
-
432
- value = fl_value_lookup_string(args, "gainBoost");
433
- if (value != nullptr && fl_value_get_type(value) == FL_VALUE_TYPE_FLOAT) {
434
- gain_boost = fl_value_get_float(value);
435
- gain_boost = std::max(0.1f, std::min(10.0f, gain_boost));
436
- }
437
-
438
- value = fl_value_lookup_string(args, "inputVolume");
439
- if (value != nullptr && fl_value_get_type(value) == FL_VALUE_TYPE_FLOAT) {
440
- input_volume = fl_value_get_float(value);
441
- input_volume = std::max(0.0f, std::min(1.0f, input_volume));
442
- }
443
- }
444
-
445
- sample_rate = std::max(sample_rate, 8000);
446
- channels = std::max(1, std::min(channels, 2));
447
- bits_per_sample = 16;
448
- chunk_duration_ms = std::max(chunk_duration_ms, 10);
449
- gain_boost = std::max(0.1f, std::min(10.0f, gain_boost));
450
- input_volume = std::max(0.0f, std::min(1.0f, input_volume));
451
-
452
- size_t chunk_size =
453
- CalculateChunkSize(sample_rate, channels, bits_per_sample,
454
- chunk_duration_ms);
455
-
456
- pa_simple* stream = nullptr;
457
- std::string error_message;
458
-
459
- if (!OpenPulseStream(sample_rate, channels, bits_per_sample, chunk_size,
460
- &stream, &error_message)) {
461
- g_warning("Failed to open PulseAudio stream: %s", error_message.c_str());
462
- return false;
463
- }
464
-
465
- g_mutex_lock(&plugin->lock);
466
- if (plugin->is_capturing) {
467
- g_mutex_unlock(&plugin->lock);
468
- pa_simple_free(stream);
469
- return false;
470
- }
471
-
472
- g_atomic_int_set(&plugin->should_stop, 0);
473
- plugin->is_capturing = TRUE;
474
-
475
- auto* context = new CaptureThreadContext{
476
- plugin,
477
- stream,
478
- chunk_size,
479
- sample_rate,
480
- channels,
481
- bits_per_sample,
482
- gain_boost,
483
- input_volume,
484
- };
485
-
486
- g_object_ref(plugin);
487
- plugin->capture_thread = g_thread_new("voxa-audio-capture", CaptureThread,
488
- context);
489
- g_mutex_unlock(&plugin->lock);
490
-
491
- if (plugin->capture_thread == nullptr) {
492
- g_warning("Failed to create capture thread");
493
- g_mutex_lock(&plugin->lock);
494
- plugin->is_capturing = FALSE;
495
- g_mutex_unlock(&plugin->lock);
496
- pa_simple_free(stream);
497
- g_object_unref(plugin);
498
- delete context;
499
- return false;
500
- }
501
-
502
- // Send status update
503
- g_mutex_lock(&plugin->lock);
504
- const gboolean has_status_listener = plugin->has_status_listener;
505
- g_mutex_unlock(&plugin->lock);
506
-
507
- if (has_status_listener && plugin->status_event_channel != nullptr) {
508
- g_autoptr(FlValue) status_map = fl_value_new_map();
509
- fl_value_set_string_take(status_map, "isActive", fl_value_new_bool(TRUE));
510
- fl_value_set_string_take(status_map, "timestamp", fl_value_new_float(g_get_real_time() / 1000000.0));
511
-
512
- g_autoptr(GError) error = nullptr;
513
- fl_event_channel_send(plugin->status_event_channel, status_map, nullptr, &error);
514
- }
515
-
516
- return true;
517
- }
518
-
519
- bool StopCapture(AudioCapturePlugin* plugin) {
520
- g_mutex_lock(&plugin->lock);
521
- if (!plugin->is_capturing) {
522
- g_mutex_unlock(&plugin->lock);
523
- return false;
524
- }
525
- g_atomic_int_set(&plugin->should_stop, 1);
526
- GThread* thread = plugin->capture_thread;
527
- g_mutex_unlock(&plugin->lock);
528
-
529
- if (thread != nullptr) {
530
- g_thread_join(thread);
531
- }
532
-
533
- g_mutex_lock(&plugin->lock);
534
- plugin->capture_thread = nullptr;
535
- plugin->is_capturing = FALSE;
536
- const gboolean has_status_listener = plugin->has_status_listener;
537
- g_mutex_unlock(&plugin->lock);
538
-
539
- // Wait a bit to ensure thread has fully stopped
540
- g_usleep(100000); // 0.1 seconds
541
-
542
- // Send status update
543
- if (has_status_listener && plugin->status_event_channel != nullptr) {
544
- g_autoptr(FlValue) status_map = fl_value_new_map();
545
- fl_value_set_string_take(status_map, "isActive", fl_value_new_bool(FALSE));
546
- fl_value_set_string_take(status_map, "timestamp", fl_value_new_float(g_get_real_time() / 1000000.0));
547
-
548
- g_autoptr(GError) error = nullptr;
549
- fl_event_channel_send(plugin->status_event_channel, status_map, nullptr, &error);
550
- }
551
-
552
- return true;
553
- }
554
-
555
- void HandleMethodCall(AudioCapturePlugin* plugin, FlMethodCall* method_call) {
556
- const gchar* method = fl_method_call_get_name(method_call);
557
- g_autoptr(FlMethodResponse) response = nullptr;
558
-
559
- if (strcmp(method, "requestPermissions") == 0) {
560
- g_autoptr(FlValue) result = fl_value_new_bool(TRUE);
561
- response = FL_METHOD_RESPONSE(fl_method_success_response_new(result));
562
- } else if (strcmp(method, "startCapture") == 0) {
563
- FlValue* args = fl_method_call_get_args(method_call);
564
- const bool started = StartCapture(plugin, args);
565
- g_autoptr(FlValue) result = fl_value_new_bool(started);
566
- response = FL_METHOD_RESPONSE(fl_method_success_response_new(result));
567
- } else if (strcmp(method, "stopCapture") == 0) {
568
- const bool stopped = StopCapture(plugin);
569
- g_autoptr(FlValue) result = fl_value_new_bool(stopped);
570
- response = FL_METHOD_RESPONSE(fl_method_success_response_new(result));
571
- } else {
572
- response = FL_METHOD_RESPONSE(fl_method_not_implemented_response_new());
573
- }
574
-
575
- g_autoptr(GError) error = nullptr;
576
- if (!fl_method_call_respond(method_call, response, &error)) {
577
- g_warning("Failed to send method call response: %s", error->message);
578
- }
579
- }
580
-
581
- static void MethodCallHandler(FlMethodChannel* channel,
582
- FlMethodCall* method_call,
583
- gpointer user_data) {
584
- AudioCapturePlugin* plugin = AUDIO_CAPTURE_PLUGIN(user_data);
585
- HandleMethodCall(plugin, method_call);
586
- }
587
-
588
- } // namespace
589
-
590
- static void audio_capture_plugin_dispose(GObject* object) {
591
- AudioCapturePlugin* plugin = AUDIO_CAPTURE_PLUGIN(object);
592
-
593
- StopCapture(plugin);
594
-
595
- if (plugin->method_channel != nullptr) {
596
- g_clear_object(&plugin->method_channel);
597
- }
598
-
599
- if (plugin->event_channel != nullptr) {
600
- g_clear_object(&plugin->event_channel);
601
- }
602
-
603
- if (plugin->status_event_channel != nullptr) {
604
- g_clear_object(&plugin->status_event_channel);
605
- }
606
-
607
- if (plugin->decibel_event_channel != nullptr) {
608
- g_clear_object(&plugin->decibel_event_channel);
609
- }
610
-
611
- if (plugin->main_context != nullptr) {
612
- g_main_context_unref(plugin->main_context);
613
- plugin->main_context = nullptr;
614
- }
615
-
616
- g_mutex_clear(&plugin->lock);
617
-
618
- G_OBJECT_CLASS(audio_capture_plugin_parent_class)->dispose(object);
619
- }
620
-
621
- static void audio_capture_plugin_class_init(AudioCapturePluginClass* klass) {
622
- GObjectClass* object_class = G_OBJECT_CLASS(klass);
623
- object_class->dispose = audio_capture_plugin_dispose;
624
- }
625
-
626
- static void audio_capture_plugin_init(AudioCapturePlugin* plugin) {
627
- g_mutex_init(&plugin->lock);
628
- plugin->main_context = g_main_context_ref_thread_default();
629
- plugin->is_capturing = FALSE;
630
- plugin->has_listener = FALSE;
631
- plugin->has_status_listener = FALSE;
632
- plugin->has_decibel_listener = FALSE;
633
- plugin->method_channel = nullptr;
634
- plugin->event_channel = nullptr;
635
- plugin->status_event_channel = nullptr;
636
- plugin->decibel_event_channel = nullptr;
637
- plugin->capture_thread = nullptr;
638
- g_atomic_int_set(&plugin->should_stop, 0);
639
- }
640
-
641
- void audio_capture_plugin_register_with_registrar(FlPluginRegistrar* registrar) {
642
- FlBinaryMessenger* messenger = fl_plugin_registrar_get_messenger(registrar);
643
- audio_capture_plugin_register_with_messenger(messenger);
644
- // Also register the mic capture plugin
645
- mic_capture_plugin_register_with_messenger(messenger);
646
- }
647
-
648
- void audio_capture_plugin_register_with_messenger(FlBinaryMessenger* messenger) {
649
- AudioCapturePlugin* plugin =
650
- AUDIO_CAPTURE_PLUGIN(g_object_new(audio_capture_plugin_get_type(), nullptr));
651
-
652
- g_autoptr(FlStandardMethodCodec) codec = fl_standard_method_codec_new();
653
-
654
- plugin->method_channel = fl_method_channel_new(
655
- messenger, kMethodChannelName, FL_METHOD_CODEC(codec));
656
- fl_method_channel_set_method_call_handler(
657
- plugin->method_channel, MethodCallHandler, g_object_ref(plugin),
658
- g_object_unref);
659
-
660
- plugin->event_channel = fl_event_channel_new(
661
- messenger, kEventChannelName, FL_METHOD_CODEC(codec));
662
-
663
- // Use the newer API with proper function signatures
664
- fl_event_channel_set_stream_handlers(
665
- plugin->event_channel,
666
- OnListenHandler,
667
- OnCancelHandler,
668
- g_object_ref(plugin),
669
- g_object_unref);
670
-
671
- // Register status event channel
672
- plugin->status_event_channel = fl_event_channel_new(
673
- messenger, kStatusEventChannelName, FL_METHOD_CODEC(codec));
674
- fl_event_channel_set_stream_handlers(
675
- plugin->status_event_channel,
676
- OnStatusListenHandler,
677
- OnStatusCancelHandler,
678
- g_object_ref(plugin),
679
- g_object_unref);
680
-
681
- // Register decibel event channel
682
- plugin->decibel_event_channel = fl_event_channel_new(
683
- messenger, kDecibelEventChannelName, FL_METHOD_CODEC(codec));
684
- fl_event_channel_set_stream_handlers(
685
- plugin->decibel_event_channel,
686
- OnDecibelListenHandler,
687
- OnDecibelCancelHandler,
688
- g_object_ref(plugin),
689
- g_object_unref);
690
-
691
- g_object_unref(plugin);
692
- }