neoagent 2.3.1-beta.2 → 2.3.1-beta.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. package/.env.example +39 -0
  2. package/README.md +2 -0
  3. package/docs/capabilities.md +2 -2
  4. package/docs/configuration.md +13 -5
  5. package/docs/integrations.md +4 -1
  6. package/flutter_app/.metadata +42 -0
  7. package/flutter_app/README.md +21 -0
  8. package/flutter_app/analysis_options.yaml +32 -0
  9. package/flutter_app/android/app/build.gradle.kts +109 -0
  10. package/flutter_app/android/app/src/debug/AndroidManifest.xml +7 -0
  11. package/flutter_app/android/app/src/main/AndroidManifest.xml +147 -0
  12. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +747 -0
  13. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthConnectGateway.kt +280 -0
  14. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncNotifications.kt +113 -0
  15. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncPayload.kt +57 -0
  16. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncScheduler.kt +78 -0
  17. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncWorker.kt +253 -0
  18. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/PermissionsRationaleActivity.kt +46 -0
  19. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +21 -0
  20. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +586 -0
  21. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +78 -0
  22. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +104 -0
  23. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiHomeWidgetProvider.kt +457 -0
  24. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiWidgetStore.kt +194 -0
  25. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/VoiceLaunchWidgetProvider.kt +67 -0
  26. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetConfigActivity.kt +228 -0
  27. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncScheduler.kt +72 -0
  28. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncWorker.kt +186 -0
  29. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetTaskRunWorker.kt +210 -0
  30. package/flutter_app/android/app/src/main/res/drawable/launch_background.xml +12 -0
  31. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_bg.xml +11 -0
  32. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_task_bg.xml +8 -0
  33. package/flutter_app/android/app/src/main/res/drawable-v21/launch_background.xml +12 -0
  34. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget.xml +138 -0
  35. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget_task_row.xml +52 -0
  36. package/flutter_app/android/app/src/main/res/layout/neoagent_voice_widget.xml +49 -0
  37. package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  38. package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  39. package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  40. package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  41. package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  42. package/flutter_app/android/app/src/main/res/values/strings.xml +12 -0
  43. package/flutter_app/android/app/src/main/res/values/styles.xml +18 -0
  44. package/flutter_app/android/app/src/main/res/values-night/styles.xml +18 -0
  45. package/flutter_app/android/app/src/main/res/xml/file_paths.xml +6 -0
  46. package/flutter_app/android/app/src/main/res/xml/neoagent_ai_widget_info.xml +12 -0
  47. package/flutter_app/android/app/src/main/res/xml/neoagent_voice_widget_info.xml +12 -0
  48. package/flutter_app/android/app/src/profile/AndroidManifest.xml +7 -0
  49. package/flutter_app/android/build.gradle.kts +24 -0
  50. package/flutter_app/android/ci-release.keystore +0 -0
  51. package/flutter_app/android/gradle/wrapper/gradle-wrapper.properties +5 -0
  52. package/flutter_app/android/gradle.properties +3 -0
  53. package/flutter_app/android/key.properties +4 -0
  54. package/flutter_app/android/settings.gradle.kts +26 -0
  55. package/flutter_app/assets/branding/app_icon_1024.png +0 -0
  56. package/flutter_app/assets/branding/app_icon_128.png +0 -0
  57. package/flutter_app/assets/branding/app_icon_192.png +0 -0
  58. package/flutter_app/assets/branding/app_icon_256.png +0 -0
  59. package/flutter_app/assets/branding/app_icon_32.png +0 -0
  60. package/flutter_app/assets/branding/app_icon_512.png +0 -0
  61. package/flutter_app/assets/branding/app_icon_64.png +0 -0
  62. package/flutter_app/assets/branding/tray_icon_template.png +0 -0
  63. package/flutter_app/lib/features/location/location_service.dart +119 -0
  64. package/flutter_app/lib/features/notifications/notification_interceptor.dart +97 -0
  65. package/flutter_app/lib/main.dart +23057 -0
  66. package/flutter_app/lib/main_app_shell.dart +1682 -0
  67. package/flutter_app/lib/main_integrations.dart +931 -0
  68. package/flutter_app/lib/main_launcher.dart +959 -0
  69. package/flutter_app/lib/main_launcher_entry.dart +5 -0
  70. package/flutter_app/lib/main_models.dart +3473 -0
  71. package/flutter_app/lib/main_shared.dart +2861 -0
  72. package/flutter_app/lib/main_theme.dart +204 -0
  73. package/flutter_app/lib/main_voice_assistant.dart +831 -0
  74. package/flutter_app/lib/src/android_apk_drop_zone.dart +32 -0
  75. package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +16 -0
  76. package/flutter_app/lib/src/android_apk_drop_zone_web.dart +348 -0
  77. package/flutter_app/lib/src/android_app_installer.dart +22 -0
  78. package/flutter_app/lib/src/android_app_installer_io.dart +122 -0
  79. package/flutter_app/lib/src/android_app_installer_stub.dart +21 -0
  80. package/flutter_app/lib/src/android_launcher_bridge.dart +239 -0
  81. package/flutter_app/lib/src/app_launch_bridge.dart +29 -0
  82. package/flutter_app/lib/src/app_release_updater.dart +511 -0
  83. package/flutter_app/lib/src/backend_client.dart +1833 -0
  84. package/flutter_app/lib/src/desktop_companion.dart +2 -0
  85. package/flutter_app/lib/src/desktop_companion_actions.dart +586 -0
  86. package/flutter_app/lib/src/desktop_companion_io.dart +538 -0
  87. package/flutter_app/lib/src/desktop_companion_stub.dart +59 -0
  88. package/flutter_app/lib/src/desktop_native_bridge.dart +91 -0
  89. package/flutter_app/lib/src/desktop_screen_capture.dart +21 -0
  90. package/flutter_app/lib/src/desktop_screen_capture_io.dart +142 -0
  91. package/flutter_app/lib/src/desktop_screen_capture_stub.dart +12 -0
  92. package/flutter_app/lib/src/diagnostics_logger.dart +119 -0
  93. package/flutter_app/lib/src/health_bridge.dart +136 -0
  94. package/flutter_app/lib/src/live_voice_capture.dart +85 -0
  95. package/flutter_app/lib/src/messaging_access_summary.dart +46 -0
  96. package/flutter_app/lib/src/network/app_http_client.dart +53 -0
  97. package/flutter_app/lib/src/network/app_http_client_factory.dart +6 -0
  98. package/flutter_app/lib/src/network/app_http_client_io.dart +138 -0
  99. package/flutter_app/lib/src/network/app_http_client_stub.dart +3 -0
  100. package/flutter_app/lib/src/network/app_http_client_web.dart +94 -0
  101. package/flutter_app/lib/src/oauth_launcher.dart +33 -0
  102. package/flutter_app/lib/src/oauth_launcher_io.dart +77 -0
  103. package/flutter_app/lib/src/oauth_launcher_stub.dart +33 -0
  104. package/flutter_app/lib/src/oauth_launcher_web.dart +107 -0
  105. package/flutter_app/lib/src/recording_bridge.dart +232 -0
  106. package/flutter_app/lib/src/recording_bridge_io.dart +1019 -0
  107. package/flutter_app/lib/src/recording_bridge_stub.dart +120 -0
  108. package/flutter_app/lib/src/recording_bridge_web.dart +689 -0
  109. package/flutter_app/lib/src/recording_payloads.dart +86 -0
  110. package/flutter_app/lib/src/theme/palette.dart +81 -0
  111. package/flutter_app/lib/src/widget_bridge.dart +49 -0
  112. package/flutter_app/linux/CMakeLists.txt +128 -0
  113. package/flutter_app/linux/flutter/CMakeLists.txt +88 -0
  114. package/flutter_app/linux/flutter/generated_plugin_registrant.cc +43 -0
  115. package/flutter_app/linux/flutter/generated_plugin_registrant.h +15 -0
  116. package/flutter_app/linux/flutter/generated_plugins.cmake +31 -0
  117. package/flutter_app/linux/runner/CMakeLists.txt +26 -0
  118. package/flutter_app/linux/runner/main.cc +6 -0
  119. package/flutter_app/linux/runner/my_application.cc +144 -0
  120. package/flutter_app/linux/runner/my_application.h +18 -0
  121. package/flutter_app/linux/runner/resources/app_icon.png +0 -0
  122. package/flutter_app/macos/Flutter/Flutter-Debug.xcconfig +2 -0
  123. package/flutter_app/macos/Flutter/Flutter-Release.xcconfig +2 -0
  124. package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +40 -0
  125. package/flutter_app/macos/Podfile +42 -0
  126. package/flutter_app/macos/Podfile.lock +87 -0
  127. package/flutter_app/macos/Runner/AppDelegate.swift +576 -0
  128. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json +68 -0
  129. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
  130. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
  131. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
  132. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
  133. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
  134. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
  135. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
  136. package/flutter_app/macos/Runner/Base.lproj/MainMenu.xib +342 -0
  137. package/flutter_app/macos/Runner/Configs/AppInfo.xcconfig +14 -0
  138. package/flutter_app/macos/Runner/Configs/Debug.xcconfig +2 -0
  139. package/flutter_app/macos/Runner/Configs/Release.xcconfig +2 -0
  140. package/flutter_app/macos/Runner/Configs/Warnings.xcconfig +13 -0
  141. package/flutter_app/macos/Runner/DebugProfile.entitlements +16 -0
  142. package/flutter_app/macos/Runner/Info.plist +36 -0
  143. package/flutter_app/macos/Runner/MainFlutterWindow.swift +19 -0
  144. package/flutter_app/macos/Runner/Release.entitlements +12 -0
  145. package/flutter_app/macos/Runner.xcodeproj/project.pbxproj +801 -0
  146. package/flutter_app/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  147. package/flutter_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +99 -0
  148. package/flutter_app/macos/Runner.xcworkspace/contents.xcworkspacedata +10 -0
  149. package/flutter_app/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  150. package/flutter_app/macos/RunnerTests/RunnerTests.swift +12 -0
  151. package/flutter_app/patch_strings.py +12 -0
  152. package/flutter_app/pubspec.lock +1088 -0
  153. package/flutter_app/pubspec.yaml +53 -0
  154. package/flutter_app/test/messaging_access_summary_test.dart +22 -0
  155. package/flutter_app/test/recording_payloads_test.dart +53 -0
  156. package/flutter_app/third_party/desktop_audio_capture/LICENSE +21 -0
  157. package/flutter_app/third_party/desktop_audio_capture/README.md +262 -0
  158. package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +65 -0
  159. package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +153 -0
  160. package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +110 -0
  161. package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +461 -0
  162. package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +91 -0
  163. package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +106 -0
  164. package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +219 -0
  165. package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +336 -0
  166. package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +101 -0
  167. package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +692 -0
  168. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +35 -0
  169. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +36 -0
  170. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +32 -0
  171. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +32 -0
  172. package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +878 -0
  173. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +27 -0
  174. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +1172 -0
  175. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +655 -0
  176. package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +12 -0
  177. package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +30 -0
  178. package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +87 -0
  179. package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +105 -0
  180. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +80 -0
  181. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +31 -0
  182. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +12 -0
  183. package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +23 -0
  184. package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +25 -0
  185. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +1117 -0
  186. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +115 -0
  187. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +777 -0
  188. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +87 -0
  189. package/flutter_app/third_party/flutter_secure_storage_linux/linux/CMakeLists.txt +30 -0
  190. package/flutter_app/third_party/flutter_secure_storage_linux/linux/flutter_secure_storage_linux_plugin.cc +215 -0
  191. package/flutter_app/third_party/flutter_secure_storage_linux/linux/include/flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h +27 -0
  192. package/flutter_app/third_party/flutter_secure_storage_linux/pubspec.yaml +20 -0
  193. package/flutter_app/tool/generate_desktop_branding.py +219 -0
  194. package/flutter_app/web/favicon.png +0 -0
  195. package/flutter_app/web/favicon.svg +12 -0
  196. package/flutter_app/web/icons/Icon-192.png +0 -0
  197. package/flutter_app/web/icons/Icon-512.png +0 -0
  198. package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
  199. package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
  200. package/flutter_app/web/index.html +39 -0
  201. package/flutter_app/web/manifest.json +35 -0
  202. package/flutter_app/windows/CMakeLists.txt +108 -0
  203. package/flutter_app/windows/flutter/CMakeLists.txt +109 -0
  204. package/flutter_app/windows/flutter/generated_plugin_registrant.cc +47 -0
  205. package/flutter_app/windows/flutter/generated_plugin_registrant.h +15 -0
  206. package/flutter_app/windows/flutter/generated_plugins.cmake +35 -0
  207. package/flutter_app/windows/runner/CMakeLists.txt +41 -0
  208. package/flutter_app/windows/runner/Runner.rc +121 -0
  209. package/flutter_app/windows/runner/flutter_window.cpp +533 -0
  210. package/flutter_app/windows/runner/flutter_window.h +37 -0
  211. package/flutter_app/windows/runner/main.cpp +53 -0
  212. package/flutter_app/windows/runner/resource.h +16 -0
  213. package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
  214. package/flutter_app/windows/runner/runner.exe.manifest +14 -0
  215. package/flutter_app/windows/runner/utils.cpp +65 -0
  216. package/flutter_app/windows/runner/utils.h +19 -0
  217. package/flutter_app/windows/runner/win32_window.cpp +299 -0
  218. package/flutter_app/windows/runner/win32_window.h +102 -0
  219. package/lib/manager.js +231 -7
  220. package/package.json +3 -1
  221. package/server/db/database.js +68 -0
  222. package/server/http/middleware.js +50 -0
  223. package/server/http/routes.js +3 -1
  224. package/server/index.js +1 -0
  225. package/server/public/.last_build_id +1 -1
  226. package/server/public/assets/NOTICES +61 -0
  227. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  228. package/server/public/flutter_bootstrap.js +1 -1
  229. package/server/public/main.dart.js +65262 -64422
  230. package/server/routes/integrations.js +86 -0
  231. package/server/routes/memory.js +11 -2
  232. package/server/routes/screenHistory.js +46 -0
  233. package/server/routes/triggers.js +81 -0
  234. package/server/services/ai/models.js +30 -0
  235. package/server/services/ai/providers/githubCopilot.js +97 -0
  236. package/server/services/ai/providers/openai.js +2 -1
  237. package/server/services/ai/providers/openaiCodex.js +31 -0
  238. package/server/services/ai/settings.js +20 -0
  239. package/server/services/ai/systemPrompt.js +1 -1
  240. package/server/services/ai/tools.js +35 -6
  241. package/server/services/browser/controller.js +47 -3
  242. package/server/services/desktop/screenRecorder.js +172 -0
  243. package/server/services/integrations/env.js +5 -0
  244. package/server/services/integrations/github/common.js +106 -0
  245. package/server/services/integrations/github/provider.js +499 -0
  246. package/server/services/integrations/github/repos.js +1124 -0
  247. package/server/services/integrations/home_assistant/provider.js +306 -26
  248. package/server/services/integrations/manager.js +63 -7
  249. package/server/services/integrations/oauth_provider.js +13 -6
  250. package/server/services/integrations/provider_config_store.js +76 -0
  251. package/server/services/integrations/registry.js +4 -0
  252. package/server/services/integrations/trello/provider.js +744 -0
  253. package/server/services/integrations/whatsapp/provider.js +6 -2
  254. package/server/services/manager.js +22 -0
  255. package/server/services/memory/manager.js +39 -2
  256. package/server/services/skills/base_catalog.js +33 -0
  257. package/server/services/tasks/adapters/index.js +1 -0
  258. package/server/services/tasks/adapters/manual.js +12 -0
  259. package/server/services/tasks/runtime.js +1 -1
  260. package/server/services/voice/openaiClient.js +4 -1
  261. package/server/services/voice/providers.js +2 -1
  262. package/server/services/widgets/service.js +49 -4
  263. package/server/utils/local_secrets.js +56 -0
  264. package/server/utils/logger.js +37 -9
@@ -0,0 +1,777 @@
1
+ #include "system_audio_capture_plugin.h"
2
+
3
+ #define NOMINMAX
4
+ #include <windows.h>
5
+ #undef max
6
+ #undef min
7
+ #include <mmdeviceapi.h>
8
+ #include <audioclient.h>
9
+ #include <functiondiscoverykeys_devpkey.h>
10
+ #include <VersionHelpers.h>
11
+ #include <comdef.h>
12
+
13
+ #include <flutter/event_channel.h>
14
+ #include <flutter/method_channel.h>
15
+ #include <flutter/plugin_registrar_windows.h>
16
+ #include <flutter/standard_method_codec.h>
17
+
18
+ #include <algorithm>
19
+ #include <chrono>
20
+ #include <cmath>
21
+ #include <cstdint>
22
+ #include <functional>
23
+ #include <memory>
24
+ #include <sstream>
25
+ #include <thread>
26
+ #include <vector>
27
+
28
+ #pragma comment(lib, "ole32.lib")
29
+ #pragma comment(lib, "oleaut32.lib")
30
+
31
+ // REFTIMES_PER_SEC is 10,000,000 (100 nanoseconds per second)
32
+ #ifndef REFTIMES_PER_SEC
33
+ #define REFTIMES_PER_SEC 10000000
34
+ #endif
35
+
36
+ namespace audio_capture {
37
+
38
+ // Custom StreamHandler implementation
39
+ template <typename T>
40
+ class StreamHandlerFunctions : public flutter::StreamHandler<T> {
41
+ public:
42
+ using OnListenHandler = std::function<std::unique_ptr<flutter::StreamHandlerError<T>>(
43
+ const T* arguments,
44
+ std::unique_ptr<flutter::EventSink<T>>&& events)>;
45
+ using OnCancelHandler = std::function<std::unique_ptr<flutter::StreamHandlerError<T>>(
46
+ const T* arguments)>;
47
+
48
+ StreamHandlerFunctions(OnListenHandler on_listen, OnCancelHandler on_cancel)
49
+ : on_listen_(std::move(on_listen)), on_cancel_(std::move(on_cancel)) {}
50
+
51
+ virtual ~StreamHandlerFunctions() = default;
52
+
53
+ protected:
54
+ std::unique_ptr<flutter::StreamHandlerError<T>> OnListenInternal(
55
+ const T* arguments,
56
+ std::unique_ptr<flutter::EventSink<T>>&& events) override {
57
+ return on_listen_(arguments, std::move(events));
58
+ }
59
+
60
+ std::unique_ptr<flutter::StreamHandlerError<T>> OnCancelInternal(
61
+ const T* arguments) override {
62
+ return on_cancel_(arguments);
63
+ }
64
+
65
+ private:
66
+ OnListenHandler on_listen_;
67
+ OnCancelHandler on_cancel_;
68
+ };
69
+
70
+ namespace {
71
+
72
+ constexpr char kMethodChannelName[] = "com.system_audio_transcriber/audio_capture";
73
+ constexpr char kEventChannelName[] = "com.system_audio_transcriber/audio_stream";
74
+ constexpr char kStatusEventChannelName[] = "com.system_audio_transcriber/audio_status";
75
+ constexpr char kDecibelEventChannelName[] = "com.system_audio_transcriber/audio_decibel";
76
+
77
+ constexpr int kDefaultSampleRate = 16000;
78
+ constexpr int kDefaultChannels = 1;
79
+ constexpr int kDefaultBitsPerSample = 16;
80
+ constexpr int kDefaultChunkDurationMs = 1000;
81
+ constexpr float kDefaultGainBoost = 2.5f;
82
+ constexpr float kDefaultInputVolume = 1.0f;
83
+
84
+ } // namespace
85
+
86
+ // static
87
+ void SystemAudioCapturePlugin::RegisterWithRegistrar(
88
+ flutter::PluginRegistrarWindows *registrar) {
89
+ auto plugin = std::make_unique<SystemAudioCapturePlugin>(registrar);
90
+ registrar->AddPlugin(std::move(plugin));
91
+ }
92
+
93
+ SystemAudioCapturePlugin::SystemAudioCapturePlugin(
94
+ flutter::PluginRegistrarWindows *registrar)
95
+ : registrar_(registrar),
96
+ is_capturing_(false),
97
+ should_stop_(false),
98
+ sample_rate_(kDefaultSampleRate),
99
+ channels_(kDefaultChannels),
100
+ bits_per_sample_(kDefaultBitsPerSample),
101
+ chunk_duration_ms_(kDefaultChunkDurationMs),
102
+ gain_boost_(kDefaultGainBoost),
103
+ input_volume_(kDefaultInputVolume),
104
+ audio_client_(nullptr),
105
+ capture_client_(nullptr),
106
+ device_(nullptr),
107
+ mix_format_(nullptr),
108
+ buffer_frame_count_(0),
109
+ com_initialized_(false) {
110
+ // Create method channel
111
+ method_channel_ =
112
+ std::make_unique<flutter::MethodChannel<flutter::EncodableValue>>(
113
+ registrar->messenger(), kMethodChannelName,
114
+ &flutter::StandardMethodCodec::GetInstance());
115
+
116
+ method_channel_->SetMethodCallHandler(
117
+ [this](const auto &call, auto result) {
118
+ this->HandleMethodCall(call, std::move(result));
119
+ });
120
+
121
+ // Create event channels
122
+ event_channel_ =
123
+ std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
124
+ registrar->messenger(), kEventChannelName,
125
+ &flutter::StandardMethodCodec::GetInstance());
126
+
127
+ event_channel_->SetStreamHandler(
128
+ std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
129
+ [this](const flutter::EncodableValue* arguments,
130
+ std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
131
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
132
+ std::lock_guard<std::mutex> lock(mutex_);
133
+ event_sink_ = std::move(events);
134
+ return nullptr;
135
+ },
136
+ [this](const flutter::EncodableValue* arguments)
137
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
138
+ std::lock_guard<std::mutex> lock(mutex_);
139
+ event_sink_.reset();
140
+ return nullptr;
141
+ }));
142
+
143
+ status_event_channel_ =
144
+ std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
145
+ registrar->messenger(), kStatusEventChannelName,
146
+ &flutter::StandardMethodCodec::GetInstance());
147
+
148
+ status_event_channel_->SetStreamHandler(
149
+ std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
150
+ [this](const flutter::EncodableValue* arguments,
151
+ std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
152
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
153
+ std::lock_guard<std::mutex> lock(mutex_);
154
+ status_event_sink_ = std::move(events);
155
+ SendStatusUpdate(is_capturing_);
156
+ return nullptr;
157
+ },
158
+ [this](const flutter::EncodableValue* arguments)
159
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
160
+ std::lock_guard<std::mutex> lock(mutex_);
161
+ status_event_sink_.reset();
162
+ return nullptr;
163
+ }));
164
+
165
+ decibel_event_channel_ =
166
+ std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
167
+ registrar->messenger(), kDecibelEventChannelName,
168
+ &flutter::StandardMethodCodec::GetInstance());
169
+
170
+ decibel_event_channel_->SetStreamHandler(
171
+ std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
172
+ [this](const flutter::EncodableValue* arguments,
173
+ std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
174
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
175
+ std::lock_guard<std::mutex> lock(mutex_);
176
+ decibel_event_sink_ = std::move(events);
177
+ return nullptr;
178
+ },
179
+ [this](const flutter::EncodableValue* arguments)
180
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
181
+ std::lock_guard<std::mutex> lock(mutex_);
182
+ decibel_event_sink_.reset();
183
+ return nullptr;
184
+ }));
185
+ }
186
+
187
+ SystemAudioCapturePlugin::~SystemAudioCapturePlugin() {
188
+ StopCapture();
189
+ }
190
+
191
+ void SystemAudioCapturePlugin::HandleMethodCall(
192
+ const flutter::MethodCall<flutter::EncodableValue> &method_call,
193
+ std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result) {
194
+ if (method_call.method_name().compare("requestPermissions") == 0) {
195
+ // On Windows, permissions are typically handled by the system
196
+ result->Success(flutter::EncodableValue(true));
197
+ } else if (method_call.method_name().compare("startCapture") == 0) {
198
+ const flutter::EncodableMap* args = nullptr;
199
+ if (method_call.arguments() &&
200
+ std::holds_alternative<flutter::EncodableMap>(*method_call.arguments())) {
201
+ args = &std::get<flutter::EncodableMap>(*method_call.arguments());
202
+ }
203
+ bool started = StartCapture(args);
204
+ result->Success(flutter::EncodableValue(started));
205
+ } else if (method_call.method_name().compare("stopCapture") == 0) {
206
+ bool stopped = StopCapture();
207
+ result->Success(flutter::EncodableValue(stopped));
208
+ } else {
209
+ result->NotImplemented();
210
+ }
211
+ }
212
+
213
+ void SystemAudioCapturePlugin::ApplyGainBoostAndConvertToMono(
214
+ const int16_t* input, int16_t* output, size_t frame_count,
215
+ int input_channels, float gain_boost) {
216
+ const float max_value = 32767.0f;
217
+ const float min_value = -32768.0f;
218
+
219
+ if (input_channels == 1) {
220
+ // Mono: just apply gain boost
221
+ for (size_t i = 0; i < frame_count; ++i) {
222
+ float sample = static_cast<float>(input[i]) * gain_boost;
223
+ float clamped_sample = (std::min)(max_value, sample);
224
+ sample = (std::max)(min_value, clamped_sample);
225
+ output[i] = static_cast<int16_t>(sample);
226
+ }
227
+ } else {
228
+ // Stereo: convert to mono and apply gain boost
229
+ for (size_t i = 0; i < frame_count; ++i) {
230
+ float left = static_cast<float>(input[i * 2]);
231
+ float right = static_cast<float>(input[i * 2 + 1]);
232
+ float mono = (left + right) / 2.0f * gain_boost;
233
+ float clamped_mono = (std::min)(max_value, mono);
234
+ mono = (std::max)(min_value, clamped_mono);
235
+ output[i] = static_cast<int16_t>(mono);
236
+ }
237
+ }
238
+ }
239
+
240
+ double SystemAudioCapturePlugin::CalculateDecibel(const int16_t* samples,
241
+ size_t sample_count) {
242
+ if (sample_count == 0) {
243
+ return -120.0;
244
+ }
245
+
246
+ // Calculate RMS (Root Mean Square)
247
+ double sum_of_squares = 0.0;
248
+ for (size_t i = 0; i < sample_count; ++i) {
249
+ double value = static_cast<double>(samples[i]);
250
+ sum_of_squares += value * value;
251
+ }
252
+ double mean_square = sum_of_squares / static_cast<double>(sample_count);
253
+ double rms = sqrt(mean_square);
254
+
255
+ // Calculate decibel: dB = 20 * log10(RMS / max_value)
256
+ const double max_value = 32767.0;
257
+ if (rms <= 0.0) {
258
+ return -120.0; // Avoid log(0)
259
+ }
260
+
261
+ double decibel = 20.0 * log10(rms / max_value);
262
+
263
+ // Clamp to reasonable range (-120 dB to 0 dB)
264
+ double clamped_decibel = (std::min)(0.0, decibel);
265
+ return (std::max)(-120.0, clamped_decibel);
266
+ }
267
+
268
+ void SystemAudioCapturePlugin::SendStatusUpdate(bool is_active) {
269
+ try {
270
+ std::lock_guard<std::mutex> lock(mutex_);
271
+ if (status_event_sink_) {
272
+ flutter::EncodableMap status_map;
273
+ status_map[flutter::EncodableValue("isActive")] = flutter::EncodableValue(is_active);
274
+
275
+ // Get current timestamp in seconds
276
+ auto now = std::chrono::system_clock::now();
277
+ auto duration = now.time_since_epoch();
278
+ auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
279
+ double timestamp = static_cast<double>(milliseconds) / 1000.0;
280
+
281
+ status_map[flutter::EncodableValue("timestamp")] = flutter::EncodableValue(timestamp);
282
+ status_event_sink_->Success(flutter::EncodableValue(status_map));
283
+ }
284
+ } catch (...) {
285
+ // Silently ignore errors to prevent crash
286
+ }
287
+ }
288
+
289
+ void SystemAudioCapturePlugin::SendDecibelUpdate(double decibel) {
290
+ try {
291
+ std::lock_guard<std::mutex> lock(mutex_);
292
+ if (decibel_event_sink_) {
293
+ flutter::EncodableMap decibel_map;
294
+ decibel_map[flutter::EncodableValue("decibel")] = flutter::EncodableValue(decibel);
295
+
296
+ // Get current timestamp in seconds
297
+ auto now = std::chrono::system_clock::now();
298
+ auto duration = now.time_since_epoch();
299
+ auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
300
+ double timestamp = static_cast<double>(milliseconds) / 1000.0;
301
+
302
+ decibel_map[flutter::EncodableValue("timestamp")] = flutter::EncodableValue(timestamp);
303
+ decibel_event_sink_->Success(flutter::EncodableValue(decibel_map));
304
+ }
305
+ } catch (...) {
306
+ // Silently ignore errors to prevent crash
307
+ }
308
+ }
309
+
310
+ bool SystemAudioCapturePlugin::StartCapture(const flutter::EncodableMap* args) {
311
+ // Always cleanup any existing capture first
312
+ {
313
+ std::lock_guard<std::mutex> lock(mutex_);
314
+ if (is_capturing_) {
315
+ should_stop_ = true;
316
+ }
317
+ }
318
+
319
+ // Wait for existing capture thread to finish
320
+ if (capture_thread_.joinable()) {
321
+ capture_thread_.join();
322
+ }
323
+
324
+ // Cleanup any remaining resources (without lock to avoid deadlock)
325
+ if (audio_client_) {
326
+ audio_client_->Stop();
327
+ audio_client_->Release();
328
+ audio_client_ = nullptr;
329
+ }
330
+ if (capture_client_) {
331
+ capture_client_->Release();
332
+ capture_client_ = nullptr;
333
+ }
334
+ if (mix_format_) {
335
+ CoTaskMemFree(mix_format_);
336
+ mix_format_ = nullptr;
337
+ }
338
+ if (device_) {
339
+ device_->Release();
340
+ device_ = nullptr;
341
+ }
342
+ if (com_initialized_) {
343
+ CoUninitialize();
344
+ com_initialized_ = false;
345
+ }
346
+
347
+ // Parse arguments
348
+ if (args) {
349
+ auto it = args->find(flutter::EncodableValue("sampleRate"));
350
+ if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
351
+ sample_rate_ = std::get<int32_t>(it->second);
352
+ }
353
+
354
+ it = args->find(flutter::EncodableValue("channels"));
355
+ if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
356
+ channels_ = std::get<int32_t>(it->second);
357
+ }
358
+
359
+ it = args->find(flutter::EncodableValue("bitsPerSample"));
360
+ if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
361
+ bits_per_sample_ = std::get<int32_t>(it->second);
362
+ }
363
+
364
+ it = args->find(flutter::EncodableValue("chunkDurationMs"));
365
+ if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
366
+ chunk_duration_ms_ = std::get<int32_t>(it->second);
367
+ }
368
+
369
+ it = args->find(flutter::EncodableValue("gainBoost"));
370
+ if (it != args->end() && std::holds_alternative<double>(it->second)) {
371
+ gain_boost_ = static_cast<float>(std::get<double>(it->second));
372
+ }
373
+
374
+ it = args->find(flutter::EncodableValue("inputVolume"));
375
+ if (it != args->end() && std::holds_alternative<double>(it->second)) {
376
+ input_volume_ = static_cast<float>(std::get<double>(it->second));
377
+ }
378
+ }
379
+
380
+ // Clamp values
381
+ sample_rate_ = (std::max)(sample_rate_, 8000);
382
+ int min_channels = (std::min)(channels_, 2);
383
+ channels_ = (std::max)(1, min_channels);
384
+ bits_per_sample_ = 16;
385
+ chunk_duration_ms_ = (std::max)(chunk_duration_ms_, 10);
386
+ float min_gain = (std::min)(10.0f, gain_boost_);
387
+ gain_boost_ = (std::max)(0.1f, min_gain);
388
+ float min_volume = (std::min)(1.0f, input_volume_);
389
+ input_volume_ = (std::max)(0.0f, min_volume);
390
+
391
+ // Initialize COM (allow RPC_E_CHANGED_MODE if already initialized)
392
+ HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
393
+ if (FAILED(hr) && hr != RPC_E_CHANGED_MODE) {
394
+ return false;
395
+ }
396
+ com_initialized_ = (hr == S_OK); // Only true if we initialized it
397
+
398
+ // Get default audio endpoint (eRender for loopback)
399
+ IMMDeviceEnumerator* enumerator = nullptr;
400
+ hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL,
401
+ __uuidof(IMMDeviceEnumerator),
402
+ reinterpret_cast<void**>(&enumerator));
403
+ if (FAILED(hr)) {
404
+ if (com_initialized_) {
405
+ CoUninitialize();
406
+ }
407
+ return false;
408
+ }
409
+
410
+ hr = enumerator->GetDefaultAudioEndpoint(eRender, eConsole, &device_);
411
+ enumerator->Release();
412
+ if (FAILED(hr)) {
413
+ if (com_initialized_) {
414
+ CoUninitialize();
415
+ }
416
+ return false;
417
+ }
418
+
419
+ // Activate IAudioClient
420
+ hr = device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr,
421
+ reinterpret_cast<void**>(&audio_client_));
422
+ if (FAILED(hr)) {
423
+ device_->Release();
424
+ device_ = nullptr;
425
+ if (com_initialized_) {
426
+ CoUninitialize();
427
+ }
428
+ return false;
429
+ }
430
+
431
+ // Get mix format - use original format for initialization
432
+ WAVEFORMATEX* device_format = nullptr;
433
+ hr = audio_client_->GetMixFormat(&device_format);
434
+ if (FAILED(hr)) {
435
+ audio_client_->Release();
436
+ audio_client_ = nullptr;
437
+ device_->Release();
438
+ device_ = nullptr;
439
+ if (com_initialized_) {
440
+ CoUninitialize();
441
+ }
442
+ return false;
443
+ }
444
+
445
+ // Store device format for use in capture thread
446
+ // We'll use the device's native format and convert in the thread
447
+ mix_format_ = device_format;
448
+
449
+ // Initialize audio client for loopback with device's native format
450
+ // FIX LATENCY: Giảm buffer duration xuống 100ms để giảm latency
451
+ REFERENCE_TIME hnsRequestedDuration = REFTIMES_PER_SEC / 10; // 100ms instead of 1 second
452
+ hr = audio_client_->Initialize(AUDCLNT_SHAREMODE_SHARED,
453
+ AUDCLNT_STREAMFLAGS_LOOPBACK,
454
+ hnsRequestedDuration, 0, mix_format_, nullptr);
455
+ if (FAILED(hr)) {
456
+ CoTaskMemFree(mix_format_);
457
+ mix_format_ = nullptr;
458
+ audio_client_->Release();
459
+ audio_client_ = nullptr;
460
+ device_->Release();
461
+ device_ = nullptr;
462
+ if (com_initialized_) {
463
+ CoUninitialize();
464
+ }
465
+ return false;
466
+ }
467
+
468
+ // Get buffer size
469
+ hr = audio_client_->GetBufferSize(&buffer_frame_count_);
470
+ if (FAILED(hr)) {
471
+ CoTaskMemFree(mix_format_);
472
+ mix_format_ = nullptr;
473
+ audio_client_->Release();
474
+ audio_client_ = nullptr;
475
+ device_->Release();
476
+ device_ = nullptr;
477
+ if (com_initialized_) {
478
+ CoUninitialize();
479
+ }
480
+ return false;
481
+ }
482
+
483
+ // Get IAudioCaptureClient
484
+ hr = audio_client_->GetService(__uuidof(IAudioCaptureClient),
485
+ reinterpret_cast<void**>(&capture_client_));
486
+ if (FAILED(hr)) {
487
+ CoTaskMemFree(mix_format_);
488
+ mix_format_ = nullptr;
489
+ audio_client_->Release();
490
+ audio_client_ = nullptr;
491
+ device_->Release();
492
+ device_ = nullptr;
493
+ if (com_initialized_) {
494
+ CoUninitialize();
495
+ }
496
+ return false;
497
+ }
498
+
499
+ // Start capture
500
+ hr = audio_client_->Start();
501
+ if (FAILED(hr)) {
502
+ capture_client_->Release();
503
+ capture_client_ = nullptr;
504
+ CoTaskMemFree(mix_format_);
505
+ mix_format_ = nullptr;
506
+ audio_client_->Release();
507
+ audio_client_ = nullptr;
508
+ device_->Release();
509
+ device_ = nullptr;
510
+ if (com_initialized_) {
511
+ CoUninitialize();
512
+ }
513
+ return false;
514
+ }
515
+
516
+ {
517
+ std::lock_guard<std::mutex> lock(mutex_);
518
+ should_stop_ = false;
519
+ is_capturing_ = true;
520
+ }
521
+
522
+ // Start capture thread
523
+ capture_thread_ = std::thread(&SystemAudioCapturePlugin::CaptureThread, this);
524
+
525
+ // Wait a bit to ensure thread has started
526
+ std::this_thread::sleep_for(std::chrono::milliseconds(200));
527
+
528
+ SendStatusUpdate(true);
529
+
530
+ return true;
531
+ }
532
+
533
+ bool SystemAudioCapturePlugin::StopCapture() {
534
+ {
535
+ std::lock_guard<std::mutex> lock(mutex_);
536
+ if (!is_capturing_) {
537
+ return false;
538
+ }
539
+ should_stop_ = true;
540
+ }
541
+
542
+ if (capture_thread_.joinable()) {
543
+ capture_thread_.join();
544
+ }
545
+
546
+ {
547
+ std::lock_guard<std::mutex> lock(mutex_);
548
+ is_capturing_ = false;
549
+
550
+ // Cleanup WASAPI resources
551
+ if (audio_client_) {
552
+ audio_client_->Stop();
553
+ }
554
+
555
+ if (capture_client_) {
556
+ capture_client_->Release();
557
+ capture_client_ = nullptr;
558
+ }
559
+
560
+ if (mix_format_) {
561
+ CoTaskMemFree(mix_format_);
562
+ mix_format_ = nullptr;
563
+ }
564
+
565
+ if (audio_client_) {
566
+ audio_client_->Release();
567
+ audio_client_ = nullptr;
568
+ }
569
+
570
+ if (device_) {
571
+ device_->Release();
572
+ device_ = nullptr;
573
+ }
574
+
575
+ // Only uninitialize COM if we initialized it
576
+ if (com_initialized_) {
577
+ CoUninitialize();
578
+ com_initialized_ = false;
579
+ }
580
+ }
581
+
582
+ SendStatusUpdate(false);
583
+
584
+ return true;
585
+ }
586
+
587
+ void SystemAudioCapturePlugin::CaptureThread() {
588
+ try {
589
+ // Set thread priority to reduce latency
590
+ SetThreadPriority();
591
+
592
+ if (!mix_format_ || !capture_client_) {
593
+ return;
594
+ }
595
+
596
+ // Use actual format from device
597
+ const UINT32 frame_size = mix_format_->nBlockAlign;
598
+ const UINT32 actual_sample_rate = mix_format_->nSamplesPerSec;
599
+ const WORD actual_channels = mix_format_->nChannels;
600
+ const WORD actual_bits_per_sample = mix_format_->wBitsPerSample;
601
+
602
+ // FIX LATENCY 1: Giảm chunk size xuống tối đa 50ms để giảm delay
603
+ // Sử dụng chunk nhỏ hơn để gửi data nhanh hơn
604
+ const int effective_chunk_ms = (std::max)(20, (std::min)(chunk_duration_ms_, 50));
605
+
606
+ // Calculate smaller chunk size for lower latency
607
+ const size_t chunk_frames = (actual_sample_rate * effective_chunk_ms / 1000);
608
+ const size_t chunk_size_bytes = chunk_frames * frame_size;
609
+ const size_t output_frame_count = (sample_rate_ * effective_chunk_ms / 1000);
610
+
611
+ std::vector<uint8_t> raw_buffer(chunk_size_bytes * 2); // Double buffer for safety
612
+ std::vector<int16_t> output_buffer(output_frame_count);
613
+ size_t raw_buffer_pos = 0;
614
+
615
+ // FIX LATENCY 2: Giảm sleep time xuống tối thiểu để giảm delay
616
+ const int sleep_time_ms = 1; // Giảm xuống 1ms để phản hồi nhanh hơn
617
+
618
+ while (!should_stop_) {
619
+ UINT32 num_frames_available = 0;
620
+ HRESULT hr = capture_client_->GetNextPacketSize(&num_frames_available);
621
+
622
+ if (FAILED(hr)) {
623
+ break;
624
+ }
625
+
626
+ // FIX LATENCY 3: Process ngay khi có data, không đợi nhiều packets
627
+ if (num_frames_available == 0) {
628
+ std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time_ms));
629
+ continue;
630
+ }
631
+
632
+ while (num_frames_available > 0 && !should_stop_) {
633
+ BYTE* data = nullptr;
634
+ UINT32 num_frames = 0;
635
+ DWORD flags = 0;
636
+ UINT64 device_position = 0;
637
+ UINT64 qpc_position = 0;
638
+
639
+ hr = capture_client_->GetBuffer(&data, &num_frames, &flags,
640
+ &device_position, &qpc_position);
641
+
642
+ if (FAILED(hr)) {
643
+ break;
644
+ }
645
+
646
+ bool is_silent = (flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0;
647
+
648
+ if (!is_silent && data != nullptr && num_frames > 0) {
649
+ const size_t data_size = num_frames * frame_size;
650
+ size_t data_offset = 0;
651
+
652
+ while (data_offset < data_size && !should_stop_) {
653
+ const size_t space_available = raw_buffer.size() - raw_buffer_pos;
654
+ const size_t data_remaining = data_size - data_offset;
655
+ const size_t copy_size = (std::min)(space_available, data_remaining);
656
+
657
+ if (copy_size > 0) {
658
+ memcpy(raw_buffer.data() + raw_buffer_pos,
659
+ reinterpret_cast<const uint8_t*>(data) + data_offset,
660
+ copy_size);
661
+ raw_buffer_pos += copy_size;
662
+ data_offset += copy_size;
663
+ }
664
+
665
+ // FIX LATENCY 4: Gửi data ngay khi đủ chunk nhỏ, không đợi buffer đầy
666
+ if (raw_buffer_pos >= chunk_size_bytes) {
667
+ const size_t input_frame_count = chunk_size_bytes / frame_size;
668
+ const size_t total_samples = input_frame_count * actual_channels;
669
+
670
+ std::vector<int16_t> converted_samples(total_samples);
671
+ bool conversion_success = false;
672
+
673
+ // Convert based on format
674
+ if (actual_bits_per_sample == 16 &&
675
+ (mix_format_->wFormatTag == WAVE_FORMAT_PCM ||
676
+ mix_format_->wFormatTag == WAVE_FORMAT_EXTENSIBLE)) {
677
+ const int16_t* raw_samples = reinterpret_cast<const int16_t*>(raw_buffer.data());
678
+ converted_samples.assign(raw_samples, raw_samples + total_samples);
679
+ conversion_success = true;
680
+ } else if (actual_bits_per_sample == 32 &&
681
+ (mix_format_->wFormatTag == WAVE_FORMAT_IEEE_FLOAT ||
682
+ mix_format_->wFormatTag == WAVE_FORMAT_EXTENSIBLE)) {
683
+ const float* float_samples = reinterpret_cast<const float*>(raw_buffer.data());
684
+ for (size_t i = 0; i < total_samples; ++i) {
685
+ float sample = (std::min)(1.0f, (std::max)(-1.0f, float_samples[i]));
686
+ converted_samples[i] = static_cast<int16_t>(sample * 32767.0f);
687
+ }
688
+ conversion_success = true;
689
+ } else if (actual_bits_per_sample == 24) {
690
+ const uint8_t* raw_bytes = raw_buffer.data();
691
+ for (size_t i = 0; i < total_samples; ++i) {
692
+ size_t byte_offset = i * 3;
693
+ int32_t sample24 = (static_cast<int32_t>(raw_bytes[byte_offset]) |
694
+ (static_cast<int32_t>(raw_bytes[byte_offset + 1]) << 8) |
695
+ (static_cast<int32_t>(raw_bytes[byte_offset + 2]) << 16));
696
+ if (sample24 & 0x800000) {
697
+ sample24 |= 0xFF000000;
698
+ }
699
+ converted_samples[i] = static_cast<int16_t>(sample24 >> 8);
700
+ }
701
+ conversion_success = true;
702
+ }
703
+
704
+ if (!conversion_success) {
705
+ raw_buffer_pos = 0;
706
+ continue;
707
+ }
708
+
709
+ // Apply input volume if needed
710
+ if (input_volume_ > 0.0f && input_volume_ < 1.0f) {
711
+ for (size_t i = 0; i < total_samples; ++i) {
712
+ converted_samples[i] = static_cast<int16_t>(
713
+ static_cast<float>(converted_samples[i]) * input_volume_);
714
+ }
715
+ }
716
+
717
+ const size_t frames_to_process = converted_samples.size() / actual_channels;
718
+ const size_t output_frames = (std::min)(frames_to_process, output_frame_count);
719
+
720
+ ApplyGainBoostAndConvertToMono(converted_samples.data(), output_buffer.data(),
721
+ output_frames, actual_channels, gain_boost_);
722
+
723
+ double decibel = CalculateDecibel(output_buffer.data(), output_frames);
724
+
725
+ // FIX LATENCY 5: Tối ưu mutex - giữ lock thời gian ngắn nhất
726
+ {
727
+ std::lock_guard<std::mutex> lock(mutex_);
728
+ if (event_sink_) {
729
+ const size_t output_bytes = output_frames * sizeof(int16_t);
730
+ std::vector<uint8_t> audio_data(
731
+ reinterpret_cast<uint8_t*>(output_buffer.data()),
732
+ reinterpret_cast<uint8_t*>(output_buffer.data()) + output_bytes);
733
+ event_sink_->Success(flutter::EncodableValue(audio_data));
734
+ }
735
+ }
736
+
737
+ SendDecibelUpdate(decibel);
738
+
739
+ // Move remaining data
740
+ if (raw_buffer_pos > chunk_size_bytes) {
741
+ const size_t remaining = raw_buffer_pos - chunk_size_bytes;
742
+ memmove(raw_buffer.data(), raw_buffer.data() + chunk_size_bytes, remaining);
743
+ raw_buffer_pos = remaining;
744
+ } else {
745
+ raw_buffer_pos = 0;
746
+ }
747
+ }
748
+ }
749
+ }
750
+
751
+ hr = capture_client_->ReleaseBuffer(num_frames);
752
+ if (FAILED(hr)) {
753
+ break;
754
+ }
755
+
756
+ hr = capture_client_->GetNextPacketSize(&num_frames_available);
757
+ if (FAILED(hr)) {
758
+ break;
759
+ }
760
+ }
761
+ }
762
+ } catch (...) {
763
+ std::lock_guard<std::mutex> lock(mutex_);
764
+ is_capturing_ = false;
765
+ }
766
+ }
767
+
768
+ // BONUS: Thêm method để set high priority cho thread để giảm latency
769
+ // Gọi ở đầu CaptureThread() để giảm latency hơn nữa
770
+ void SystemAudioCapturePlugin::SetThreadPriority() {
771
+ HANDLE current_thread = GetCurrentThread();
772
+ // Sử dụng THREAD_PRIORITY_HIGHEST thay vì TIME_CRITICAL để tránh gây vấn đề
773
+ ::SetThreadPriority(current_thread, THREAD_PRIORITY_HIGHEST);
774
+ }
775
+
776
+ } // namespace audio_capture
777
+