neoagent 2.3.1-beta.2 → 2.3.1-beta.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. package/.env.example +39 -0
  2. package/README.md +2 -0
  3. package/docs/capabilities.md +2 -2
  4. package/docs/configuration.md +13 -5
  5. package/docs/integrations.md +4 -1
  6. package/flutter_app/.metadata +42 -0
  7. package/flutter_app/README.md +21 -0
  8. package/flutter_app/analysis_options.yaml +32 -0
  9. package/flutter_app/android/app/build.gradle.kts +109 -0
  10. package/flutter_app/android/app/src/debug/AndroidManifest.xml +7 -0
  11. package/flutter_app/android/app/src/main/AndroidManifest.xml +147 -0
  12. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +747 -0
  13. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthConnectGateway.kt +280 -0
  14. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncNotifications.kt +113 -0
  15. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncPayload.kt +57 -0
  16. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncScheduler.kt +78 -0
  17. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncWorker.kt +253 -0
  18. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/PermissionsRationaleActivity.kt +46 -0
  19. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +21 -0
  20. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +586 -0
  21. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +78 -0
  22. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +104 -0
  23. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiHomeWidgetProvider.kt +457 -0
  24. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiWidgetStore.kt +194 -0
  25. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/VoiceLaunchWidgetProvider.kt +67 -0
  26. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetConfigActivity.kt +228 -0
  27. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncScheduler.kt +72 -0
  28. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncWorker.kt +186 -0
  29. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetTaskRunWorker.kt +210 -0
  30. package/flutter_app/android/app/src/main/res/drawable/launch_background.xml +12 -0
  31. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_bg.xml +11 -0
  32. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_task_bg.xml +8 -0
  33. package/flutter_app/android/app/src/main/res/drawable-v21/launch_background.xml +12 -0
  34. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget.xml +138 -0
  35. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget_task_row.xml +52 -0
  36. package/flutter_app/android/app/src/main/res/layout/neoagent_voice_widget.xml +49 -0
  37. package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  38. package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  39. package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  40. package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  41. package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  42. package/flutter_app/android/app/src/main/res/values/strings.xml +12 -0
  43. package/flutter_app/android/app/src/main/res/values/styles.xml +18 -0
  44. package/flutter_app/android/app/src/main/res/values-night/styles.xml +18 -0
  45. package/flutter_app/android/app/src/main/res/xml/file_paths.xml +6 -0
  46. package/flutter_app/android/app/src/main/res/xml/neoagent_ai_widget_info.xml +12 -0
  47. package/flutter_app/android/app/src/main/res/xml/neoagent_voice_widget_info.xml +12 -0
  48. package/flutter_app/android/app/src/profile/AndroidManifest.xml +7 -0
  49. package/flutter_app/android/build.gradle.kts +24 -0
  50. package/flutter_app/android/ci-release.keystore +0 -0
  51. package/flutter_app/android/gradle/wrapper/gradle-wrapper.properties +5 -0
  52. package/flutter_app/android/gradle.properties +3 -0
  53. package/flutter_app/android/key.properties +4 -0
  54. package/flutter_app/android/settings.gradle.kts +26 -0
  55. package/flutter_app/assets/branding/app_icon_1024.png +0 -0
  56. package/flutter_app/assets/branding/app_icon_128.png +0 -0
  57. package/flutter_app/assets/branding/app_icon_192.png +0 -0
  58. package/flutter_app/assets/branding/app_icon_256.png +0 -0
  59. package/flutter_app/assets/branding/app_icon_32.png +0 -0
  60. package/flutter_app/assets/branding/app_icon_512.png +0 -0
  61. package/flutter_app/assets/branding/app_icon_64.png +0 -0
  62. package/flutter_app/assets/branding/tray_icon_template.png +0 -0
  63. package/flutter_app/lib/features/location/location_service.dart +119 -0
  64. package/flutter_app/lib/features/notifications/notification_interceptor.dart +97 -0
  65. package/flutter_app/lib/main.dart +23057 -0
  66. package/flutter_app/lib/main_app_shell.dart +1682 -0
  67. package/flutter_app/lib/main_integrations.dart +931 -0
  68. package/flutter_app/lib/main_launcher.dart +959 -0
  69. package/flutter_app/lib/main_launcher_entry.dart +5 -0
  70. package/flutter_app/lib/main_models.dart +3473 -0
  71. package/flutter_app/lib/main_shared.dart +2861 -0
  72. package/flutter_app/lib/main_theme.dart +204 -0
  73. package/flutter_app/lib/main_voice_assistant.dart +831 -0
  74. package/flutter_app/lib/src/android_apk_drop_zone.dart +32 -0
  75. package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +16 -0
  76. package/flutter_app/lib/src/android_apk_drop_zone_web.dart +348 -0
  77. package/flutter_app/lib/src/android_app_installer.dart +22 -0
  78. package/flutter_app/lib/src/android_app_installer_io.dart +122 -0
  79. package/flutter_app/lib/src/android_app_installer_stub.dart +21 -0
  80. package/flutter_app/lib/src/android_launcher_bridge.dart +239 -0
  81. package/flutter_app/lib/src/app_launch_bridge.dart +29 -0
  82. package/flutter_app/lib/src/app_release_updater.dart +511 -0
  83. package/flutter_app/lib/src/backend_client.dart +1833 -0
  84. package/flutter_app/lib/src/desktop_companion.dart +2 -0
  85. package/flutter_app/lib/src/desktop_companion_actions.dart +586 -0
  86. package/flutter_app/lib/src/desktop_companion_io.dart +538 -0
  87. package/flutter_app/lib/src/desktop_companion_stub.dart +59 -0
  88. package/flutter_app/lib/src/desktop_native_bridge.dart +91 -0
  89. package/flutter_app/lib/src/desktop_screen_capture.dart +21 -0
  90. package/flutter_app/lib/src/desktop_screen_capture_io.dart +142 -0
  91. package/flutter_app/lib/src/desktop_screen_capture_stub.dart +12 -0
  92. package/flutter_app/lib/src/diagnostics_logger.dart +119 -0
  93. package/flutter_app/lib/src/health_bridge.dart +136 -0
  94. package/flutter_app/lib/src/live_voice_capture.dart +85 -0
  95. package/flutter_app/lib/src/messaging_access_summary.dart +46 -0
  96. package/flutter_app/lib/src/network/app_http_client.dart +53 -0
  97. package/flutter_app/lib/src/network/app_http_client_factory.dart +6 -0
  98. package/flutter_app/lib/src/network/app_http_client_io.dart +138 -0
  99. package/flutter_app/lib/src/network/app_http_client_stub.dart +3 -0
  100. package/flutter_app/lib/src/network/app_http_client_web.dart +94 -0
  101. package/flutter_app/lib/src/oauth_launcher.dart +33 -0
  102. package/flutter_app/lib/src/oauth_launcher_io.dart +77 -0
  103. package/flutter_app/lib/src/oauth_launcher_stub.dart +33 -0
  104. package/flutter_app/lib/src/oauth_launcher_web.dart +107 -0
  105. package/flutter_app/lib/src/recording_bridge.dart +232 -0
  106. package/flutter_app/lib/src/recording_bridge_io.dart +1019 -0
  107. package/flutter_app/lib/src/recording_bridge_stub.dart +120 -0
  108. package/flutter_app/lib/src/recording_bridge_web.dart +689 -0
  109. package/flutter_app/lib/src/recording_payloads.dart +86 -0
  110. package/flutter_app/lib/src/theme/palette.dart +81 -0
  111. package/flutter_app/lib/src/widget_bridge.dart +49 -0
  112. package/flutter_app/linux/CMakeLists.txt +128 -0
  113. package/flutter_app/linux/flutter/CMakeLists.txt +88 -0
  114. package/flutter_app/linux/flutter/generated_plugin_registrant.cc +43 -0
  115. package/flutter_app/linux/flutter/generated_plugin_registrant.h +15 -0
  116. package/flutter_app/linux/flutter/generated_plugins.cmake +31 -0
  117. package/flutter_app/linux/runner/CMakeLists.txt +26 -0
  118. package/flutter_app/linux/runner/main.cc +6 -0
  119. package/flutter_app/linux/runner/my_application.cc +144 -0
  120. package/flutter_app/linux/runner/my_application.h +18 -0
  121. package/flutter_app/linux/runner/resources/app_icon.png +0 -0
  122. package/flutter_app/macos/Flutter/Flutter-Debug.xcconfig +2 -0
  123. package/flutter_app/macos/Flutter/Flutter-Release.xcconfig +2 -0
  124. package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +40 -0
  125. package/flutter_app/macos/Podfile +42 -0
  126. package/flutter_app/macos/Podfile.lock +87 -0
  127. package/flutter_app/macos/Runner/AppDelegate.swift +576 -0
  128. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json +68 -0
  129. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
  130. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
  131. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
  132. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
  133. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
  134. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
  135. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
  136. package/flutter_app/macos/Runner/Base.lproj/MainMenu.xib +342 -0
  137. package/flutter_app/macos/Runner/Configs/AppInfo.xcconfig +14 -0
  138. package/flutter_app/macos/Runner/Configs/Debug.xcconfig +2 -0
  139. package/flutter_app/macos/Runner/Configs/Release.xcconfig +2 -0
  140. package/flutter_app/macos/Runner/Configs/Warnings.xcconfig +13 -0
  141. package/flutter_app/macos/Runner/DebugProfile.entitlements +16 -0
  142. package/flutter_app/macos/Runner/Info.plist +36 -0
  143. package/flutter_app/macos/Runner/MainFlutterWindow.swift +19 -0
  144. package/flutter_app/macos/Runner/Release.entitlements +12 -0
  145. package/flutter_app/macos/Runner.xcodeproj/project.pbxproj +801 -0
  146. package/flutter_app/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  147. package/flutter_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +99 -0
  148. package/flutter_app/macos/Runner.xcworkspace/contents.xcworkspacedata +10 -0
  149. package/flutter_app/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  150. package/flutter_app/macos/RunnerTests/RunnerTests.swift +12 -0
  151. package/flutter_app/patch_strings.py +12 -0
  152. package/flutter_app/pubspec.lock +1088 -0
  153. package/flutter_app/pubspec.yaml +53 -0
  154. package/flutter_app/test/messaging_access_summary_test.dart +22 -0
  155. package/flutter_app/test/recording_payloads_test.dart +53 -0
  156. package/flutter_app/third_party/desktop_audio_capture/LICENSE +21 -0
  157. package/flutter_app/third_party/desktop_audio_capture/README.md +262 -0
  158. package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +65 -0
  159. package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +153 -0
  160. package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +110 -0
  161. package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +461 -0
  162. package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +91 -0
  163. package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +106 -0
  164. package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +219 -0
  165. package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +336 -0
  166. package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +101 -0
  167. package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +692 -0
  168. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +35 -0
  169. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +36 -0
  170. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +32 -0
  171. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +32 -0
  172. package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +878 -0
  173. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +27 -0
  174. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +1172 -0
  175. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +655 -0
  176. package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +12 -0
  177. package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +30 -0
  178. package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +87 -0
  179. package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +105 -0
  180. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +80 -0
  181. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +31 -0
  182. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +12 -0
  183. package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +23 -0
  184. package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +25 -0
  185. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +1117 -0
  186. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +115 -0
  187. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +777 -0
  188. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +87 -0
  189. package/flutter_app/third_party/flutter_secure_storage_linux/linux/CMakeLists.txt +30 -0
  190. package/flutter_app/third_party/flutter_secure_storage_linux/linux/flutter_secure_storage_linux_plugin.cc +215 -0
  191. package/flutter_app/third_party/flutter_secure_storage_linux/linux/include/flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h +27 -0
  192. package/flutter_app/third_party/flutter_secure_storage_linux/pubspec.yaml +20 -0
  193. package/flutter_app/tool/generate_desktop_branding.py +219 -0
  194. package/flutter_app/web/favicon.png +0 -0
  195. package/flutter_app/web/favicon.svg +12 -0
  196. package/flutter_app/web/icons/Icon-192.png +0 -0
  197. package/flutter_app/web/icons/Icon-512.png +0 -0
  198. package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
  199. package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
  200. package/flutter_app/web/index.html +39 -0
  201. package/flutter_app/web/manifest.json +35 -0
  202. package/flutter_app/windows/CMakeLists.txt +108 -0
  203. package/flutter_app/windows/flutter/CMakeLists.txt +109 -0
  204. package/flutter_app/windows/flutter/generated_plugin_registrant.cc +47 -0
  205. package/flutter_app/windows/flutter/generated_plugin_registrant.h +15 -0
  206. package/flutter_app/windows/flutter/generated_plugins.cmake +35 -0
  207. package/flutter_app/windows/runner/CMakeLists.txt +41 -0
  208. package/flutter_app/windows/runner/Runner.rc +121 -0
  209. package/flutter_app/windows/runner/flutter_window.cpp +533 -0
  210. package/flutter_app/windows/runner/flutter_window.h +37 -0
  211. package/flutter_app/windows/runner/main.cpp +53 -0
  212. package/flutter_app/windows/runner/resource.h +16 -0
  213. package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
  214. package/flutter_app/windows/runner/runner.exe.manifest +14 -0
  215. package/flutter_app/windows/runner/utils.cpp +65 -0
  216. package/flutter_app/windows/runner/utils.h +19 -0
  217. package/flutter_app/windows/runner/win32_window.cpp +299 -0
  218. package/flutter_app/windows/runner/win32_window.h +102 -0
  219. package/lib/manager.js +231 -7
  220. package/package.json +3 -1
  221. package/server/db/database.js +68 -0
  222. package/server/http/middleware.js +50 -0
  223. package/server/http/routes.js +3 -1
  224. package/server/index.js +1 -0
  225. package/server/public/.last_build_id +1 -1
  226. package/server/public/assets/NOTICES +61 -0
  227. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  228. package/server/public/flutter_bootstrap.js +1 -1
  229. package/server/public/main.dart.js +65262 -64422
  230. package/server/routes/integrations.js +86 -0
  231. package/server/routes/memory.js +11 -2
  232. package/server/routes/screenHistory.js +46 -0
  233. package/server/routes/triggers.js +81 -0
  234. package/server/services/ai/models.js +30 -0
  235. package/server/services/ai/providers/githubCopilot.js +97 -0
  236. package/server/services/ai/providers/openai.js +2 -1
  237. package/server/services/ai/providers/openaiCodex.js +31 -0
  238. package/server/services/ai/settings.js +20 -0
  239. package/server/services/ai/systemPrompt.js +1 -1
  240. package/server/services/ai/tools.js +35 -6
  241. package/server/services/browser/controller.js +47 -3
  242. package/server/services/desktop/screenRecorder.js +172 -0
  243. package/server/services/integrations/env.js +5 -0
  244. package/server/services/integrations/github/common.js +106 -0
  245. package/server/services/integrations/github/provider.js +499 -0
  246. package/server/services/integrations/github/repos.js +1124 -0
  247. package/server/services/integrations/home_assistant/provider.js +306 -26
  248. package/server/services/integrations/manager.js +63 -7
  249. package/server/services/integrations/oauth_provider.js +13 -6
  250. package/server/services/integrations/provider_config_store.js +76 -0
  251. package/server/services/integrations/registry.js +4 -0
  252. package/server/services/integrations/trello/provider.js +744 -0
  253. package/server/services/integrations/whatsapp/provider.js +6 -2
  254. package/server/services/manager.js +22 -0
  255. package/server/services/memory/manager.js +39 -2
  256. package/server/services/skills/base_catalog.js +33 -0
  257. package/server/services/tasks/adapters/index.js +1 -0
  258. package/server/services/tasks/adapters/manual.js +12 -0
  259. package/server/services/tasks/runtime.js +1 -1
  260. package/server/services/voice/openaiClient.js +4 -1
  261. package/server/services/voice/providers.js +2 -1
  262. package/server/services/widgets/service.js +49 -4
  263. package/server/utils/local_secrets.js +56 -0
  264. package/server/utils/logger.js +37 -9
@@ -0,0 +1,1117 @@
1
+ #include "mic_capture_plugin.h"
2
+
3
+ #ifndef NOMINMAX
4
+ #define NOMINMAX
5
+ #endif
6
+ #include <windows.h>
7
+ #undef max
8
+ #undef min
9
+ #include <mmdeviceapi.h>
10
+ #include <audioclient.h>
11
+ #include <functiondiscoverykeys_devpkey.h>
12
+ #include <VersionHelpers.h>
13
+ #include <comdef.h>
14
+ #include <propkey.h>
15
+ #include <propvarutil.h>
16
+
17
+ #include <flutter/event_channel.h>
18
+ #include <flutter/method_channel.h>
19
+ #include <flutter/plugin_registrar_windows.h>
20
+ #include <flutter/standard_method_codec.h>
21
+
22
+ #include <algorithm>
23
+ #include <chrono>
24
+ #include <cmath>
25
+ #include <cstdint>
26
+ #include <functional>
27
+ #include <memory>
28
+ #include <sstream>
29
+ #include <string>
30
+ #include <thread>
31
+ #include <vector>
32
+ // After existing includes, add:
33
+ #include <queue>
34
+ #include <chrono>
35
+
36
+ #pragma comment(lib, "ole32.lib")
37
+ #pragma comment(lib, "oleaut32.lib")
38
+
39
+ // REFTIMES_PER_SEC is 10,000,000 (100 nanoseconds per second)
40
+ #ifndef REFTIMES_PER_SEC
41
+ #define REFTIMES_PER_SEC 10000000
42
+ #endif
43
+
44
+ namespace audio_capture {
45
+
46
+ // Custom StreamHandler implementation
47
+ template <typename T>
48
+ class StreamHandlerFunctions : public flutter::StreamHandler<T> {
49
+ public:
50
+ using OnListenHandler = std::function<std::unique_ptr<flutter::StreamHandlerError<T>>(
51
+ const T* arguments,
52
+ std::unique_ptr<flutter::EventSink<T>>&& events)>;
53
+ using OnCancelHandler = std::function<std::unique_ptr<flutter::StreamHandlerError<T>>(
54
+ const T* arguments)>;
55
+
56
+ StreamHandlerFunctions(OnListenHandler on_listen, OnCancelHandler on_cancel)
57
+ : on_listen_(std::move(on_listen)), on_cancel_(std::move(on_cancel)) {}
58
+
59
+ virtual ~StreamHandlerFunctions() = default;
60
+
61
+ protected:
62
+ std::unique_ptr<flutter::StreamHandlerError<T>> OnListenInternal(
63
+ const T* arguments,
64
+ std::unique_ptr<flutter::EventSink<T>>&& events) override {
65
+ return on_listen_(arguments, std::move(events));
66
+ }
67
+
68
+ std::unique_ptr<flutter::StreamHandlerError<T>> OnCancelInternal(
69
+ const T* arguments) override {
70
+ return on_cancel_(arguments);
71
+ }
72
+
73
+ private:
74
+ OnListenHandler on_listen_;
75
+ OnCancelHandler on_cancel_;
76
+ };
77
+
78
+ namespace {
79
+
80
+ constexpr char kMethodChannelName[] = "com.mic_audio_transcriber/mic_capture";
81
+ constexpr char kEventChannelName[] = "com.mic_audio_transcriber/mic_stream";
82
+ constexpr char kStatusEventChannelName[] = "com.mic_audio_transcriber/mic_status";
83
+ constexpr char kDecibelEventChannelName[] = "com.mic_audio_transcriber/mic_decibel";
84
+
85
+ constexpr int kDefaultSampleRate = 16000;
86
+ constexpr int kDefaultChannels = 1;
87
+ constexpr int kDefaultBitsPerSample = 16;
88
+ constexpr float kDefaultGainBoost = 2.5f;
89
+ constexpr float kDefaultInputVolume = 1.0f;
90
+
91
+ } // namespace
92
+
93
+ // static
94
+ void MicCapturePlugin::RegisterWithRegistrar(
95
+ flutter::PluginRegistrarWindows *registrar) {
96
+ auto plugin = std::make_unique<MicCapturePlugin>(registrar);
97
+ registrar->AddPlugin(std::move(plugin));
98
+ }
99
+
100
+ MicCapturePlugin::MicCapturePlugin(flutter::PluginRegistrarWindows *registrar)
101
+ : registrar_(registrar),
102
+ is_capturing_(false),
103
+ should_stop_(false),
104
+ sample_rate_(kDefaultSampleRate),
105
+ channels_(kDefaultChannels),
106
+ bits_per_sample_(kDefaultBitsPerSample),
107
+ gain_boost_(kDefaultGainBoost),
108
+ input_volume_(kDefaultInputVolume),
109
+ audio_client_(nullptr),
110
+ capture_client_(nullptr),
111
+ device_(nullptr),
112
+ mix_format_(nullptr),
113
+ buffer_frame_count_(0),
114
+ com_initialized_(false) {
115
+ // Create method channel
116
+ method_channel_ =
117
+ std::make_unique<flutter::MethodChannel<flutter::EncodableValue>>(
118
+ registrar->messenger(), kMethodChannelName,
119
+ &flutter::StandardMethodCodec::GetInstance());
120
+
121
+ method_channel_->SetMethodCallHandler(
122
+ [this](const auto &call, auto result) {
123
+ this->HandleMethodCall(call, std::move(result));
124
+ });
125
+
126
+ // Create event channels
127
+ event_channel_ =
128
+ std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
129
+ registrar->messenger(), kEventChannelName,
130
+ &flutter::StandardMethodCodec::GetInstance());
131
+
132
+ event_channel_->SetStreamHandler(
133
+ std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
134
+ [this](const flutter::EncodableValue* arguments,
135
+ std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
136
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
137
+ std::lock_guard<std::mutex> lock(mutex_);
138
+ event_sink_ = std::move(events);
139
+ return nullptr;
140
+ },
141
+ [this](const flutter::EncodableValue* arguments)
142
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
143
+ std::lock_guard<std::mutex> lock(mutex_);
144
+ event_sink_.reset();
145
+ return nullptr;
146
+ }));
147
+
148
+ status_event_channel_ =
149
+ std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
150
+ registrar->messenger(), kStatusEventChannelName,
151
+ &flutter::StandardMethodCodec::GetInstance());
152
+
153
+ status_event_channel_->SetStreamHandler(
154
+ std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
155
+ [this](const flutter::EncodableValue* arguments,
156
+ std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
157
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
158
+ std::lock_guard<std::mutex> lock(mutex_);
159
+ status_event_sink_ = std::move(events);
160
+ SendStatusUpdate(is_capturing_, current_device_name_);
161
+ return nullptr;
162
+ },
163
+ [this](const flutter::EncodableValue* arguments)
164
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
165
+ std::lock_guard<std::mutex> lock(mutex_);
166
+ status_event_sink_.reset();
167
+ return nullptr;
168
+ }));
169
+
170
+ decibel_event_channel_ =
171
+ std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
172
+ registrar->messenger(), kDecibelEventChannelName,
173
+ &flutter::StandardMethodCodec::GetInstance());
174
+
175
+ decibel_event_channel_->SetStreamHandler(
176
+ std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
177
+ [this](const flutter::EncodableValue* arguments,
178
+ std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
179
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
180
+ std::lock_guard<std::mutex> lock(mutex_);
181
+ decibel_event_sink_ = std::move(events);
182
+ return nullptr;
183
+ },
184
+ [this](const flutter::EncodableValue* arguments)
185
+ -> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
186
+ std::lock_guard<std::mutex> lock(mutex_);
187
+ decibel_event_sink_.reset();
188
+ return nullptr;
189
+ }));
190
+ }
191
+
192
+ MicCapturePlugin::~MicCapturePlugin() {
193
+ StopCapture();
194
+ }
195
+
196
+ // NEW: Queue audio data from background thread
197
+ void MicCapturePlugin::QueueAudioData(std::vector<uint8_t> data, double decibel) {
198
+ std::lock_guard<std::mutex> lock(queue_mutex_);
199
+
200
+ // Prevent queue overflow
201
+ if (audio_queue_.size() >= kMaxQueueSize) {
202
+ audio_queue_.pop(); // Remove oldest
203
+ }
204
+
205
+ AudioDataPacket packet;
206
+ packet.data = std::move(data);
207
+ packet.decibel = decibel;
208
+ packet.timestamp = std::chrono::steady_clock::now();
209
+
210
+ audio_queue_.push(std::move(packet));
211
+
212
+ // Post task to platform thread to process queue
213
+ registrar_->messenger()->Send(
214
+ "", // Empty channel name - just for triggering callback
215
+ nullptr,
216
+ 0,
217
+ [this](const uint8_t* reply, size_t reply_size) {
218
+ this->ProcessQueue();
219
+ }
220
+ );
221
+ }
222
+
223
+ // NEW: Process queue on platform thread
224
+ void MicCapturePlugin::ProcessQueue() {
225
+ std::lock_guard<std::mutex> lock(queue_mutex_);
226
+
227
+ // Process all queued packets
228
+ while (!audio_queue_.empty()) {
229
+ AudioDataPacket& packet = audio_queue_.front();
230
+
231
+ // Send audio data
232
+ {
233
+ std::lock_guard<std::mutex> sink_lock(mutex_);
234
+ if (event_sink_) {
235
+ try {
236
+ event_sink_->Success(flutter::EncodableValue(packet.data));
237
+ } catch (...) {
238
+ // Ignore errors
239
+ }
240
+ }
241
+ }
242
+
243
+ // Send decibel data
244
+ {
245
+ std::lock_guard<std::mutex> sink_lock(mutex_);
246
+ if (decibel_event_sink_) {
247
+ try {
248
+ flutter::EncodableMap decibel_map;
249
+ decibel_map[flutter::EncodableValue("decibel")] =
250
+ flutter::EncodableValue(packet.decibel);
251
+
252
+ auto now = std::chrono::system_clock::now();
253
+ auto duration = now.time_since_epoch();
254
+ auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
255
+ double timestamp = static_cast<double>(ms) / 1000.0;
256
+
257
+ decibel_map[flutter::EncodableValue("timestamp")] =
258
+ flutter::EncodableValue(timestamp);
259
+
260
+ decibel_event_sink_->Success(flutter::EncodableValue(decibel_map));
261
+ } catch (...) {
262
+ // Ignore errors
263
+ }
264
+ }
265
+ }
266
+
267
+ audio_queue_.pop();
268
+ }
269
+ }
270
+
271
+ void MicCapturePlugin::HandleMethodCall(
272
+ const flutter::MethodCall<flutter::EncodableValue> &method_call,
273
+ std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result) {
274
+ if (method_call.method_name().compare("requestPermissions") == 0) {
275
+ // On Windows, permissions are typically handled by the system
276
+ result->Success(flutter::EncodableValue(true));
277
+ } else if (method_call.method_name().compare("hasInputDevice") == 0) {
278
+ bool has_device = HasInputDevice();
279
+ result->Success(flutter::EncodableValue(has_device));
280
+ } else if (method_call.method_name().compare("isSupported") == 0) {
281
+ bool has_device = HasInputDevice();
282
+ result->Success(flutter::EncodableValue(has_device));
283
+ } else if (method_call.method_name().compare("checkMicSupport") == 0) {
284
+ bool has_device = HasInputDevice();
285
+ result->Success(flutter::EncodableValue(has_device));
286
+ } else if (method_call.method_name().compare("getAvailableInputDevices") == 0) {
287
+ auto devices = GetAvailableInputDevices();
288
+ result->Success(flutter::EncodableValue(devices));
289
+ } else if (method_call.method_name().compare("startCapture") == 0) {
290
+ const flutter::EncodableMap* args = nullptr;
291
+ if (method_call.arguments() &&
292
+ std::holds_alternative<flutter::EncodableMap>(*method_call.arguments())) {
293
+ args = &std::get<flutter::EncodableMap>(*method_call.arguments());
294
+ }
295
+ bool started = StartCapture(args);
296
+ result->Success(flutter::EncodableValue(started));
297
+ } else if (method_call.method_name().compare("stopCapture") == 0) {
298
+ bool stopped = StopCapture();
299
+ result->Success(flutter::EncodableValue(stopped));
300
+ } else {
301
+ result->NotImplemented();
302
+ }
303
+ }
304
+
305
+ void MicCapturePlugin::ApplyGainBoostAndConvertToMono(
306
+ const int16_t* input, int16_t* output, size_t frame_count,
307
+ int input_channels, float gain_boost) {
308
+ const float max_value = 32767.0f;
309
+ const float min_value = -32768.0f;
310
+
311
+ if (input_channels == 1) {
312
+ // Mono: just apply gain boost
313
+ for (size_t i = 0; i < frame_count; ++i) {
314
+ float sample = static_cast<float>(input[i]) * gain_boost;
315
+ float clamped_sample = (std::min)(max_value, sample);
316
+ sample = (std::max)(min_value, clamped_sample);
317
+ output[i] = static_cast<int16_t>(sample);
318
+ }
319
+ } else {
320
+ // Stereo: convert to mono and apply gain boost
321
+ for (size_t i = 0; i < frame_count; ++i) {
322
+ float left = static_cast<float>(input[i * 2]);
323
+ float right = static_cast<float>(input[i * 2 + 1]);
324
+ float mono = (left + right) / 2.0f * gain_boost;
325
+ float clamped_mono = (std::min)(max_value, mono);
326
+ mono = (std::max)(min_value, clamped_mono);
327
+ output[i] = static_cast<int16_t>(mono);
328
+ }
329
+ }
330
+ }
331
+
332
+ double MicCapturePlugin::CalculateDecibel(const int16_t* samples,
333
+ size_t sample_count) {
334
+ if (sample_count == 0) {
335
+ return -120.0;
336
+ }
337
+
338
+ // Calculate RMS (Root Mean Square)
339
+ double sum_of_squares = 0.0;
340
+ for (size_t i = 0; i < sample_count; ++i) {
341
+ double value = static_cast<double>(samples[i]);
342
+ sum_of_squares += value * value;
343
+ }
344
+ double mean_square = sum_of_squares / static_cast<double>(sample_count);
345
+ double rms = sqrt(mean_square);
346
+
347
+ // Calculate decibel: dB = 20 * log10(RMS / max_value)
348
+ const double max_value = 32767.0;
349
+ if (rms <= 0.0) {
350
+ return -120.0; // Avoid log(0)
351
+ }
352
+
353
+ double decibel = 20.0 * log10(rms / max_value);
354
+
355
+ // Clamp to reasonable range (-120 dB to 0 dB)
356
+ double clamped_decibel = (std::min)(0.0, decibel);
357
+ return (std::max)(-120.0, clamped_decibel);
358
+ }
359
+
360
+ // UPDATED: SendStatusUpdate - also post to platform thread
361
+ void MicCapturePlugin::SendStatusUpdate(bool is_active, const std::string& device_name) {
362
+ registrar_->messenger()->Send(
363
+ "",
364
+ nullptr,
365
+ 0,
366
+ [this, is_active, device_name](const uint8_t* reply, size_t reply_size) {
367
+ std::lock_guard<std::mutex> lock(mutex_);
368
+ if (status_event_sink_) {
369
+ try {
370
+ flutter::EncodableMap status_map;
371
+ status_map[flutter::EncodableValue("isActive")] = flutter::EncodableValue(is_active);
372
+
373
+ auto now = std::chrono::system_clock::now();
374
+ auto duration = now.time_since_epoch();
375
+ auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
376
+ double timestamp = static_cast<double>(ms) / 1000.0;
377
+
378
+ status_map[flutter::EncodableValue("timestamp")] = flutter::EncodableValue(timestamp);
379
+
380
+ if (!device_name.empty()) {
381
+ status_map[flutter::EncodableValue("deviceName")] = flutter::EncodableValue(device_name);
382
+ }
383
+
384
+ status_event_sink_->Success(flutter::EncodableValue(status_map));
385
+ } catch (...) {}
386
+ }
387
+ }
388
+ );
389
+ }
390
+
391
+ std::string MicCapturePlugin::GetCurrentDeviceName() {
392
+ if (!device_) {
393
+ return "Default Microphone";
394
+ }
395
+
396
+ IPropertyStore* props = nullptr;
397
+ HRESULT hr = device_->OpenPropertyStore(STGM_READ, &props);
398
+ if (FAILED(hr)) {
399
+ return "Default Microphone";
400
+ }
401
+
402
+ PROPVARIANT varName;
403
+ PropVariantInit(&varName);
404
+ hr = props->GetValue(PKEY_Device_FriendlyName, &varName);
405
+ props->Release();
406
+
407
+ std::string device_name = "Default Microphone";
408
+ if (SUCCEEDED(hr) && varName.vt == VT_LPWSTR) {
409
+ // Convert wide string to narrow string
410
+ int size_needed = WideCharToMultiByte(CP_UTF8, 0, varName.pwszVal, -1, nullptr, 0, nullptr, nullptr);
411
+ if (size_needed > 0) {
412
+ std::vector<char> buffer(size_needed);
413
+ WideCharToMultiByte(CP_UTF8, 0, varName.pwszVal, -1, buffer.data(), size_needed, nullptr, nullptr);
414
+ device_name = std::string(buffer.data());
415
+ }
416
+ }
417
+ PropVariantClear(&varName);
418
+
419
+ return device_name;
420
+ }
421
+
422
+ bool MicCapturePlugin::IsBluetoothDevice() {
423
+ // Check device name for Bluetooth keywords
424
+ std::string device_name = GetCurrentDeviceName();
425
+
426
+ // Convert to lowercase safely
427
+ for (size_t i = 0; i < device_name.length(); ++i) {
428
+ device_name[i] = static_cast<char>(::tolower(static_cast<unsigned char>(device_name[i])));
429
+ }
430
+
431
+ const char* bluetooth_keywords[] = {
432
+ "bluetooth", "airpods", "beats", "jabra", "sony", "bose", "jbl"
433
+ };
434
+
435
+ for (size_t i = 0; i < sizeof(bluetooth_keywords) / sizeof(bluetooth_keywords[0]); ++i) {
436
+ if (device_name.find(bluetooth_keywords[i]) != std::string::npos) {
437
+ return true;
438
+ }
439
+ }
440
+
441
+ return false;
442
+ }
443
+
444
+ void MicCapturePlugin::CleanupExistingCapture() {
445
+ std::lock_guard<std::mutex> lock(mutex_);
446
+
447
+ if (is_capturing_ && capture_thread_.joinable()) {
448
+ should_stop_ = true;
449
+ mutex_.unlock();
450
+
451
+ capture_thread_.join();
452
+
453
+ mutex_.lock();
454
+ is_capturing_ = false;
455
+ }
456
+
457
+ current_device_name_.clear();
458
+
459
+ // Small delay for cleanup to complete
460
+ std::this_thread::sleep_for(std::chrono::milliseconds(500));
461
+ }
462
+
463
+ bool MicCapturePlugin::OpenWASAPIStreamWithRetry(
464
+ int sample_rate, int channels, int bits_per_sample, bool is_bluetooth,
465
+ void** out_audio_client, void** out_capture_client,
466
+ std::string* error_message) {
467
+ const int max_retries = is_bluetooth ? 5 : 3;
468
+ const double initial_wait = is_bluetooth ? 1.5 : 0.3;
469
+ // ĐÚNG - Khởi tạo array riêng
470
+ const double* retry_delays;
471
+ const double bluetooth_delays[] = {0.5, 1.0, 1.5, 2.0, 2.5};
472
+ const double normal_delays[] = {0.3, 0.6, 1.0, 0.0, 0.0};
473
+ retry_delays = is_bluetooth ? bluetooth_delays : normal_delays;
474
+
475
+ // Initial wait for device to be ready
476
+ std::this_thread::sleep_for(
477
+ std::chrono::milliseconds(static_cast<int>(initial_wait * 1000)));
478
+
479
+ for (int attempt = 1; attempt <= max_retries; ++attempt) {
480
+ // Initialize COM (allow RPC_E_CHANGED_MODE if already initialized)
481
+ HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
482
+ bool com_initialized_this_attempt = (hr == S_OK);
483
+ if (FAILED(hr) && hr != RPC_E_CHANGED_MODE) {
484
+ if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
485
+ std::this_thread::sleep_for(
486
+ std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
487
+ continue;
488
+ }
489
+ if (error_message) {
490
+ *error_message = "Failed to initialize COM";
491
+ }
492
+ return false;
493
+ }
494
+
495
+ // Get default audio endpoint (eCapture for microphone)
496
+ IMMDeviceEnumerator* enumerator = nullptr;
497
+ hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL,
498
+ __uuidof(IMMDeviceEnumerator),
499
+ reinterpret_cast<void**>(&enumerator));
500
+ if (FAILED(hr)) {
501
+ if (com_initialized_this_attempt) {
502
+ CoUninitialize();
503
+ }
504
+ if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
505
+ std::this_thread::sleep_for(
506
+ std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
507
+ continue;
508
+ }
509
+ if (error_message) {
510
+ *error_message = "Failed to create device enumerator";
511
+ }
512
+ return false;
513
+ }
514
+
515
+ hr = enumerator->GetDefaultAudioEndpoint(eCapture, eConsole, &device_);
516
+ enumerator->Release();
517
+ if (FAILED(hr)) {
518
+ if (com_initialized_this_attempt) {
519
+ CoUninitialize();
520
+ }
521
+ if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
522
+ std::this_thread::sleep_for(
523
+ std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
524
+ continue;
525
+ }
526
+ if (error_message) {
527
+ *error_message = "Failed to get default audio endpoint";
528
+ }
529
+ return false;
530
+ }
531
+
532
+ // Activate IAudioClient
533
+ hr = device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr,
534
+ reinterpret_cast<void**>(&audio_client_));
535
+ if (FAILED(hr)) {
536
+ device_->Release();
537
+ device_ = nullptr;
538
+ if (com_initialized_this_attempt) {
539
+ CoUninitialize();
540
+ }
541
+ if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
542
+ std::this_thread::sleep_for(
543
+ std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
544
+ continue;
545
+ }
546
+ if (error_message) {
547
+ *error_message = "Failed to activate IAudioClient";
548
+ }
549
+ return false;
550
+ }
551
+
552
+ // Get mix format - use original format for initialization
553
+ WAVEFORMATEX* device_format = nullptr;
554
+ hr = audio_client_->GetMixFormat(&device_format);
555
+ if (FAILED(hr)) {
556
+ audio_client_->Release();
557
+ audio_client_ = nullptr;
558
+ device_->Release();
559
+ device_ = nullptr;
560
+ if (com_initialized_this_attempt) {
561
+ CoUninitialize();
562
+ }
563
+ if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
564
+ std::this_thread::sleep_for(
565
+ std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
566
+ continue;
567
+ }
568
+ if (error_message) {
569
+ *error_message = "Failed to get mix format";
570
+ }
571
+ return false;
572
+ }
573
+
574
+ // Store device format for use in capture thread
575
+ // We'll use the device's native format and convert in the thread
576
+ mix_format_ = device_format;
577
+
578
+ // Initialize audio client for capture with device's native format
579
+ // FIX LATENCY: Giảm buffer duration xuống 100ms để giảm latency
580
+ REFERENCE_TIME hnsRequestedDuration = REFTIMES_PER_SEC / 10; // 100ms instead of 1 second
581
+ hr = audio_client_->Initialize(AUDCLNT_SHAREMODE_SHARED, 0,
582
+ hnsRequestedDuration, 0, mix_format_, nullptr);
583
+ if (FAILED(hr)) {
584
+ CoTaskMemFree(mix_format_);
585
+ mix_format_ = nullptr;
586
+ audio_client_->Release();
587
+ audio_client_ = nullptr;
588
+ device_->Release();
589
+ device_ = nullptr;
590
+ if (com_initialized_this_attempt) {
591
+ CoUninitialize();
592
+ }
593
+ if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
594
+ std::this_thread::sleep_for(
595
+ std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
596
+ continue;
597
+ }
598
+ if (error_message) {
599
+ *error_message = "Failed to initialize audio client";
600
+ }
601
+ return false;
602
+ }
603
+
604
+ // Get buffer size
605
+ hr = audio_client_->GetBufferSize(&buffer_frame_count_);
606
+ if (FAILED(hr)) {
607
+ CoTaskMemFree(mix_format_);
608
+ mix_format_ = nullptr;
609
+ audio_client_->Release();
610
+ audio_client_ = nullptr;
611
+ device_->Release();
612
+ device_ = nullptr;
613
+ if (com_initialized_this_attempt) {
614
+ CoUninitialize();
615
+ }
616
+ if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
617
+ std::this_thread::sleep_for(
618
+ std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
619
+ continue;
620
+ }
621
+ if (error_message) {
622
+ *error_message = "Failed to get buffer size";
623
+ }
624
+ return false;
625
+ }
626
+
627
+ // Get IAudioCaptureClient
628
+ hr = audio_client_->GetService(__uuidof(IAudioCaptureClient),
629
+ reinterpret_cast<void**>(&capture_client_));
630
+ if (FAILED(hr)) {
631
+ CoTaskMemFree(mix_format_);
632
+ mix_format_ = nullptr;
633
+ audio_client_->Release();
634
+ audio_client_ = nullptr;
635
+ device_->Release();
636
+ device_ = nullptr;
637
+ if (com_initialized_this_attempt) {
638
+ CoUninitialize();
639
+ }
640
+ if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
641
+ std::this_thread::sleep_for(
642
+ std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
643
+ continue;
644
+ }
645
+ if (error_message) {
646
+ *error_message = "Failed to get IAudioCaptureClient";
647
+ }
648
+ return false;
649
+ }
650
+
651
+ // Start capture
652
+ hr = audio_client_->Start();
653
+ if (FAILED(hr)) {
654
+ capture_client_->Release();
655
+ capture_client_ = nullptr;
656
+ CoTaskMemFree(mix_format_);
657
+ mix_format_ = nullptr;
658
+ audio_client_->Release();
659
+ audio_client_ = nullptr;
660
+ device_->Release();
661
+ device_ = nullptr;
662
+ if (com_initialized_this_attempt) {
663
+ CoUninitialize();
664
+ }
665
+ if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
666
+ std::this_thread::sleep_for(
667
+ std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
668
+ continue;
669
+ }
670
+ if (error_message) {
671
+ *error_message = "Failed to start audio client";
672
+ }
673
+ return false;
674
+ }
675
+
676
+ // Success - track that we initialized COM
677
+ com_initialized_ = com_initialized_this_attempt;
678
+ *out_audio_client = audio_client_;
679
+ *out_capture_client = capture_client_;
680
+ return true;
681
+ }
682
+
683
+ if (error_message) {
684
+ *error_message = "Failed to open WASAPI stream after retries";
685
+ }
686
+ return false;
687
+ }
688
+
689
+ bool MicCapturePlugin::HasInputDevice() {
690
+ HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
691
+ bool com_initialized = (hr == S_OK);
692
+ if (FAILED(hr) && hr != RPC_E_CHANGED_MODE) {
693
+ return false;
694
+ }
695
+
696
+ IMMDeviceEnumerator* enumerator = nullptr;
697
+ hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL,
698
+ __uuidof(IMMDeviceEnumerator),
699
+ reinterpret_cast<void**>(&enumerator));
700
+ if (FAILED(hr)) {
701
+ if (com_initialized) {
702
+ CoUninitialize();
703
+ }
704
+ return false;
705
+ }
706
+
707
+ IMMDevice* device = nullptr;
708
+ hr = enumerator->GetDefaultAudioEndpoint(eCapture, eConsole, &device);
709
+ enumerator->Release();
710
+
711
+ bool has_device = SUCCEEDED(hr) && device != nullptr;
712
+ if (device) {
713
+ device->Release();
714
+ }
715
+ if (com_initialized) {
716
+ CoUninitialize();
717
+ }
718
+
719
+ return has_device;
720
+ }
721
+
722
+ std::vector<flutter::EncodableValue> MicCapturePlugin::GetAvailableInputDevices() {
723
+ std::vector<flutter::EncodableValue> device_list;
724
+
725
+ // Get default device info
726
+ std::string device_name = GetCurrentDeviceName();
727
+ bool is_bluetooth = IsBluetoothDevice();
728
+
729
+ flutter::EncodableMap device_map;
730
+ device_map[flutter::EncodableValue("id")] = flutter::EncodableValue("default");
731
+ device_map[flutter::EncodableValue("name")] = flutter::EncodableValue(device_name);
732
+ device_map[flutter::EncodableValue("type")] =
733
+ flutter::EncodableValue(is_bluetooth ? "bluetooth" : "external");
734
+ device_map[flutter::EncodableValue("channelCount")] = flutter::EncodableValue(1);
735
+ device_map[flutter::EncodableValue("isDefault")] = flutter::EncodableValue(true);
736
+
737
+ device_list.push_back(flutter::EncodableValue(device_map));
738
+
739
+ return device_list;
740
+ }
741
+
742
+ bool MicCapturePlugin::StartCapture(const flutter::EncodableMap* args) {
743
+ // Always cleanup any existing capture first
744
+ CleanupExistingCapture();
745
+
746
+ // Parse arguments
747
+ if (args) {
748
+ auto it = args->find(flutter::EncodableValue("sampleRate"));
749
+ if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
750
+ sample_rate_ = std::get<int32_t>(it->second);
751
+ }
752
+
753
+ it = args->find(flutter::EncodableValue("channels"));
754
+ if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
755
+ channels_ = std::get<int32_t>(it->second);
756
+ }
757
+
758
+ it = args->find(flutter::EncodableValue("bitDepth"));
759
+ if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
760
+ bits_per_sample_ = std::get<int32_t>(it->second);
761
+ }
762
+
763
+ it = args->find(flutter::EncodableValue("gainBoost"));
764
+ if (it != args->end() && std::holds_alternative<double>(it->second)) {
765
+ gain_boost_ = static_cast<float>(std::get<double>(it->second));
766
+ }
767
+
768
+ it = args->find(flutter::EncodableValue("inputVolume"));
769
+ if (it != args->end() && std::holds_alternative<double>(it->second)) {
770
+ input_volume_ = static_cast<float>(std::get<double>(it->second));
771
+ }
772
+ }
773
+
774
+ // Clamp values
775
+ sample_rate_ = (std::max)(sample_rate_, 8000);
776
+ int min_channels = (std::min)(channels_, 2);
777
+ channels_ = (std::max)(1, min_channels);
778
+ bits_per_sample_ = 16; // Force 16-bit
779
+ float min_gain = (std::min)(10.0f, gain_boost_);
780
+ gain_boost_ = (std::max)(0.1f, min_gain);
781
+ float min_volume = (std::min)(1.0f, input_volume_);
782
+ input_volume_ = (std::max)(0.0f, min_volume);
783
+
784
+ // Detect if device is Bluetooth
785
+ bool is_bluetooth = IsBluetoothDevice();
786
+
787
+ std::string error_message;
788
+ if (!OpenWASAPIStreamWithRetry(sample_rate_, channels_, bits_per_sample_,
789
+ is_bluetooth,
790
+ reinterpret_cast<void**>(&audio_client_),
791
+ reinterpret_cast<void**>(&capture_client_),
792
+ &error_message)) {
793
+ // Log error
794
+ return false;
795
+ }
796
+
797
+ // Get device name (device_ is already set by OpenWASAPIStreamWithRetry)
798
+ current_device_name_ = GetCurrentDeviceName();
799
+
800
+ {
801
+ std::lock_guard<std::mutex> lock(mutex_);
802
+ if (is_capturing_) {
803
+ return false;
804
+ }
805
+ should_stop_ = false;
806
+ is_capturing_ = true;
807
+ }
808
+
809
+ // Start capture thread
810
+ capture_thread_ = std::thread(&MicCapturePlugin::CaptureThread, this);
811
+
812
+ // Wait a bit to ensure thread has started
813
+ std::this_thread::sleep_for(std::chrono::milliseconds(200));
814
+
815
+ SendStatusUpdate(true, current_device_name_);
816
+
817
+ return true;
818
+ }
819
+
820
+ bool MicCapturePlugin::StopCapture() {
821
+ {
822
+ std::lock_guard<std::mutex> lock(mutex_);
823
+ if (!is_capturing_) {
824
+ return false;
825
+ }
826
+ should_stop_ = true;
827
+ }
828
+
829
+ if (capture_thread_.joinable()) {
830
+ capture_thread_.join();
831
+ }
832
+
833
+ {
834
+ std::lock_guard<std::mutex> lock(mutex_);
835
+ is_capturing_ = false;
836
+ current_device_name_.clear();
837
+
838
+ // Cleanup WASAPI resources
839
+ if (audio_client_) {
840
+ audio_client_->Stop();
841
+ }
842
+
843
+ if (capture_client_) {
844
+ capture_client_->Release();
845
+ capture_client_ = nullptr;
846
+ }
847
+
848
+ if (mix_format_) {
849
+ CoTaskMemFree(mix_format_);
850
+ mix_format_ = nullptr;
851
+ }
852
+
853
+ if (audio_client_) {
854
+ audio_client_->Release();
855
+ audio_client_ = nullptr;
856
+ }
857
+
858
+ if (device_) {
859
+ device_->Release();
860
+ device_ = nullptr;
861
+ }
862
+
863
+ // Only uninitialize COM if we initialized it
864
+ if (com_initialized_) {
865
+ CoUninitialize();
866
+ com_initialized_ = false;
867
+ }
868
+ }
869
+
870
+ // Wait a bit to ensure thread has fully stopped
871
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
872
+
873
+ SendStatusUpdate(false);
874
+
875
+ return true;
876
+ }
877
+
878
+ // UPDATED: CaptureThread - optimized for lower latency
879
+ void MicCapturePlugin::CaptureThread() {
880
+ try {
881
+ // Set thread priority to reduce latency
882
+ SetThreadPriority();
883
+
884
+ if (!mix_format_ || !capture_client_) {
885
+ return;
886
+ }
887
+
888
+ const UINT32 frame_size = mix_format_->nBlockAlign;
889
+ const UINT32 actual_sample_rate = mix_format_->nSamplesPerSec;
890
+ const WORD actual_channels = mix_format_->nChannels;
891
+ const WORD actual_bits_per_sample = mix_format_->wBitsPerSample;
892
+ const WORD format_tag = mix_format_->wFormatTag;
893
+
894
+ // Detect format
895
+ bool is_float_format = false;
896
+ bool is_pcm_format = false;
897
+
898
+ if (format_tag == WAVE_FORMAT_EXTENSIBLE && mix_format_->cbSize >= 22) {
899
+ WAVEFORMATEXTENSIBLE* wfex = reinterpret_cast<WAVEFORMATEXTENSIBLE*>(mix_format_);
900
+ if (IsEqualGUID(wfex->SubFormat, KSDATAFORMAT_SUBTYPE_IEEE_FLOAT)) {
901
+ is_float_format = true;
902
+ } else if (IsEqualGUID(wfex->SubFormat, KSDATAFORMAT_SUBTYPE_PCM)) {
903
+ is_pcm_format = true;
904
+ }
905
+ } else if (format_tag == WAVE_FORMAT_IEEE_FLOAT) {
906
+ is_float_format = true;
907
+ } else if (format_tag == WAVE_FORMAT_PCM) {
908
+ is_pcm_format = true;
909
+ }
910
+
911
+ // FIX LATENCY: Sử dụng chunk size nhỏ hơn (20-50ms) để giảm delay
912
+ const int effective_chunk_ms = 30; // 30ms chunk size for lower latency
913
+ const size_t chunk_frames = (actual_sample_rate * effective_chunk_ms / 1000);
914
+ const size_t chunk_size_bytes = chunk_frames * frame_size;
915
+ const size_t output_frame_count = (sample_rate_ * effective_chunk_ms / 1000);
916
+
917
+ std::vector<uint8_t> raw_buffer(chunk_size_bytes * 2); // Double buffer for safety
918
+ std::vector<int16_t> output_buffer(output_frame_count);
919
+ size_t raw_buffer_pos = 0;
920
+
921
+ while (!should_stop_) {
922
+ UINT32 num_frames_available = 0;
923
+ HRESULT hr = capture_client_->GetNextPacketSize(&num_frames_available);
924
+
925
+ if (FAILED(hr)) break;
926
+
927
+ while (num_frames_available > 0 && !should_stop_) {
928
+ BYTE* data = nullptr;
929
+ UINT32 num_frames = 0;
930
+ DWORD flags = 0;
931
+ UINT64 device_position = 0;
932
+ UINT64 qpc_position = 0;
933
+
934
+ hr = capture_client_->GetBuffer(&data, &num_frames, &flags,
935
+ &device_position, &qpc_position);
936
+ if (FAILED(hr)) break;
937
+
938
+ bool is_silent = (flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0;
939
+
940
+ if (!is_silent && data != nullptr && num_frames > 0) {
941
+ const size_t data_size = num_frames * frame_size;
942
+ size_t data_offset = 0;
943
+
944
+ while (data_offset < data_size && !should_stop_) {
945
+ const size_t space_available = raw_buffer.size() - raw_buffer_pos;
946
+ const size_t data_remaining = data_size - data_offset;
947
+ const size_t copy_size = (std::min)(space_available, data_remaining);
948
+
949
+ if (copy_size > 0) {
950
+ memcpy(raw_buffer.data() + raw_buffer_pos,
951
+ reinterpret_cast<const uint8_t*>(data) + data_offset,
952
+ copy_size);
953
+ raw_buffer_pos += copy_size;
954
+ data_offset += copy_size;
955
+ }
956
+
957
+ if (raw_buffer_pos >= chunk_size_bytes) {
958
+ const size_t input_frame_count = chunk_size_bytes / frame_size;
959
+ const size_t total_samples = input_frame_count * actual_channels;
960
+
961
+ std::vector<int16_t> converted_samples(total_samples);
962
+ bool conversion_success = false;
963
+
964
+ // Format conversion
965
+ if (is_pcm_format && actual_bits_per_sample == 16) {
966
+ const int16_t* raw_samples = reinterpret_cast<const int16_t*>(raw_buffer.data());
967
+ converted_samples.assign(raw_samples, raw_samples + total_samples);
968
+ conversion_success = true;
969
+ } else if (is_float_format && actual_bits_per_sample == 32) {
970
+ const float* float_samples = reinterpret_cast<const float*>(raw_buffer.data());
971
+ for (size_t i = 0; i < total_samples; ++i) {
972
+ float sample = (std::min)(1.0f, (std::max)(-1.0f, float_samples[i]));
973
+ converted_samples[i] = static_cast<int16_t>(sample * 32767.0f);
974
+ }
975
+ conversion_success = true;
976
+ } else if (is_pcm_format && actual_bits_per_sample == 24) {
977
+ const uint8_t* raw_bytes = raw_buffer.data();
978
+ for (size_t i = 0; i < total_samples; ++i) {
979
+ size_t byte_offset = i * 3;
980
+ int32_t sample24 = (static_cast<int32_t>(raw_bytes[byte_offset]) |
981
+ (static_cast<int32_t>(raw_bytes[byte_offset + 1]) << 8) |
982
+ (static_cast<int32_t>(raw_bytes[byte_offset + 2]) << 16));
983
+ if (sample24 & 0x800000) sample24 |= 0xFF000000;
984
+ converted_samples[i] = static_cast<int16_t>(sample24 >> 8);
985
+ }
986
+ conversion_success = true;
987
+ } else if (is_pcm_format && actual_bits_per_sample == 32) {
988
+ const int32_t* int32_samples = reinterpret_cast<const int32_t*>(raw_buffer.data());
989
+ for (size_t i = 0; i < total_samples; ++i) {
990
+ converted_samples[i] = static_cast<int16_t>(int32_samples[i] >> 16);
991
+ }
992
+ conversion_success = true;
993
+ }
994
+
995
+ if (!conversion_success) {
996
+ raw_buffer_pos = 0;
997
+ continue;
998
+ }
999
+
1000
+ if (input_volume_ > 0.0f && input_volume_ < 1.0f) {
1001
+ for (size_t i = 0; i < total_samples; ++i) {
1002
+ converted_samples[i] = static_cast<int16_t>(
1003
+ static_cast<float>(converted_samples[i]) * input_volume_);
1004
+ }
1005
+ }
1006
+
1007
+ const size_t input_frames = converted_samples.size() / actual_channels;
1008
+
1009
+ // First: Convert to mono and apply gain boost (at input sample rate)
1010
+ std::vector<int16_t> mono_buffer(input_frames);
1011
+ ApplyGainBoostAndConvertToMono(converted_samples.data(), mono_buffer.data(),
1012
+ input_frames, actual_channels, gain_boost_);
1013
+
1014
+ // Second: Resample if sample rates differ
1015
+ // Calculate correct output frames based on resampling ratio
1016
+ size_t output_frames;
1017
+ if (actual_sample_rate != static_cast<UINT32>(sample_rate_)) {
1018
+ // Calculate output frames after resampling
1019
+ output_frames = static_cast<size_t>(
1020
+ static_cast<double>(input_frames) * static_cast<double>(sample_rate_) /
1021
+ static_cast<double>(actual_sample_rate));
1022
+ output_frames = (std::min)(output_frames, output_frame_count);
1023
+
1024
+ // Resample from actual_sample_rate to sample_rate_
1025
+ ResampleAudio(mono_buffer.data(), input_frames,
1026
+ output_buffer.data(), output_frames,
1027
+ actual_sample_rate, sample_rate_);
1028
+ } else {
1029
+ // No resampling needed, just copy
1030
+ output_frames = (std::min)(input_frames, output_frame_count);
1031
+ memcpy(output_buffer.data(), mono_buffer.data(), output_frames * sizeof(int16_t));
1032
+ }
1033
+
1034
+ double decibel = CalculateDecibel(output_buffer.data(), output_frames);
1035
+
1036
+ // CHANGED: Queue instead of direct send
1037
+ const size_t output_bytes = output_frames * sizeof(int16_t);
1038
+ std::vector<uint8_t> audio_data(
1039
+ reinterpret_cast<uint8_t*>(output_buffer.data()),
1040
+ reinterpret_cast<uint8_t*>(output_buffer.data()) + output_bytes);
1041
+
1042
+ QueueAudioData(std::move(audio_data), decibel);
1043
+
1044
+ if (raw_buffer_pos > chunk_size_bytes) {
1045
+ const size_t remaining = raw_buffer_pos - chunk_size_bytes;
1046
+ memmove(raw_buffer.data(), raw_buffer.data() + chunk_size_bytes, remaining);
1047
+ raw_buffer_pos = remaining;
1048
+ } else {
1049
+ raw_buffer_pos = 0;
1050
+ }
1051
+ }
1052
+ }
1053
+ }
1054
+
1055
+ hr = capture_client_->ReleaseBuffer(num_frames);
1056
+ if (FAILED(hr)) break;
1057
+
1058
+ hr = capture_client_->GetNextPacketSize(&num_frames_available);
1059
+ if (FAILED(hr)) break;
1060
+ }
1061
+
1062
+ // FIX LATENCY: Giảm sleep time xuống 1ms để phản hồi nhanh hơn
1063
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
1064
+ }
1065
+ } catch (...) {
1066
+ std::lock_guard<std::mutex> lock(mutex_);
1067
+ is_capturing_ = false;
1068
+ }
1069
+ }
1070
+
1071
+ // Resample audio using linear interpolation
1072
+ void MicCapturePlugin::ResampleAudio(const int16_t* input, size_t input_frames,
1073
+ int16_t* output, size_t output_frames,
1074
+ int input_sample_rate, int output_sample_rate) {
1075
+ if (input_frames == 0 || output_frames == 0) {
1076
+ return;
1077
+ }
1078
+
1079
+ if (input_sample_rate == output_sample_rate) {
1080
+ const size_t copy_frames = (std::min)(input_frames, output_frames);
1081
+ memcpy(output, input, copy_frames * sizeof(int16_t));
1082
+ return;
1083
+ }
1084
+
1085
+ // Linear interpolation resampling
1086
+ const double ratio = static_cast<double>(input_sample_rate) / static_cast<double>(output_sample_rate);
1087
+
1088
+ for (size_t i = 0; i < output_frames; ++i) {
1089
+ const double src_pos = static_cast<double>(i) * ratio;
1090
+ const size_t src_index = static_cast<size_t>(src_pos);
1091
+ const double fraction = src_pos - static_cast<double>(src_index);
1092
+
1093
+ if (src_index + 1 < input_frames) {
1094
+ // Linear interpolation between two samples
1095
+ const double sample0 = static_cast<double>(input[src_index]);
1096
+ const double sample1 = static_cast<double>(input[src_index + 1]);
1097
+ const double interpolated = sample0 + (sample1 - sample0) * fraction;
1098
+ output[i] = static_cast<int16_t>((std::max)(-32768.0, (std::min)(32767.0, interpolated)));
1099
+ } else if (src_index < input_frames) {
1100
+ // Last sample, no interpolation
1101
+ output[i] = input[src_index];
1102
+ } else {
1103
+ // Beyond input, use last sample
1104
+ output[i] = input[input_frames - 1];
1105
+ }
1106
+ }
1107
+ }
1108
+
1109
+ // Set high priority for capture thread to reduce latency
1110
+ void MicCapturePlugin::SetThreadPriority() {
1111
+ HANDLE current_thread = GetCurrentThread();
1112
+ // Sử dụng THREAD_PRIORITY_HIGHEST để giảm latency
1113
+ ::SetThreadPriority(current_thread, THREAD_PRIORITY_HIGHEST);
1114
+ }
1115
+
1116
+ } // namespace audio_capture
1117
+