neoagent 2.3.1-beta.2 → 2.3.1-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +39 -0
- package/README.md +2 -0
- package/docs/capabilities.md +2 -2
- package/docs/configuration.md +13 -5
- package/docs/integrations.md +4 -1
- package/flutter_app/.metadata +42 -0
- package/flutter_app/README.md +21 -0
- package/flutter_app/analysis_options.yaml +32 -0
- package/flutter_app/android/app/build.gradle.kts +109 -0
- package/flutter_app/android/app/src/debug/AndroidManifest.xml +7 -0
- package/flutter_app/android/app/src/main/AndroidManifest.xml +147 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +747 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthConnectGateway.kt +280 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncNotifications.kt +113 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncPayload.kt +57 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncScheduler.kt +78 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncWorker.kt +253 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/PermissionsRationaleActivity.kt +46 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +21 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +586 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +78 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +104 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiHomeWidgetProvider.kt +457 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiWidgetStore.kt +194 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/VoiceLaunchWidgetProvider.kt +67 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetConfigActivity.kt +228 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncScheduler.kt +72 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncWorker.kt +186 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetTaskRunWorker.kt +210 -0
- package/flutter_app/android/app/src/main/res/drawable/launch_background.xml +12 -0
- package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_bg.xml +11 -0
- package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_task_bg.xml +8 -0
- package/flutter_app/android/app/src/main/res/drawable-v21/launch_background.xml +12 -0
- package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget.xml +138 -0
- package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget_task_row.xml +52 -0
- package/flutter_app/android/app/src/main/res/layout/neoagent_voice_widget.xml +49 -0
- package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/values/strings.xml +12 -0
- package/flutter_app/android/app/src/main/res/values/styles.xml +18 -0
- package/flutter_app/android/app/src/main/res/values-night/styles.xml +18 -0
- package/flutter_app/android/app/src/main/res/xml/file_paths.xml +6 -0
- package/flutter_app/android/app/src/main/res/xml/neoagent_ai_widget_info.xml +12 -0
- package/flutter_app/android/app/src/main/res/xml/neoagent_voice_widget_info.xml +12 -0
- package/flutter_app/android/app/src/profile/AndroidManifest.xml +7 -0
- package/flutter_app/android/build.gradle.kts +24 -0
- package/flutter_app/android/ci-release.keystore +0 -0
- package/flutter_app/android/gradle/wrapper/gradle-wrapper.properties +5 -0
- package/flutter_app/android/gradle.properties +3 -0
- package/flutter_app/android/key.properties +4 -0
- package/flutter_app/android/settings.gradle.kts +26 -0
- package/flutter_app/assets/branding/app_icon_1024.png +0 -0
- package/flutter_app/assets/branding/app_icon_128.png +0 -0
- package/flutter_app/assets/branding/app_icon_192.png +0 -0
- package/flutter_app/assets/branding/app_icon_256.png +0 -0
- package/flutter_app/assets/branding/app_icon_32.png +0 -0
- package/flutter_app/assets/branding/app_icon_512.png +0 -0
- package/flutter_app/assets/branding/app_icon_64.png +0 -0
- package/flutter_app/assets/branding/tray_icon_template.png +0 -0
- package/flutter_app/lib/features/location/location_service.dart +119 -0
- package/flutter_app/lib/features/notifications/notification_interceptor.dart +97 -0
- package/flutter_app/lib/main.dart +23057 -0
- package/flutter_app/lib/main_app_shell.dart +1682 -0
- package/flutter_app/lib/main_integrations.dart +931 -0
- package/flutter_app/lib/main_launcher.dart +959 -0
- package/flutter_app/lib/main_launcher_entry.dart +5 -0
- package/flutter_app/lib/main_models.dart +3473 -0
- package/flutter_app/lib/main_shared.dart +2861 -0
- package/flutter_app/lib/main_theme.dart +204 -0
- package/flutter_app/lib/main_voice_assistant.dart +831 -0
- package/flutter_app/lib/src/android_apk_drop_zone.dart +32 -0
- package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +16 -0
- package/flutter_app/lib/src/android_apk_drop_zone_web.dart +348 -0
- package/flutter_app/lib/src/android_app_installer.dart +22 -0
- package/flutter_app/lib/src/android_app_installer_io.dart +122 -0
- package/flutter_app/lib/src/android_app_installer_stub.dart +21 -0
- package/flutter_app/lib/src/android_launcher_bridge.dart +239 -0
- package/flutter_app/lib/src/app_launch_bridge.dart +29 -0
- package/flutter_app/lib/src/app_release_updater.dart +511 -0
- package/flutter_app/lib/src/backend_client.dart +1833 -0
- package/flutter_app/lib/src/desktop_companion.dart +2 -0
- package/flutter_app/lib/src/desktop_companion_actions.dart +586 -0
- package/flutter_app/lib/src/desktop_companion_io.dart +538 -0
- package/flutter_app/lib/src/desktop_companion_stub.dart +59 -0
- package/flutter_app/lib/src/desktop_native_bridge.dart +91 -0
- package/flutter_app/lib/src/desktop_screen_capture.dart +21 -0
- package/flutter_app/lib/src/desktop_screen_capture_io.dart +142 -0
- package/flutter_app/lib/src/desktop_screen_capture_stub.dart +12 -0
- package/flutter_app/lib/src/diagnostics_logger.dart +119 -0
- package/flutter_app/lib/src/health_bridge.dart +136 -0
- package/flutter_app/lib/src/live_voice_capture.dart +85 -0
- package/flutter_app/lib/src/messaging_access_summary.dart +46 -0
- package/flutter_app/lib/src/network/app_http_client.dart +53 -0
- package/flutter_app/lib/src/network/app_http_client_factory.dart +6 -0
- package/flutter_app/lib/src/network/app_http_client_io.dart +138 -0
- package/flutter_app/lib/src/network/app_http_client_stub.dart +3 -0
- package/flutter_app/lib/src/network/app_http_client_web.dart +94 -0
- package/flutter_app/lib/src/oauth_launcher.dart +33 -0
- package/flutter_app/lib/src/oauth_launcher_io.dart +77 -0
- package/flutter_app/lib/src/oauth_launcher_stub.dart +33 -0
- package/flutter_app/lib/src/oauth_launcher_web.dart +107 -0
- package/flutter_app/lib/src/recording_bridge.dart +232 -0
- package/flutter_app/lib/src/recording_bridge_io.dart +1019 -0
- package/flutter_app/lib/src/recording_bridge_stub.dart +120 -0
- package/flutter_app/lib/src/recording_bridge_web.dart +689 -0
- package/flutter_app/lib/src/recording_payloads.dart +86 -0
- package/flutter_app/lib/src/theme/palette.dart +81 -0
- package/flutter_app/lib/src/widget_bridge.dart +49 -0
- package/flutter_app/linux/CMakeLists.txt +128 -0
- package/flutter_app/linux/flutter/CMakeLists.txt +88 -0
- package/flutter_app/linux/flutter/generated_plugin_registrant.cc +43 -0
- package/flutter_app/linux/flutter/generated_plugin_registrant.h +15 -0
- package/flutter_app/linux/flutter/generated_plugins.cmake +31 -0
- package/flutter_app/linux/runner/CMakeLists.txt +26 -0
- package/flutter_app/linux/runner/main.cc +6 -0
- package/flutter_app/linux/runner/my_application.cc +144 -0
- package/flutter_app/linux/runner/my_application.h +18 -0
- package/flutter_app/linux/runner/resources/app_icon.png +0 -0
- package/flutter_app/macos/Flutter/Flutter-Debug.xcconfig +2 -0
- package/flutter_app/macos/Flutter/Flutter-Release.xcconfig +2 -0
- package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +40 -0
- package/flutter_app/macos/Podfile +42 -0
- package/flutter_app/macos/Podfile.lock +87 -0
- package/flutter_app/macos/Runner/AppDelegate.swift +576 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json +68 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
- package/flutter_app/macos/Runner/Base.lproj/MainMenu.xib +342 -0
- package/flutter_app/macos/Runner/Configs/AppInfo.xcconfig +14 -0
- package/flutter_app/macos/Runner/Configs/Debug.xcconfig +2 -0
- package/flutter_app/macos/Runner/Configs/Release.xcconfig +2 -0
- package/flutter_app/macos/Runner/Configs/Warnings.xcconfig +13 -0
- package/flutter_app/macos/Runner/DebugProfile.entitlements +16 -0
- package/flutter_app/macos/Runner/Info.plist +36 -0
- package/flutter_app/macos/Runner/MainFlutterWindow.swift +19 -0
- package/flutter_app/macos/Runner/Release.entitlements +12 -0
- package/flutter_app/macos/Runner.xcodeproj/project.pbxproj +801 -0
- package/flutter_app/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
- package/flutter_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +99 -0
- package/flutter_app/macos/Runner.xcworkspace/contents.xcworkspacedata +10 -0
- package/flutter_app/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
- package/flutter_app/macos/RunnerTests/RunnerTests.swift +12 -0
- package/flutter_app/patch_strings.py +12 -0
- package/flutter_app/pubspec.lock +1088 -0
- package/flutter_app/pubspec.yaml +53 -0
- package/flutter_app/test/messaging_access_summary_test.dart +22 -0
- package/flutter_app/test/recording_payloads_test.dart +53 -0
- package/flutter_app/third_party/desktop_audio_capture/LICENSE +21 -0
- package/flutter_app/third_party/desktop_audio_capture/README.md +262 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +65 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +153 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +110 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +461 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +91 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +106 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +219 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +336 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +101 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +692 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +35 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +36 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +32 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +32 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +878 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +27 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +1172 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +655 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +12 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +30 -0
- package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +87 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +105 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +80 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +31 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +12 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +23 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +25 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +1117 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +115 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +777 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +87 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/linux/CMakeLists.txt +30 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/linux/flutter_secure_storage_linux_plugin.cc +215 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/linux/include/flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h +27 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/pubspec.yaml +20 -0
- package/flutter_app/tool/generate_desktop_branding.py +219 -0
- package/flutter_app/web/favicon.png +0 -0
- package/flutter_app/web/favicon.svg +12 -0
- package/flutter_app/web/icons/Icon-192.png +0 -0
- package/flutter_app/web/icons/Icon-512.png +0 -0
- package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
- package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
- package/flutter_app/web/index.html +39 -0
- package/flutter_app/web/manifest.json +35 -0
- package/flutter_app/windows/CMakeLists.txt +108 -0
- package/flutter_app/windows/flutter/CMakeLists.txt +109 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.cc +47 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.h +15 -0
- package/flutter_app/windows/flutter/generated_plugins.cmake +35 -0
- package/flutter_app/windows/runner/CMakeLists.txt +41 -0
- package/flutter_app/windows/runner/Runner.rc +121 -0
- package/flutter_app/windows/runner/flutter_window.cpp +533 -0
- package/flutter_app/windows/runner/flutter_window.h +37 -0
- package/flutter_app/windows/runner/main.cpp +53 -0
- package/flutter_app/windows/runner/resource.h +16 -0
- package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
- package/flutter_app/windows/runner/runner.exe.manifest +14 -0
- package/flutter_app/windows/runner/utils.cpp +65 -0
- package/flutter_app/windows/runner/utils.h +19 -0
- package/flutter_app/windows/runner/win32_window.cpp +299 -0
- package/flutter_app/windows/runner/win32_window.h +102 -0
- package/lib/manager.js +231 -7
- package/package.json +3 -1
- package/server/db/database.js +68 -0
- package/server/http/middleware.js +50 -0
- package/server/http/routes.js +3 -1
- package/server/index.js +1 -0
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/NOTICES +61 -0
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +65262 -64422
- package/server/routes/integrations.js +86 -0
- package/server/routes/memory.js +11 -2
- package/server/routes/screenHistory.js +46 -0
- package/server/routes/triggers.js +81 -0
- package/server/services/ai/models.js +30 -0
- package/server/services/ai/providers/githubCopilot.js +97 -0
- package/server/services/ai/providers/openai.js +2 -1
- package/server/services/ai/providers/openaiCodex.js +31 -0
- package/server/services/ai/settings.js +20 -0
- package/server/services/ai/systemPrompt.js +1 -1
- package/server/services/ai/tools.js +35 -6
- package/server/services/browser/controller.js +47 -3
- package/server/services/desktop/screenRecorder.js +172 -0
- package/server/services/integrations/env.js +5 -0
- package/server/services/integrations/github/common.js +106 -0
- package/server/services/integrations/github/provider.js +499 -0
- package/server/services/integrations/github/repos.js +1124 -0
- package/server/services/integrations/home_assistant/provider.js +306 -26
- package/server/services/integrations/manager.js +63 -7
- package/server/services/integrations/oauth_provider.js +13 -6
- package/server/services/integrations/provider_config_store.js +76 -0
- package/server/services/integrations/registry.js +4 -0
- package/server/services/integrations/trello/provider.js +744 -0
- package/server/services/integrations/whatsapp/provider.js +6 -2
- package/server/services/manager.js +22 -0
- package/server/services/memory/manager.js +39 -2
- package/server/services/skills/base_catalog.js +33 -0
- package/server/services/tasks/adapters/index.js +1 -0
- package/server/services/tasks/adapters/manual.js +12 -0
- package/server/services/tasks/runtime.js +1 -1
- package/server/services/voice/openaiClient.js +4 -1
- package/server/services/voice/providers.js +2 -1
- package/server/services/widgets/service.js +49 -4
- package/server/utils/local_secrets.js +56 -0
- package/server/utils/logger.js +37 -9
|
@@ -0,0 +1,1117 @@
|
|
|
1
|
+
#include "mic_capture_plugin.h"
|
|
2
|
+
|
|
3
|
+
#ifndef NOMINMAX
|
|
4
|
+
#define NOMINMAX
|
|
5
|
+
#endif
|
|
6
|
+
#include <windows.h>
|
|
7
|
+
#undef max
|
|
8
|
+
#undef min
|
|
9
|
+
#include <mmdeviceapi.h>
|
|
10
|
+
#include <audioclient.h>
|
|
11
|
+
#include <functiondiscoverykeys_devpkey.h>
|
|
12
|
+
#include <VersionHelpers.h>
|
|
13
|
+
#include <comdef.h>
|
|
14
|
+
#include <propkey.h>
|
|
15
|
+
#include <propvarutil.h>
|
|
16
|
+
|
|
17
|
+
#include <flutter/event_channel.h>
|
|
18
|
+
#include <flutter/method_channel.h>
|
|
19
|
+
#include <flutter/plugin_registrar_windows.h>
|
|
20
|
+
#include <flutter/standard_method_codec.h>
|
|
21
|
+
|
|
22
|
+
#include <algorithm>
|
|
23
|
+
#include <chrono>
|
|
24
|
+
#include <cmath>
|
|
25
|
+
#include <cstdint>
|
|
26
|
+
#include <functional>
|
|
27
|
+
#include <memory>
|
|
28
|
+
#include <sstream>
|
|
29
|
+
#include <string>
|
|
30
|
+
#include <thread>
|
|
31
|
+
#include <vector>
|
|
32
|
+
// After existing includes, add:
|
|
33
|
+
#include <queue>
|
|
34
|
+
#include <chrono>
|
|
35
|
+
|
|
36
|
+
#pragma comment(lib, "ole32.lib")
|
|
37
|
+
#pragma comment(lib, "oleaut32.lib")
|
|
38
|
+
|
|
39
|
+
// REFTIMES_PER_SEC is 10,000,000 (100 nanoseconds per second)
|
|
40
|
+
#ifndef REFTIMES_PER_SEC
|
|
41
|
+
#define REFTIMES_PER_SEC 10000000
|
|
42
|
+
#endif
|
|
43
|
+
|
|
44
|
+
namespace audio_capture {
|
|
45
|
+
|
|
46
|
+
// Custom StreamHandler implementation
|
|
47
|
+
template <typename T>
|
|
48
|
+
class StreamHandlerFunctions : public flutter::StreamHandler<T> {
|
|
49
|
+
public:
|
|
50
|
+
using OnListenHandler = std::function<std::unique_ptr<flutter::StreamHandlerError<T>>(
|
|
51
|
+
const T* arguments,
|
|
52
|
+
std::unique_ptr<flutter::EventSink<T>>&& events)>;
|
|
53
|
+
using OnCancelHandler = std::function<std::unique_ptr<flutter::StreamHandlerError<T>>(
|
|
54
|
+
const T* arguments)>;
|
|
55
|
+
|
|
56
|
+
StreamHandlerFunctions(OnListenHandler on_listen, OnCancelHandler on_cancel)
|
|
57
|
+
: on_listen_(std::move(on_listen)), on_cancel_(std::move(on_cancel)) {}
|
|
58
|
+
|
|
59
|
+
virtual ~StreamHandlerFunctions() = default;
|
|
60
|
+
|
|
61
|
+
protected:
|
|
62
|
+
std::unique_ptr<flutter::StreamHandlerError<T>> OnListenInternal(
|
|
63
|
+
const T* arguments,
|
|
64
|
+
std::unique_ptr<flutter::EventSink<T>>&& events) override {
|
|
65
|
+
return on_listen_(arguments, std::move(events));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
std::unique_ptr<flutter::StreamHandlerError<T>> OnCancelInternal(
|
|
69
|
+
const T* arguments) override {
|
|
70
|
+
return on_cancel_(arguments);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private:
|
|
74
|
+
OnListenHandler on_listen_;
|
|
75
|
+
OnCancelHandler on_cancel_;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
namespace {
|
|
79
|
+
|
|
80
|
+
constexpr char kMethodChannelName[] = "com.mic_audio_transcriber/mic_capture";
|
|
81
|
+
constexpr char kEventChannelName[] = "com.mic_audio_transcriber/mic_stream";
|
|
82
|
+
constexpr char kStatusEventChannelName[] = "com.mic_audio_transcriber/mic_status";
|
|
83
|
+
constexpr char kDecibelEventChannelName[] = "com.mic_audio_transcriber/mic_decibel";
|
|
84
|
+
|
|
85
|
+
constexpr int kDefaultSampleRate = 16000;
|
|
86
|
+
constexpr int kDefaultChannels = 1;
|
|
87
|
+
constexpr int kDefaultBitsPerSample = 16;
|
|
88
|
+
constexpr float kDefaultGainBoost = 2.5f;
|
|
89
|
+
constexpr float kDefaultInputVolume = 1.0f;
|
|
90
|
+
|
|
91
|
+
} // namespace
|
|
92
|
+
|
|
93
|
+
// static
|
|
94
|
+
void MicCapturePlugin::RegisterWithRegistrar(
|
|
95
|
+
flutter::PluginRegistrarWindows *registrar) {
|
|
96
|
+
auto plugin = std::make_unique<MicCapturePlugin>(registrar);
|
|
97
|
+
registrar->AddPlugin(std::move(plugin));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
MicCapturePlugin::MicCapturePlugin(flutter::PluginRegistrarWindows *registrar)
|
|
101
|
+
: registrar_(registrar),
|
|
102
|
+
is_capturing_(false),
|
|
103
|
+
should_stop_(false),
|
|
104
|
+
sample_rate_(kDefaultSampleRate),
|
|
105
|
+
channels_(kDefaultChannels),
|
|
106
|
+
bits_per_sample_(kDefaultBitsPerSample),
|
|
107
|
+
gain_boost_(kDefaultGainBoost),
|
|
108
|
+
input_volume_(kDefaultInputVolume),
|
|
109
|
+
audio_client_(nullptr),
|
|
110
|
+
capture_client_(nullptr),
|
|
111
|
+
device_(nullptr),
|
|
112
|
+
mix_format_(nullptr),
|
|
113
|
+
buffer_frame_count_(0),
|
|
114
|
+
com_initialized_(false) {
|
|
115
|
+
// Create method channel
|
|
116
|
+
method_channel_ =
|
|
117
|
+
std::make_unique<flutter::MethodChannel<flutter::EncodableValue>>(
|
|
118
|
+
registrar->messenger(), kMethodChannelName,
|
|
119
|
+
&flutter::StandardMethodCodec::GetInstance());
|
|
120
|
+
|
|
121
|
+
method_channel_->SetMethodCallHandler(
|
|
122
|
+
[this](const auto &call, auto result) {
|
|
123
|
+
this->HandleMethodCall(call, std::move(result));
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
// Create event channels
|
|
127
|
+
event_channel_ =
|
|
128
|
+
std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
|
|
129
|
+
registrar->messenger(), kEventChannelName,
|
|
130
|
+
&flutter::StandardMethodCodec::GetInstance());
|
|
131
|
+
|
|
132
|
+
event_channel_->SetStreamHandler(
|
|
133
|
+
std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
|
|
134
|
+
[this](const flutter::EncodableValue* arguments,
|
|
135
|
+
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
|
|
136
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
137
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
138
|
+
event_sink_ = std::move(events);
|
|
139
|
+
return nullptr;
|
|
140
|
+
},
|
|
141
|
+
[this](const flutter::EncodableValue* arguments)
|
|
142
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
143
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
144
|
+
event_sink_.reset();
|
|
145
|
+
return nullptr;
|
|
146
|
+
}));
|
|
147
|
+
|
|
148
|
+
status_event_channel_ =
|
|
149
|
+
std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
|
|
150
|
+
registrar->messenger(), kStatusEventChannelName,
|
|
151
|
+
&flutter::StandardMethodCodec::GetInstance());
|
|
152
|
+
|
|
153
|
+
status_event_channel_->SetStreamHandler(
|
|
154
|
+
std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
|
|
155
|
+
[this](const flutter::EncodableValue* arguments,
|
|
156
|
+
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
|
|
157
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
158
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
159
|
+
status_event_sink_ = std::move(events);
|
|
160
|
+
SendStatusUpdate(is_capturing_, current_device_name_);
|
|
161
|
+
return nullptr;
|
|
162
|
+
},
|
|
163
|
+
[this](const flutter::EncodableValue* arguments)
|
|
164
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
165
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
166
|
+
status_event_sink_.reset();
|
|
167
|
+
return nullptr;
|
|
168
|
+
}));
|
|
169
|
+
|
|
170
|
+
decibel_event_channel_ =
|
|
171
|
+
std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
|
|
172
|
+
registrar->messenger(), kDecibelEventChannelName,
|
|
173
|
+
&flutter::StandardMethodCodec::GetInstance());
|
|
174
|
+
|
|
175
|
+
decibel_event_channel_->SetStreamHandler(
|
|
176
|
+
std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
|
|
177
|
+
[this](const flutter::EncodableValue* arguments,
|
|
178
|
+
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
|
|
179
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
180
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
181
|
+
decibel_event_sink_ = std::move(events);
|
|
182
|
+
return nullptr;
|
|
183
|
+
},
|
|
184
|
+
[this](const flutter::EncodableValue* arguments)
|
|
185
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
186
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
187
|
+
decibel_event_sink_.reset();
|
|
188
|
+
return nullptr;
|
|
189
|
+
}));
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
MicCapturePlugin::~MicCapturePlugin() {
|
|
193
|
+
StopCapture();
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// NEW: Queue audio data from background thread
|
|
197
|
+
void MicCapturePlugin::QueueAudioData(std::vector<uint8_t> data, double decibel) {
|
|
198
|
+
std::lock_guard<std::mutex> lock(queue_mutex_);
|
|
199
|
+
|
|
200
|
+
// Prevent queue overflow
|
|
201
|
+
if (audio_queue_.size() >= kMaxQueueSize) {
|
|
202
|
+
audio_queue_.pop(); // Remove oldest
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
AudioDataPacket packet;
|
|
206
|
+
packet.data = std::move(data);
|
|
207
|
+
packet.decibel = decibel;
|
|
208
|
+
packet.timestamp = std::chrono::steady_clock::now();
|
|
209
|
+
|
|
210
|
+
audio_queue_.push(std::move(packet));
|
|
211
|
+
|
|
212
|
+
// Post task to platform thread to process queue
|
|
213
|
+
registrar_->messenger()->Send(
|
|
214
|
+
"", // Empty channel name - just for triggering callback
|
|
215
|
+
nullptr,
|
|
216
|
+
0,
|
|
217
|
+
[this](const uint8_t* reply, size_t reply_size) {
|
|
218
|
+
this->ProcessQueue();
|
|
219
|
+
}
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// NEW: Process queue on platform thread
|
|
224
|
+
void MicCapturePlugin::ProcessQueue() {
|
|
225
|
+
std::lock_guard<std::mutex> lock(queue_mutex_);
|
|
226
|
+
|
|
227
|
+
// Process all queued packets
|
|
228
|
+
while (!audio_queue_.empty()) {
|
|
229
|
+
AudioDataPacket& packet = audio_queue_.front();
|
|
230
|
+
|
|
231
|
+
// Send audio data
|
|
232
|
+
{
|
|
233
|
+
std::lock_guard<std::mutex> sink_lock(mutex_);
|
|
234
|
+
if (event_sink_) {
|
|
235
|
+
try {
|
|
236
|
+
event_sink_->Success(flutter::EncodableValue(packet.data));
|
|
237
|
+
} catch (...) {
|
|
238
|
+
// Ignore errors
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// Send decibel data
|
|
244
|
+
{
|
|
245
|
+
std::lock_guard<std::mutex> sink_lock(mutex_);
|
|
246
|
+
if (decibel_event_sink_) {
|
|
247
|
+
try {
|
|
248
|
+
flutter::EncodableMap decibel_map;
|
|
249
|
+
decibel_map[flutter::EncodableValue("decibel")] =
|
|
250
|
+
flutter::EncodableValue(packet.decibel);
|
|
251
|
+
|
|
252
|
+
auto now = std::chrono::system_clock::now();
|
|
253
|
+
auto duration = now.time_since_epoch();
|
|
254
|
+
auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
|
|
255
|
+
double timestamp = static_cast<double>(ms) / 1000.0;
|
|
256
|
+
|
|
257
|
+
decibel_map[flutter::EncodableValue("timestamp")] =
|
|
258
|
+
flutter::EncodableValue(timestamp);
|
|
259
|
+
|
|
260
|
+
decibel_event_sink_->Success(flutter::EncodableValue(decibel_map));
|
|
261
|
+
} catch (...) {
|
|
262
|
+
// Ignore errors
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
audio_queue_.pop();
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
void MicCapturePlugin::HandleMethodCall(
|
|
272
|
+
const flutter::MethodCall<flutter::EncodableValue> &method_call,
|
|
273
|
+
std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result) {
|
|
274
|
+
if (method_call.method_name().compare("requestPermissions") == 0) {
|
|
275
|
+
// On Windows, permissions are typically handled by the system
|
|
276
|
+
result->Success(flutter::EncodableValue(true));
|
|
277
|
+
} else if (method_call.method_name().compare("hasInputDevice") == 0) {
|
|
278
|
+
bool has_device = HasInputDevice();
|
|
279
|
+
result->Success(flutter::EncodableValue(has_device));
|
|
280
|
+
} else if (method_call.method_name().compare("isSupported") == 0) {
|
|
281
|
+
bool has_device = HasInputDevice();
|
|
282
|
+
result->Success(flutter::EncodableValue(has_device));
|
|
283
|
+
} else if (method_call.method_name().compare("checkMicSupport") == 0) {
|
|
284
|
+
bool has_device = HasInputDevice();
|
|
285
|
+
result->Success(flutter::EncodableValue(has_device));
|
|
286
|
+
} else if (method_call.method_name().compare("getAvailableInputDevices") == 0) {
|
|
287
|
+
auto devices = GetAvailableInputDevices();
|
|
288
|
+
result->Success(flutter::EncodableValue(devices));
|
|
289
|
+
} else if (method_call.method_name().compare("startCapture") == 0) {
|
|
290
|
+
const flutter::EncodableMap* args = nullptr;
|
|
291
|
+
if (method_call.arguments() &&
|
|
292
|
+
std::holds_alternative<flutter::EncodableMap>(*method_call.arguments())) {
|
|
293
|
+
args = &std::get<flutter::EncodableMap>(*method_call.arguments());
|
|
294
|
+
}
|
|
295
|
+
bool started = StartCapture(args);
|
|
296
|
+
result->Success(flutter::EncodableValue(started));
|
|
297
|
+
} else if (method_call.method_name().compare("stopCapture") == 0) {
|
|
298
|
+
bool stopped = StopCapture();
|
|
299
|
+
result->Success(flutter::EncodableValue(stopped));
|
|
300
|
+
} else {
|
|
301
|
+
result->NotImplemented();
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
void MicCapturePlugin::ApplyGainBoostAndConvertToMono(
|
|
306
|
+
const int16_t* input, int16_t* output, size_t frame_count,
|
|
307
|
+
int input_channels, float gain_boost) {
|
|
308
|
+
const float max_value = 32767.0f;
|
|
309
|
+
const float min_value = -32768.0f;
|
|
310
|
+
|
|
311
|
+
if (input_channels == 1) {
|
|
312
|
+
// Mono: just apply gain boost
|
|
313
|
+
for (size_t i = 0; i < frame_count; ++i) {
|
|
314
|
+
float sample = static_cast<float>(input[i]) * gain_boost;
|
|
315
|
+
float clamped_sample = (std::min)(max_value, sample);
|
|
316
|
+
sample = (std::max)(min_value, clamped_sample);
|
|
317
|
+
output[i] = static_cast<int16_t>(sample);
|
|
318
|
+
}
|
|
319
|
+
} else {
|
|
320
|
+
// Stereo: convert to mono and apply gain boost
|
|
321
|
+
for (size_t i = 0; i < frame_count; ++i) {
|
|
322
|
+
float left = static_cast<float>(input[i * 2]);
|
|
323
|
+
float right = static_cast<float>(input[i * 2 + 1]);
|
|
324
|
+
float mono = (left + right) / 2.0f * gain_boost;
|
|
325
|
+
float clamped_mono = (std::min)(max_value, mono);
|
|
326
|
+
mono = (std::max)(min_value, clamped_mono);
|
|
327
|
+
output[i] = static_cast<int16_t>(mono);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
double MicCapturePlugin::CalculateDecibel(const int16_t* samples,
|
|
333
|
+
size_t sample_count) {
|
|
334
|
+
if (sample_count == 0) {
|
|
335
|
+
return -120.0;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// Calculate RMS (Root Mean Square)
|
|
339
|
+
double sum_of_squares = 0.0;
|
|
340
|
+
for (size_t i = 0; i < sample_count; ++i) {
|
|
341
|
+
double value = static_cast<double>(samples[i]);
|
|
342
|
+
sum_of_squares += value * value;
|
|
343
|
+
}
|
|
344
|
+
double mean_square = sum_of_squares / static_cast<double>(sample_count);
|
|
345
|
+
double rms = sqrt(mean_square);
|
|
346
|
+
|
|
347
|
+
// Calculate decibel: dB = 20 * log10(RMS / max_value)
|
|
348
|
+
const double max_value = 32767.0;
|
|
349
|
+
if (rms <= 0.0) {
|
|
350
|
+
return -120.0; // Avoid log(0)
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
double decibel = 20.0 * log10(rms / max_value);
|
|
354
|
+
|
|
355
|
+
// Clamp to reasonable range (-120 dB to 0 dB)
|
|
356
|
+
double clamped_decibel = (std::min)(0.0, decibel);
|
|
357
|
+
return (std::max)(-120.0, clamped_decibel);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// UPDATED: SendStatusUpdate - also post to platform thread
|
|
361
|
+
void MicCapturePlugin::SendStatusUpdate(bool is_active, const std::string& device_name) {
|
|
362
|
+
registrar_->messenger()->Send(
|
|
363
|
+
"",
|
|
364
|
+
nullptr,
|
|
365
|
+
0,
|
|
366
|
+
[this, is_active, device_name](const uint8_t* reply, size_t reply_size) {
|
|
367
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
368
|
+
if (status_event_sink_) {
|
|
369
|
+
try {
|
|
370
|
+
flutter::EncodableMap status_map;
|
|
371
|
+
status_map[flutter::EncodableValue("isActive")] = flutter::EncodableValue(is_active);
|
|
372
|
+
|
|
373
|
+
auto now = std::chrono::system_clock::now();
|
|
374
|
+
auto duration = now.time_since_epoch();
|
|
375
|
+
auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
|
|
376
|
+
double timestamp = static_cast<double>(ms) / 1000.0;
|
|
377
|
+
|
|
378
|
+
status_map[flutter::EncodableValue("timestamp")] = flutter::EncodableValue(timestamp);
|
|
379
|
+
|
|
380
|
+
if (!device_name.empty()) {
|
|
381
|
+
status_map[flutter::EncodableValue("deviceName")] = flutter::EncodableValue(device_name);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
status_event_sink_->Success(flutter::EncodableValue(status_map));
|
|
385
|
+
} catch (...) {}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
std::string MicCapturePlugin::GetCurrentDeviceName() {
|
|
392
|
+
if (!device_) {
|
|
393
|
+
return "Default Microphone";
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
IPropertyStore* props = nullptr;
|
|
397
|
+
HRESULT hr = device_->OpenPropertyStore(STGM_READ, &props);
|
|
398
|
+
if (FAILED(hr)) {
|
|
399
|
+
return "Default Microphone";
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
PROPVARIANT varName;
|
|
403
|
+
PropVariantInit(&varName);
|
|
404
|
+
hr = props->GetValue(PKEY_Device_FriendlyName, &varName);
|
|
405
|
+
props->Release();
|
|
406
|
+
|
|
407
|
+
std::string device_name = "Default Microphone";
|
|
408
|
+
if (SUCCEEDED(hr) && varName.vt == VT_LPWSTR) {
|
|
409
|
+
// Convert wide string to narrow string
|
|
410
|
+
int size_needed = WideCharToMultiByte(CP_UTF8, 0, varName.pwszVal, -1, nullptr, 0, nullptr, nullptr);
|
|
411
|
+
if (size_needed > 0) {
|
|
412
|
+
std::vector<char> buffer(size_needed);
|
|
413
|
+
WideCharToMultiByte(CP_UTF8, 0, varName.pwszVal, -1, buffer.data(), size_needed, nullptr, nullptr);
|
|
414
|
+
device_name = std::string(buffer.data());
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
PropVariantClear(&varName);
|
|
418
|
+
|
|
419
|
+
return device_name;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
bool MicCapturePlugin::IsBluetoothDevice() {
|
|
423
|
+
// Check device name for Bluetooth keywords
|
|
424
|
+
std::string device_name = GetCurrentDeviceName();
|
|
425
|
+
|
|
426
|
+
// Convert to lowercase safely
|
|
427
|
+
for (size_t i = 0; i < device_name.length(); ++i) {
|
|
428
|
+
device_name[i] = static_cast<char>(::tolower(static_cast<unsigned char>(device_name[i])));
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
const char* bluetooth_keywords[] = {
|
|
432
|
+
"bluetooth", "airpods", "beats", "jabra", "sony", "bose", "jbl"
|
|
433
|
+
};
|
|
434
|
+
|
|
435
|
+
for (size_t i = 0; i < sizeof(bluetooth_keywords) / sizeof(bluetooth_keywords[0]); ++i) {
|
|
436
|
+
if (device_name.find(bluetooth_keywords[i]) != std::string::npos) {
|
|
437
|
+
return true;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
return false;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
void MicCapturePlugin::CleanupExistingCapture() {
|
|
445
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
446
|
+
|
|
447
|
+
if (is_capturing_ && capture_thread_.joinable()) {
|
|
448
|
+
should_stop_ = true;
|
|
449
|
+
mutex_.unlock();
|
|
450
|
+
|
|
451
|
+
capture_thread_.join();
|
|
452
|
+
|
|
453
|
+
mutex_.lock();
|
|
454
|
+
is_capturing_ = false;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
current_device_name_.clear();
|
|
458
|
+
|
|
459
|
+
// Small delay for cleanup to complete
|
|
460
|
+
std::this_thread::sleep_for(std::chrono::milliseconds(500));
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
bool MicCapturePlugin::OpenWASAPIStreamWithRetry(
|
|
464
|
+
int sample_rate, int channels, int bits_per_sample, bool is_bluetooth,
|
|
465
|
+
void** out_audio_client, void** out_capture_client,
|
|
466
|
+
std::string* error_message) {
|
|
467
|
+
const int max_retries = is_bluetooth ? 5 : 3;
|
|
468
|
+
const double initial_wait = is_bluetooth ? 1.5 : 0.3;
|
|
469
|
+
// ĐÚNG - Khởi tạo array riêng
|
|
470
|
+
const double* retry_delays;
|
|
471
|
+
const double bluetooth_delays[] = {0.5, 1.0, 1.5, 2.0, 2.5};
|
|
472
|
+
const double normal_delays[] = {0.3, 0.6, 1.0, 0.0, 0.0};
|
|
473
|
+
retry_delays = is_bluetooth ? bluetooth_delays : normal_delays;
|
|
474
|
+
|
|
475
|
+
// Initial wait for device to be ready
|
|
476
|
+
std::this_thread::sleep_for(
|
|
477
|
+
std::chrono::milliseconds(static_cast<int>(initial_wait * 1000)));
|
|
478
|
+
|
|
479
|
+
for (int attempt = 1; attempt <= max_retries; ++attempt) {
|
|
480
|
+
// Initialize COM (allow RPC_E_CHANGED_MODE if already initialized)
|
|
481
|
+
HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
|
|
482
|
+
bool com_initialized_this_attempt = (hr == S_OK);
|
|
483
|
+
if (FAILED(hr) && hr != RPC_E_CHANGED_MODE) {
|
|
484
|
+
if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
|
|
485
|
+
std::this_thread::sleep_for(
|
|
486
|
+
std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
if (error_message) {
|
|
490
|
+
*error_message = "Failed to initialize COM";
|
|
491
|
+
}
|
|
492
|
+
return false;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// Get default audio endpoint (eCapture for microphone)
|
|
496
|
+
IMMDeviceEnumerator* enumerator = nullptr;
|
|
497
|
+
hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL,
|
|
498
|
+
__uuidof(IMMDeviceEnumerator),
|
|
499
|
+
reinterpret_cast<void**>(&enumerator));
|
|
500
|
+
if (FAILED(hr)) {
|
|
501
|
+
if (com_initialized_this_attempt) {
|
|
502
|
+
CoUninitialize();
|
|
503
|
+
}
|
|
504
|
+
if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
|
|
505
|
+
std::this_thread::sleep_for(
|
|
506
|
+
std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
|
|
507
|
+
continue;
|
|
508
|
+
}
|
|
509
|
+
if (error_message) {
|
|
510
|
+
*error_message = "Failed to create device enumerator";
|
|
511
|
+
}
|
|
512
|
+
return false;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
hr = enumerator->GetDefaultAudioEndpoint(eCapture, eConsole, &device_);
|
|
516
|
+
enumerator->Release();
|
|
517
|
+
if (FAILED(hr)) {
|
|
518
|
+
if (com_initialized_this_attempt) {
|
|
519
|
+
CoUninitialize();
|
|
520
|
+
}
|
|
521
|
+
if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
|
|
522
|
+
std::this_thread::sleep_for(
|
|
523
|
+
std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
|
|
524
|
+
continue;
|
|
525
|
+
}
|
|
526
|
+
if (error_message) {
|
|
527
|
+
*error_message = "Failed to get default audio endpoint";
|
|
528
|
+
}
|
|
529
|
+
return false;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// Activate IAudioClient
|
|
533
|
+
hr = device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr,
|
|
534
|
+
reinterpret_cast<void**>(&audio_client_));
|
|
535
|
+
if (FAILED(hr)) {
|
|
536
|
+
device_->Release();
|
|
537
|
+
device_ = nullptr;
|
|
538
|
+
if (com_initialized_this_attempt) {
|
|
539
|
+
CoUninitialize();
|
|
540
|
+
}
|
|
541
|
+
if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
|
|
542
|
+
std::this_thread::sleep_for(
|
|
543
|
+
std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
|
|
544
|
+
continue;
|
|
545
|
+
}
|
|
546
|
+
if (error_message) {
|
|
547
|
+
*error_message = "Failed to activate IAudioClient";
|
|
548
|
+
}
|
|
549
|
+
return false;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
// Get mix format - use original format for initialization
|
|
553
|
+
WAVEFORMATEX* device_format = nullptr;
|
|
554
|
+
hr = audio_client_->GetMixFormat(&device_format);
|
|
555
|
+
if (FAILED(hr)) {
|
|
556
|
+
audio_client_->Release();
|
|
557
|
+
audio_client_ = nullptr;
|
|
558
|
+
device_->Release();
|
|
559
|
+
device_ = nullptr;
|
|
560
|
+
if (com_initialized_this_attempt) {
|
|
561
|
+
CoUninitialize();
|
|
562
|
+
}
|
|
563
|
+
if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
|
|
564
|
+
std::this_thread::sleep_for(
|
|
565
|
+
std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
|
|
566
|
+
continue;
|
|
567
|
+
}
|
|
568
|
+
if (error_message) {
|
|
569
|
+
*error_message = "Failed to get mix format";
|
|
570
|
+
}
|
|
571
|
+
return false;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
// Store device format for use in capture thread
|
|
575
|
+
// We'll use the device's native format and convert in the thread
|
|
576
|
+
mix_format_ = device_format;
|
|
577
|
+
|
|
578
|
+
// Initialize audio client for capture with device's native format
|
|
579
|
+
// FIX LATENCY: Giảm buffer duration xuống 100ms để giảm latency
|
|
580
|
+
REFERENCE_TIME hnsRequestedDuration = REFTIMES_PER_SEC / 10; // 100ms instead of 1 second
|
|
581
|
+
hr = audio_client_->Initialize(AUDCLNT_SHAREMODE_SHARED, 0,
|
|
582
|
+
hnsRequestedDuration, 0, mix_format_, nullptr);
|
|
583
|
+
if (FAILED(hr)) {
|
|
584
|
+
CoTaskMemFree(mix_format_);
|
|
585
|
+
mix_format_ = nullptr;
|
|
586
|
+
audio_client_->Release();
|
|
587
|
+
audio_client_ = nullptr;
|
|
588
|
+
device_->Release();
|
|
589
|
+
device_ = nullptr;
|
|
590
|
+
if (com_initialized_this_attempt) {
|
|
591
|
+
CoUninitialize();
|
|
592
|
+
}
|
|
593
|
+
if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
|
|
594
|
+
std::this_thread::sleep_for(
|
|
595
|
+
std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
|
|
596
|
+
continue;
|
|
597
|
+
}
|
|
598
|
+
if (error_message) {
|
|
599
|
+
*error_message = "Failed to initialize audio client";
|
|
600
|
+
}
|
|
601
|
+
return false;
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
// Get buffer size
|
|
605
|
+
hr = audio_client_->GetBufferSize(&buffer_frame_count_);
|
|
606
|
+
if (FAILED(hr)) {
|
|
607
|
+
CoTaskMemFree(mix_format_);
|
|
608
|
+
mix_format_ = nullptr;
|
|
609
|
+
audio_client_->Release();
|
|
610
|
+
audio_client_ = nullptr;
|
|
611
|
+
device_->Release();
|
|
612
|
+
device_ = nullptr;
|
|
613
|
+
if (com_initialized_this_attempt) {
|
|
614
|
+
CoUninitialize();
|
|
615
|
+
}
|
|
616
|
+
if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
|
|
617
|
+
std::this_thread::sleep_for(
|
|
618
|
+
std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
|
|
619
|
+
continue;
|
|
620
|
+
}
|
|
621
|
+
if (error_message) {
|
|
622
|
+
*error_message = "Failed to get buffer size";
|
|
623
|
+
}
|
|
624
|
+
return false;
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
// Get IAudioCaptureClient
|
|
628
|
+
hr = audio_client_->GetService(__uuidof(IAudioCaptureClient),
|
|
629
|
+
reinterpret_cast<void**>(&capture_client_));
|
|
630
|
+
if (FAILED(hr)) {
|
|
631
|
+
CoTaskMemFree(mix_format_);
|
|
632
|
+
mix_format_ = nullptr;
|
|
633
|
+
audio_client_->Release();
|
|
634
|
+
audio_client_ = nullptr;
|
|
635
|
+
device_->Release();
|
|
636
|
+
device_ = nullptr;
|
|
637
|
+
if (com_initialized_this_attempt) {
|
|
638
|
+
CoUninitialize();
|
|
639
|
+
}
|
|
640
|
+
if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
|
|
641
|
+
std::this_thread::sleep_for(
|
|
642
|
+
std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
645
|
+
if (error_message) {
|
|
646
|
+
*error_message = "Failed to get IAudioCaptureClient";
|
|
647
|
+
}
|
|
648
|
+
return false;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
// Start capture
|
|
652
|
+
hr = audio_client_->Start();
|
|
653
|
+
if (FAILED(hr)) {
|
|
654
|
+
capture_client_->Release();
|
|
655
|
+
capture_client_ = nullptr;
|
|
656
|
+
CoTaskMemFree(mix_format_);
|
|
657
|
+
mix_format_ = nullptr;
|
|
658
|
+
audio_client_->Release();
|
|
659
|
+
audio_client_ = nullptr;
|
|
660
|
+
device_->Release();
|
|
661
|
+
device_ = nullptr;
|
|
662
|
+
if (com_initialized_this_attempt) {
|
|
663
|
+
CoUninitialize();
|
|
664
|
+
}
|
|
665
|
+
if (attempt < max_retries && retry_delays[attempt - 1] > 0.0) {
|
|
666
|
+
std::this_thread::sleep_for(
|
|
667
|
+
std::chrono::milliseconds(static_cast<int>(retry_delays[attempt - 1] * 1000)));
|
|
668
|
+
continue;
|
|
669
|
+
}
|
|
670
|
+
if (error_message) {
|
|
671
|
+
*error_message = "Failed to start audio client";
|
|
672
|
+
}
|
|
673
|
+
return false;
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
// Success - track that we initialized COM
|
|
677
|
+
com_initialized_ = com_initialized_this_attempt;
|
|
678
|
+
*out_audio_client = audio_client_;
|
|
679
|
+
*out_capture_client = capture_client_;
|
|
680
|
+
return true;
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
if (error_message) {
|
|
684
|
+
*error_message = "Failed to open WASAPI stream after retries";
|
|
685
|
+
}
|
|
686
|
+
return false;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
bool MicCapturePlugin::HasInputDevice() {
|
|
690
|
+
HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
|
|
691
|
+
bool com_initialized = (hr == S_OK);
|
|
692
|
+
if (FAILED(hr) && hr != RPC_E_CHANGED_MODE) {
|
|
693
|
+
return false;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
IMMDeviceEnumerator* enumerator = nullptr;
|
|
697
|
+
hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL,
|
|
698
|
+
__uuidof(IMMDeviceEnumerator),
|
|
699
|
+
reinterpret_cast<void**>(&enumerator));
|
|
700
|
+
if (FAILED(hr)) {
|
|
701
|
+
if (com_initialized) {
|
|
702
|
+
CoUninitialize();
|
|
703
|
+
}
|
|
704
|
+
return false;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
IMMDevice* device = nullptr;
|
|
708
|
+
hr = enumerator->GetDefaultAudioEndpoint(eCapture, eConsole, &device);
|
|
709
|
+
enumerator->Release();
|
|
710
|
+
|
|
711
|
+
bool has_device = SUCCEEDED(hr) && device != nullptr;
|
|
712
|
+
if (device) {
|
|
713
|
+
device->Release();
|
|
714
|
+
}
|
|
715
|
+
if (com_initialized) {
|
|
716
|
+
CoUninitialize();
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
return has_device;
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
std::vector<flutter::EncodableValue> MicCapturePlugin::GetAvailableInputDevices() {
|
|
723
|
+
std::vector<flutter::EncodableValue> device_list;
|
|
724
|
+
|
|
725
|
+
// Get default device info
|
|
726
|
+
std::string device_name = GetCurrentDeviceName();
|
|
727
|
+
bool is_bluetooth = IsBluetoothDevice();
|
|
728
|
+
|
|
729
|
+
flutter::EncodableMap device_map;
|
|
730
|
+
device_map[flutter::EncodableValue("id")] = flutter::EncodableValue("default");
|
|
731
|
+
device_map[flutter::EncodableValue("name")] = flutter::EncodableValue(device_name);
|
|
732
|
+
device_map[flutter::EncodableValue("type")] =
|
|
733
|
+
flutter::EncodableValue(is_bluetooth ? "bluetooth" : "external");
|
|
734
|
+
device_map[flutter::EncodableValue("channelCount")] = flutter::EncodableValue(1);
|
|
735
|
+
device_map[flutter::EncodableValue("isDefault")] = flutter::EncodableValue(true);
|
|
736
|
+
|
|
737
|
+
device_list.push_back(flutter::EncodableValue(device_map));
|
|
738
|
+
|
|
739
|
+
return device_list;
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
bool MicCapturePlugin::StartCapture(const flutter::EncodableMap* args) {
|
|
743
|
+
// Always cleanup any existing capture first
|
|
744
|
+
CleanupExistingCapture();
|
|
745
|
+
|
|
746
|
+
// Parse arguments
|
|
747
|
+
if (args) {
|
|
748
|
+
auto it = args->find(flutter::EncodableValue("sampleRate"));
|
|
749
|
+
if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
|
|
750
|
+
sample_rate_ = std::get<int32_t>(it->second);
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
it = args->find(flutter::EncodableValue("channels"));
|
|
754
|
+
if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
|
|
755
|
+
channels_ = std::get<int32_t>(it->second);
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
it = args->find(flutter::EncodableValue("bitDepth"));
|
|
759
|
+
if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
|
|
760
|
+
bits_per_sample_ = std::get<int32_t>(it->second);
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
it = args->find(flutter::EncodableValue("gainBoost"));
|
|
764
|
+
if (it != args->end() && std::holds_alternative<double>(it->second)) {
|
|
765
|
+
gain_boost_ = static_cast<float>(std::get<double>(it->second));
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
it = args->find(flutter::EncodableValue("inputVolume"));
|
|
769
|
+
if (it != args->end() && std::holds_alternative<double>(it->second)) {
|
|
770
|
+
input_volume_ = static_cast<float>(std::get<double>(it->second));
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
// Clamp values
|
|
775
|
+
sample_rate_ = (std::max)(sample_rate_, 8000);
|
|
776
|
+
int min_channels = (std::min)(channels_, 2);
|
|
777
|
+
channels_ = (std::max)(1, min_channels);
|
|
778
|
+
bits_per_sample_ = 16; // Force 16-bit
|
|
779
|
+
float min_gain = (std::min)(10.0f, gain_boost_);
|
|
780
|
+
gain_boost_ = (std::max)(0.1f, min_gain);
|
|
781
|
+
float min_volume = (std::min)(1.0f, input_volume_);
|
|
782
|
+
input_volume_ = (std::max)(0.0f, min_volume);
|
|
783
|
+
|
|
784
|
+
// Detect if device is Bluetooth
|
|
785
|
+
bool is_bluetooth = IsBluetoothDevice();
|
|
786
|
+
|
|
787
|
+
std::string error_message;
|
|
788
|
+
if (!OpenWASAPIStreamWithRetry(sample_rate_, channels_, bits_per_sample_,
|
|
789
|
+
is_bluetooth,
|
|
790
|
+
reinterpret_cast<void**>(&audio_client_),
|
|
791
|
+
reinterpret_cast<void**>(&capture_client_),
|
|
792
|
+
&error_message)) {
|
|
793
|
+
// Log error
|
|
794
|
+
return false;
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
// Get device name (device_ is already set by OpenWASAPIStreamWithRetry)
|
|
798
|
+
current_device_name_ = GetCurrentDeviceName();
|
|
799
|
+
|
|
800
|
+
{
|
|
801
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
802
|
+
if (is_capturing_) {
|
|
803
|
+
return false;
|
|
804
|
+
}
|
|
805
|
+
should_stop_ = false;
|
|
806
|
+
is_capturing_ = true;
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
// Start capture thread
|
|
810
|
+
capture_thread_ = std::thread(&MicCapturePlugin::CaptureThread, this);
|
|
811
|
+
|
|
812
|
+
// Wait a bit to ensure thread has started
|
|
813
|
+
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
|
814
|
+
|
|
815
|
+
SendStatusUpdate(true, current_device_name_);
|
|
816
|
+
|
|
817
|
+
return true;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
bool MicCapturePlugin::StopCapture() {
|
|
821
|
+
{
|
|
822
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
823
|
+
if (!is_capturing_) {
|
|
824
|
+
return false;
|
|
825
|
+
}
|
|
826
|
+
should_stop_ = true;
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
if (capture_thread_.joinable()) {
|
|
830
|
+
capture_thread_.join();
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
{
|
|
834
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
835
|
+
is_capturing_ = false;
|
|
836
|
+
current_device_name_.clear();
|
|
837
|
+
|
|
838
|
+
// Cleanup WASAPI resources
|
|
839
|
+
if (audio_client_) {
|
|
840
|
+
audio_client_->Stop();
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
if (capture_client_) {
|
|
844
|
+
capture_client_->Release();
|
|
845
|
+
capture_client_ = nullptr;
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
if (mix_format_) {
|
|
849
|
+
CoTaskMemFree(mix_format_);
|
|
850
|
+
mix_format_ = nullptr;
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
if (audio_client_) {
|
|
854
|
+
audio_client_->Release();
|
|
855
|
+
audio_client_ = nullptr;
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
if (device_) {
|
|
859
|
+
device_->Release();
|
|
860
|
+
device_ = nullptr;
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// Only uninitialize COM if we initialized it
|
|
864
|
+
if (com_initialized_) {
|
|
865
|
+
CoUninitialize();
|
|
866
|
+
com_initialized_ = false;
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
// Wait a bit to ensure thread has fully stopped
|
|
871
|
+
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
|
872
|
+
|
|
873
|
+
SendStatusUpdate(false);
|
|
874
|
+
|
|
875
|
+
return true;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
// UPDATED: CaptureThread - optimized for lower latency
|
|
879
|
+
void MicCapturePlugin::CaptureThread() {
|
|
880
|
+
try {
|
|
881
|
+
// Set thread priority to reduce latency
|
|
882
|
+
SetThreadPriority();
|
|
883
|
+
|
|
884
|
+
if (!mix_format_ || !capture_client_) {
|
|
885
|
+
return;
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
const UINT32 frame_size = mix_format_->nBlockAlign;
|
|
889
|
+
const UINT32 actual_sample_rate = mix_format_->nSamplesPerSec;
|
|
890
|
+
const WORD actual_channels = mix_format_->nChannels;
|
|
891
|
+
const WORD actual_bits_per_sample = mix_format_->wBitsPerSample;
|
|
892
|
+
const WORD format_tag = mix_format_->wFormatTag;
|
|
893
|
+
|
|
894
|
+
// Detect format
|
|
895
|
+
bool is_float_format = false;
|
|
896
|
+
bool is_pcm_format = false;
|
|
897
|
+
|
|
898
|
+
if (format_tag == WAVE_FORMAT_EXTENSIBLE && mix_format_->cbSize >= 22) {
|
|
899
|
+
WAVEFORMATEXTENSIBLE* wfex = reinterpret_cast<WAVEFORMATEXTENSIBLE*>(mix_format_);
|
|
900
|
+
if (IsEqualGUID(wfex->SubFormat, KSDATAFORMAT_SUBTYPE_IEEE_FLOAT)) {
|
|
901
|
+
is_float_format = true;
|
|
902
|
+
} else if (IsEqualGUID(wfex->SubFormat, KSDATAFORMAT_SUBTYPE_PCM)) {
|
|
903
|
+
is_pcm_format = true;
|
|
904
|
+
}
|
|
905
|
+
} else if (format_tag == WAVE_FORMAT_IEEE_FLOAT) {
|
|
906
|
+
is_float_format = true;
|
|
907
|
+
} else if (format_tag == WAVE_FORMAT_PCM) {
|
|
908
|
+
is_pcm_format = true;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
// FIX LATENCY: Sử dụng chunk size nhỏ hơn (20-50ms) để giảm delay
|
|
912
|
+
const int effective_chunk_ms = 30; // 30ms chunk size for lower latency
|
|
913
|
+
const size_t chunk_frames = (actual_sample_rate * effective_chunk_ms / 1000);
|
|
914
|
+
const size_t chunk_size_bytes = chunk_frames * frame_size;
|
|
915
|
+
const size_t output_frame_count = (sample_rate_ * effective_chunk_ms / 1000);
|
|
916
|
+
|
|
917
|
+
std::vector<uint8_t> raw_buffer(chunk_size_bytes * 2); // Double buffer for safety
|
|
918
|
+
std::vector<int16_t> output_buffer(output_frame_count);
|
|
919
|
+
size_t raw_buffer_pos = 0;
|
|
920
|
+
|
|
921
|
+
while (!should_stop_) {
|
|
922
|
+
UINT32 num_frames_available = 0;
|
|
923
|
+
HRESULT hr = capture_client_->GetNextPacketSize(&num_frames_available);
|
|
924
|
+
|
|
925
|
+
if (FAILED(hr)) break;
|
|
926
|
+
|
|
927
|
+
while (num_frames_available > 0 && !should_stop_) {
|
|
928
|
+
BYTE* data = nullptr;
|
|
929
|
+
UINT32 num_frames = 0;
|
|
930
|
+
DWORD flags = 0;
|
|
931
|
+
UINT64 device_position = 0;
|
|
932
|
+
UINT64 qpc_position = 0;
|
|
933
|
+
|
|
934
|
+
hr = capture_client_->GetBuffer(&data, &num_frames, &flags,
|
|
935
|
+
&device_position, &qpc_position);
|
|
936
|
+
if (FAILED(hr)) break;
|
|
937
|
+
|
|
938
|
+
bool is_silent = (flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0;
|
|
939
|
+
|
|
940
|
+
if (!is_silent && data != nullptr && num_frames > 0) {
|
|
941
|
+
const size_t data_size = num_frames * frame_size;
|
|
942
|
+
size_t data_offset = 0;
|
|
943
|
+
|
|
944
|
+
while (data_offset < data_size && !should_stop_) {
|
|
945
|
+
const size_t space_available = raw_buffer.size() - raw_buffer_pos;
|
|
946
|
+
const size_t data_remaining = data_size - data_offset;
|
|
947
|
+
const size_t copy_size = (std::min)(space_available, data_remaining);
|
|
948
|
+
|
|
949
|
+
if (copy_size > 0) {
|
|
950
|
+
memcpy(raw_buffer.data() + raw_buffer_pos,
|
|
951
|
+
reinterpret_cast<const uint8_t*>(data) + data_offset,
|
|
952
|
+
copy_size);
|
|
953
|
+
raw_buffer_pos += copy_size;
|
|
954
|
+
data_offset += copy_size;
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
if (raw_buffer_pos >= chunk_size_bytes) {
|
|
958
|
+
const size_t input_frame_count = chunk_size_bytes / frame_size;
|
|
959
|
+
const size_t total_samples = input_frame_count * actual_channels;
|
|
960
|
+
|
|
961
|
+
std::vector<int16_t> converted_samples(total_samples);
|
|
962
|
+
bool conversion_success = false;
|
|
963
|
+
|
|
964
|
+
// Format conversion
|
|
965
|
+
if (is_pcm_format && actual_bits_per_sample == 16) {
|
|
966
|
+
const int16_t* raw_samples = reinterpret_cast<const int16_t*>(raw_buffer.data());
|
|
967
|
+
converted_samples.assign(raw_samples, raw_samples + total_samples);
|
|
968
|
+
conversion_success = true;
|
|
969
|
+
} else if (is_float_format && actual_bits_per_sample == 32) {
|
|
970
|
+
const float* float_samples = reinterpret_cast<const float*>(raw_buffer.data());
|
|
971
|
+
for (size_t i = 0; i < total_samples; ++i) {
|
|
972
|
+
float sample = (std::min)(1.0f, (std::max)(-1.0f, float_samples[i]));
|
|
973
|
+
converted_samples[i] = static_cast<int16_t>(sample * 32767.0f);
|
|
974
|
+
}
|
|
975
|
+
conversion_success = true;
|
|
976
|
+
} else if (is_pcm_format && actual_bits_per_sample == 24) {
|
|
977
|
+
const uint8_t* raw_bytes = raw_buffer.data();
|
|
978
|
+
for (size_t i = 0; i < total_samples; ++i) {
|
|
979
|
+
size_t byte_offset = i * 3;
|
|
980
|
+
int32_t sample24 = (static_cast<int32_t>(raw_bytes[byte_offset]) |
|
|
981
|
+
(static_cast<int32_t>(raw_bytes[byte_offset + 1]) << 8) |
|
|
982
|
+
(static_cast<int32_t>(raw_bytes[byte_offset + 2]) << 16));
|
|
983
|
+
if (sample24 & 0x800000) sample24 |= 0xFF000000;
|
|
984
|
+
converted_samples[i] = static_cast<int16_t>(sample24 >> 8);
|
|
985
|
+
}
|
|
986
|
+
conversion_success = true;
|
|
987
|
+
} else if (is_pcm_format && actual_bits_per_sample == 32) {
|
|
988
|
+
const int32_t* int32_samples = reinterpret_cast<const int32_t*>(raw_buffer.data());
|
|
989
|
+
for (size_t i = 0; i < total_samples; ++i) {
|
|
990
|
+
converted_samples[i] = static_cast<int16_t>(int32_samples[i] >> 16);
|
|
991
|
+
}
|
|
992
|
+
conversion_success = true;
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
if (!conversion_success) {
|
|
996
|
+
raw_buffer_pos = 0;
|
|
997
|
+
continue;
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
if (input_volume_ > 0.0f && input_volume_ < 1.0f) {
|
|
1001
|
+
for (size_t i = 0; i < total_samples; ++i) {
|
|
1002
|
+
converted_samples[i] = static_cast<int16_t>(
|
|
1003
|
+
static_cast<float>(converted_samples[i]) * input_volume_);
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
const size_t input_frames = converted_samples.size() / actual_channels;
|
|
1008
|
+
|
|
1009
|
+
// First: Convert to mono and apply gain boost (at input sample rate)
|
|
1010
|
+
std::vector<int16_t> mono_buffer(input_frames);
|
|
1011
|
+
ApplyGainBoostAndConvertToMono(converted_samples.data(), mono_buffer.data(),
|
|
1012
|
+
input_frames, actual_channels, gain_boost_);
|
|
1013
|
+
|
|
1014
|
+
// Second: Resample if sample rates differ
|
|
1015
|
+
// Calculate correct output frames based on resampling ratio
|
|
1016
|
+
size_t output_frames;
|
|
1017
|
+
if (actual_sample_rate != static_cast<UINT32>(sample_rate_)) {
|
|
1018
|
+
// Calculate output frames after resampling
|
|
1019
|
+
output_frames = static_cast<size_t>(
|
|
1020
|
+
static_cast<double>(input_frames) * static_cast<double>(sample_rate_) /
|
|
1021
|
+
static_cast<double>(actual_sample_rate));
|
|
1022
|
+
output_frames = (std::min)(output_frames, output_frame_count);
|
|
1023
|
+
|
|
1024
|
+
// Resample from actual_sample_rate to sample_rate_
|
|
1025
|
+
ResampleAudio(mono_buffer.data(), input_frames,
|
|
1026
|
+
output_buffer.data(), output_frames,
|
|
1027
|
+
actual_sample_rate, sample_rate_);
|
|
1028
|
+
} else {
|
|
1029
|
+
// No resampling needed, just copy
|
|
1030
|
+
output_frames = (std::min)(input_frames, output_frame_count);
|
|
1031
|
+
memcpy(output_buffer.data(), mono_buffer.data(), output_frames * sizeof(int16_t));
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
double decibel = CalculateDecibel(output_buffer.data(), output_frames);
|
|
1035
|
+
|
|
1036
|
+
// CHANGED: Queue instead of direct send
|
|
1037
|
+
const size_t output_bytes = output_frames * sizeof(int16_t);
|
|
1038
|
+
std::vector<uint8_t> audio_data(
|
|
1039
|
+
reinterpret_cast<uint8_t*>(output_buffer.data()),
|
|
1040
|
+
reinterpret_cast<uint8_t*>(output_buffer.data()) + output_bytes);
|
|
1041
|
+
|
|
1042
|
+
QueueAudioData(std::move(audio_data), decibel);
|
|
1043
|
+
|
|
1044
|
+
if (raw_buffer_pos > chunk_size_bytes) {
|
|
1045
|
+
const size_t remaining = raw_buffer_pos - chunk_size_bytes;
|
|
1046
|
+
memmove(raw_buffer.data(), raw_buffer.data() + chunk_size_bytes, remaining);
|
|
1047
|
+
raw_buffer_pos = remaining;
|
|
1048
|
+
} else {
|
|
1049
|
+
raw_buffer_pos = 0;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
hr = capture_client_->ReleaseBuffer(num_frames);
|
|
1056
|
+
if (FAILED(hr)) break;
|
|
1057
|
+
|
|
1058
|
+
hr = capture_client_->GetNextPacketSize(&num_frames_available);
|
|
1059
|
+
if (FAILED(hr)) break;
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
// FIX LATENCY: Giảm sleep time xuống 1ms để phản hồi nhanh hơn
|
|
1063
|
+
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
|
1064
|
+
}
|
|
1065
|
+
} catch (...) {
|
|
1066
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
1067
|
+
is_capturing_ = false;
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
// Resample audio using linear interpolation
|
|
1072
|
+
void MicCapturePlugin::ResampleAudio(const int16_t* input, size_t input_frames,
|
|
1073
|
+
int16_t* output, size_t output_frames,
|
|
1074
|
+
int input_sample_rate, int output_sample_rate) {
|
|
1075
|
+
if (input_frames == 0 || output_frames == 0) {
|
|
1076
|
+
return;
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
if (input_sample_rate == output_sample_rate) {
|
|
1080
|
+
const size_t copy_frames = (std::min)(input_frames, output_frames);
|
|
1081
|
+
memcpy(output, input, copy_frames * sizeof(int16_t));
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
// Linear interpolation resampling
|
|
1086
|
+
const double ratio = static_cast<double>(input_sample_rate) / static_cast<double>(output_sample_rate);
|
|
1087
|
+
|
|
1088
|
+
for (size_t i = 0; i < output_frames; ++i) {
|
|
1089
|
+
const double src_pos = static_cast<double>(i) * ratio;
|
|
1090
|
+
const size_t src_index = static_cast<size_t>(src_pos);
|
|
1091
|
+
const double fraction = src_pos - static_cast<double>(src_index);
|
|
1092
|
+
|
|
1093
|
+
if (src_index + 1 < input_frames) {
|
|
1094
|
+
// Linear interpolation between two samples
|
|
1095
|
+
const double sample0 = static_cast<double>(input[src_index]);
|
|
1096
|
+
const double sample1 = static_cast<double>(input[src_index + 1]);
|
|
1097
|
+
const double interpolated = sample0 + (sample1 - sample0) * fraction;
|
|
1098
|
+
output[i] = static_cast<int16_t>((std::max)(-32768.0, (std::min)(32767.0, interpolated)));
|
|
1099
|
+
} else if (src_index < input_frames) {
|
|
1100
|
+
// Last sample, no interpolation
|
|
1101
|
+
output[i] = input[src_index];
|
|
1102
|
+
} else {
|
|
1103
|
+
// Beyond input, use last sample
|
|
1104
|
+
output[i] = input[input_frames - 1];
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
// Set high priority for capture thread to reduce latency
|
|
1110
|
+
void MicCapturePlugin::SetThreadPriority() {
|
|
1111
|
+
HANDLE current_thread = GetCurrentThread();
|
|
1112
|
+
// Sử dụng THREAD_PRIORITY_HIGHEST để giảm latency
|
|
1113
|
+
::SetThreadPriority(current_thread, THREAD_PRIORITY_HIGHEST);
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
} // namespace audio_capture
|
|
1117
|
+
|