neoagent 2.3.1-beta.2 → 2.3.1-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +39 -0
- package/README.md +2 -0
- package/docs/capabilities.md +2 -2
- package/docs/configuration.md +13 -5
- package/docs/integrations.md +4 -1
- package/flutter_app/.metadata +42 -0
- package/flutter_app/README.md +21 -0
- package/flutter_app/analysis_options.yaml +32 -0
- package/flutter_app/android/app/build.gradle.kts +109 -0
- package/flutter_app/android/app/src/debug/AndroidManifest.xml +7 -0
- package/flutter_app/android/app/src/main/AndroidManifest.xml +147 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +747 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthConnectGateway.kt +280 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncNotifications.kt +113 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncPayload.kt +57 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncScheduler.kt +78 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncWorker.kt +253 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/PermissionsRationaleActivity.kt +46 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +21 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +586 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +78 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +104 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiHomeWidgetProvider.kt +457 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiWidgetStore.kt +194 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/VoiceLaunchWidgetProvider.kt +67 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetConfigActivity.kt +228 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncScheduler.kt +72 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncWorker.kt +186 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetTaskRunWorker.kt +210 -0
- package/flutter_app/android/app/src/main/res/drawable/launch_background.xml +12 -0
- package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_bg.xml +11 -0
- package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_task_bg.xml +8 -0
- package/flutter_app/android/app/src/main/res/drawable-v21/launch_background.xml +12 -0
- package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget.xml +138 -0
- package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget_task_row.xml +52 -0
- package/flutter_app/android/app/src/main/res/layout/neoagent_voice_widget.xml +49 -0
- package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/values/strings.xml +12 -0
- package/flutter_app/android/app/src/main/res/values/styles.xml +18 -0
- package/flutter_app/android/app/src/main/res/values-night/styles.xml +18 -0
- package/flutter_app/android/app/src/main/res/xml/file_paths.xml +6 -0
- package/flutter_app/android/app/src/main/res/xml/neoagent_ai_widget_info.xml +12 -0
- package/flutter_app/android/app/src/main/res/xml/neoagent_voice_widget_info.xml +12 -0
- package/flutter_app/android/app/src/profile/AndroidManifest.xml +7 -0
- package/flutter_app/android/build.gradle.kts +24 -0
- package/flutter_app/android/ci-release.keystore +0 -0
- package/flutter_app/android/gradle/wrapper/gradle-wrapper.properties +5 -0
- package/flutter_app/android/gradle.properties +3 -0
- package/flutter_app/android/key.properties +4 -0
- package/flutter_app/android/settings.gradle.kts +26 -0
- package/flutter_app/assets/branding/app_icon_1024.png +0 -0
- package/flutter_app/assets/branding/app_icon_128.png +0 -0
- package/flutter_app/assets/branding/app_icon_192.png +0 -0
- package/flutter_app/assets/branding/app_icon_256.png +0 -0
- package/flutter_app/assets/branding/app_icon_32.png +0 -0
- package/flutter_app/assets/branding/app_icon_512.png +0 -0
- package/flutter_app/assets/branding/app_icon_64.png +0 -0
- package/flutter_app/assets/branding/tray_icon_template.png +0 -0
- package/flutter_app/lib/features/location/location_service.dart +119 -0
- package/flutter_app/lib/features/notifications/notification_interceptor.dart +97 -0
- package/flutter_app/lib/main.dart +23057 -0
- package/flutter_app/lib/main_app_shell.dart +1682 -0
- package/flutter_app/lib/main_integrations.dart +931 -0
- package/flutter_app/lib/main_launcher.dart +959 -0
- package/flutter_app/lib/main_launcher_entry.dart +5 -0
- package/flutter_app/lib/main_models.dart +3473 -0
- package/flutter_app/lib/main_shared.dart +2861 -0
- package/flutter_app/lib/main_theme.dart +204 -0
- package/flutter_app/lib/main_voice_assistant.dart +831 -0
- package/flutter_app/lib/src/android_apk_drop_zone.dart +32 -0
- package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +16 -0
- package/flutter_app/lib/src/android_apk_drop_zone_web.dart +348 -0
- package/flutter_app/lib/src/android_app_installer.dart +22 -0
- package/flutter_app/lib/src/android_app_installer_io.dart +122 -0
- package/flutter_app/lib/src/android_app_installer_stub.dart +21 -0
- package/flutter_app/lib/src/android_launcher_bridge.dart +239 -0
- package/flutter_app/lib/src/app_launch_bridge.dart +29 -0
- package/flutter_app/lib/src/app_release_updater.dart +511 -0
- package/flutter_app/lib/src/backend_client.dart +1833 -0
- package/flutter_app/lib/src/desktop_companion.dart +2 -0
- package/flutter_app/lib/src/desktop_companion_actions.dart +586 -0
- package/flutter_app/lib/src/desktop_companion_io.dart +538 -0
- package/flutter_app/lib/src/desktop_companion_stub.dart +59 -0
- package/flutter_app/lib/src/desktop_native_bridge.dart +91 -0
- package/flutter_app/lib/src/desktop_screen_capture.dart +21 -0
- package/flutter_app/lib/src/desktop_screen_capture_io.dart +142 -0
- package/flutter_app/lib/src/desktop_screen_capture_stub.dart +12 -0
- package/flutter_app/lib/src/diagnostics_logger.dart +119 -0
- package/flutter_app/lib/src/health_bridge.dart +136 -0
- package/flutter_app/lib/src/live_voice_capture.dart +85 -0
- package/flutter_app/lib/src/messaging_access_summary.dart +46 -0
- package/flutter_app/lib/src/network/app_http_client.dart +53 -0
- package/flutter_app/lib/src/network/app_http_client_factory.dart +6 -0
- package/flutter_app/lib/src/network/app_http_client_io.dart +138 -0
- package/flutter_app/lib/src/network/app_http_client_stub.dart +3 -0
- package/flutter_app/lib/src/network/app_http_client_web.dart +94 -0
- package/flutter_app/lib/src/oauth_launcher.dart +33 -0
- package/flutter_app/lib/src/oauth_launcher_io.dart +77 -0
- package/flutter_app/lib/src/oauth_launcher_stub.dart +33 -0
- package/flutter_app/lib/src/oauth_launcher_web.dart +107 -0
- package/flutter_app/lib/src/recording_bridge.dart +232 -0
- package/flutter_app/lib/src/recording_bridge_io.dart +1019 -0
- package/flutter_app/lib/src/recording_bridge_stub.dart +120 -0
- package/flutter_app/lib/src/recording_bridge_web.dart +689 -0
- package/flutter_app/lib/src/recording_payloads.dart +86 -0
- package/flutter_app/lib/src/theme/palette.dart +81 -0
- package/flutter_app/lib/src/widget_bridge.dart +49 -0
- package/flutter_app/linux/CMakeLists.txt +128 -0
- package/flutter_app/linux/flutter/CMakeLists.txt +88 -0
- package/flutter_app/linux/flutter/generated_plugin_registrant.cc +43 -0
- package/flutter_app/linux/flutter/generated_plugin_registrant.h +15 -0
- package/flutter_app/linux/flutter/generated_plugins.cmake +31 -0
- package/flutter_app/linux/runner/CMakeLists.txt +26 -0
- package/flutter_app/linux/runner/main.cc +6 -0
- package/flutter_app/linux/runner/my_application.cc +144 -0
- package/flutter_app/linux/runner/my_application.h +18 -0
- package/flutter_app/linux/runner/resources/app_icon.png +0 -0
- package/flutter_app/macos/Flutter/Flutter-Debug.xcconfig +2 -0
- package/flutter_app/macos/Flutter/Flutter-Release.xcconfig +2 -0
- package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +40 -0
- package/flutter_app/macos/Podfile +42 -0
- package/flutter_app/macos/Podfile.lock +87 -0
- package/flutter_app/macos/Runner/AppDelegate.swift +576 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json +68 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
- package/flutter_app/macos/Runner/Base.lproj/MainMenu.xib +342 -0
- package/flutter_app/macos/Runner/Configs/AppInfo.xcconfig +14 -0
- package/flutter_app/macos/Runner/Configs/Debug.xcconfig +2 -0
- package/flutter_app/macos/Runner/Configs/Release.xcconfig +2 -0
- package/flutter_app/macos/Runner/Configs/Warnings.xcconfig +13 -0
- package/flutter_app/macos/Runner/DebugProfile.entitlements +16 -0
- package/flutter_app/macos/Runner/Info.plist +36 -0
- package/flutter_app/macos/Runner/MainFlutterWindow.swift +19 -0
- package/flutter_app/macos/Runner/Release.entitlements +12 -0
- package/flutter_app/macos/Runner.xcodeproj/project.pbxproj +801 -0
- package/flutter_app/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
- package/flutter_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +99 -0
- package/flutter_app/macos/Runner.xcworkspace/contents.xcworkspacedata +10 -0
- package/flutter_app/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
- package/flutter_app/macos/RunnerTests/RunnerTests.swift +12 -0
- package/flutter_app/patch_strings.py +12 -0
- package/flutter_app/pubspec.lock +1088 -0
- package/flutter_app/pubspec.yaml +53 -0
- package/flutter_app/test/messaging_access_summary_test.dart +22 -0
- package/flutter_app/test/recording_payloads_test.dart +53 -0
- package/flutter_app/third_party/desktop_audio_capture/LICENSE +21 -0
- package/flutter_app/third_party/desktop_audio_capture/README.md +262 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +65 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +153 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +110 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +461 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +91 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +106 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +219 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +336 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +101 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +692 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +35 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +36 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +32 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +32 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +878 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +27 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +1172 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +655 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +12 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +30 -0
- package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +87 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +105 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +80 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +31 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +12 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +23 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +25 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +1117 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +115 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +777 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +87 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/linux/CMakeLists.txt +30 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/linux/flutter_secure_storage_linux_plugin.cc +215 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/linux/include/flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h +27 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/pubspec.yaml +20 -0
- package/flutter_app/tool/generate_desktop_branding.py +219 -0
- package/flutter_app/web/favicon.png +0 -0
- package/flutter_app/web/favicon.svg +12 -0
- package/flutter_app/web/icons/Icon-192.png +0 -0
- package/flutter_app/web/icons/Icon-512.png +0 -0
- package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
- package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
- package/flutter_app/web/index.html +39 -0
- package/flutter_app/web/manifest.json +35 -0
- package/flutter_app/windows/CMakeLists.txt +108 -0
- package/flutter_app/windows/flutter/CMakeLists.txt +109 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.cc +47 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.h +15 -0
- package/flutter_app/windows/flutter/generated_plugins.cmake +35 -0
- package/flutter_app/windows/runner/CMakeLists.txt +41 -0
- package/flutter_app/windows/runner/Runner.rc +121 -0
- package/flutter_app/windows/runner/flutter_window.cpp +533 -0
- package/flutter_app/windows/runner/flutter_window.h +37 -0
- package/flutter_app/windows/runner/main.cpp +53 -0
- package/flutter_app/windows/runner/resource.h +16 -0
- package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
- package/flutter_app/windows/runner/runner.exe.manifest +14 -0
- package/flutter_app/windows/runner/utils.cpp +65 -0
- package/flutter_app/windows/runner/utils.h +19 -0
- package/flutter_app/windows/runner/win32_window.cpp +299 -0
- package/flutter_app/windows/runner/win32_window.h +102 -0
- package/lib/manager.js +231 -7
- package/package.json +3 -1
- package/server/db/database.js +68 -0
- package/server/http/middleware.js +50 -0
- package/server/http/routes.js +3 -1
- package/server/index.js +1 -0
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/NOTICES +61 -0
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +65262 -64422
- package/server/routes/integrations.js +86 -0
- package/server/routes/memory.js +11 -2
- package/server/routes/screenHistory.js +46 -0
- package/server/routes/triggers.js +81 -0
- package/server/services/ai/models.js +30 -0
- package/server/services/ai/providers/githubCopilot.js +97 -0
- package/server/services/ai/providers/openai.js +2 -1
- package/server/services/ai/providers/openaiCodex.js +31 -0
- package/server/services/ai/settings.js +20 -0
- package/server/services/ai/systemPrompt.js +1 -1
- package/server/services/ai/tools.js +35 -6
- package/server/services/browser/controller.js +47 -3
- package/server/services/desktop/screenRecorder.js +172 -0
- package/server/services/integrations/env.js +5 -0
- package/server/services/integrations/github/common.js +106 -0
- package/server/services/integrations/github/provider.js +499 -0
- package/server/services/integrations/github/repos.js +1124 -0
- package/server/services/integrations/home_assistant/provider.js +306 -26
- package/server/services/integrations/manager.js +63 -7
- package/server/services/integrations/oauth_provider.js +13 -6
- package/server/services/integrations/provider_config_store.js +76 -0
- package/server/services/integrations/registry.js +4 -0
- package/server/services/integrations/trello/provider.js +744 -0
- package/server/services/integrations/whatsapp/provider.js +6 -2
- package/server/services/manager.js +22 -0
- package/server/services/memory/manager.js +39 -2
- package/server/services/skills/base_catalog.js +33 -0
- package/server/services/tasks/adapters/index.js +1 -0
- package/server/services/tasks/adapters/manual.js +12 -0
- package/server/services/tasks/runtime.js +1 -1
- package/server/services/voice/openaiClient.js +4 -1
- package/server/services/voice/providers.js +2 -1
- package/server/services/widgets/service.js +49 -4
- package/server/utils/local_secrets.js +56 -0
- package/server/utils/logger.js +37 -9
|
@@ -0,0 +1,777 @@
|
|
|
1
|
+
#include "system_audio_capture_plugin.h"
|
|
2
|
+
|
|
3
|
+
#define NOMINMAX
|
|
4
|
+
#include <windows.h>
|
|
5
|
+
#undef max
|
|
6
|
+
#undef min
|
|
7
|
+
#include <mmdeviceapi.h>
|
|
8
|
+
#include <audioclient.h>
|
|
9
|
+
#include <functiondiscoverykeys_devpkey.h>
|
|
10
|
+
#include <VersionHelpers.h>
|
|
11
|
+
#include <comdef.h>
|
|
12
|
+
|
|
13
|
+
#include <flutter/event_channel.h>
|
|
14
|
+
#include <flutter/method_channel.h>
|
|
15
|
+
#include <flutter/plugin_registrar_windows.h>
|
|
16
|
+
#include <flutter/standard_method_codec.h>
|
|
17
|
+
|
|
18
|
+
#include <algorithm>
|
|
19
|
+
#include <chrono>
|
|
20
|
+
#include <cmath>
|
|
21
|
+
#include <cstdint>
|
|
22
|
+
#include <functional>
|
|
23
|
+
#include <memory>
|
|
24
|
+
#include <sstream>
|
|
25
|
+
#include <thread>
|
|
26
|
+
#include <vector>
|
|
27
|
+
|
|
28
|
+
#pragma comment(lib, "ole32.lib")
|
|
29
|
+
#pragma comment(lib, "oleaut32.lib")
|
|
30
|
+
|
|
31
|
+
// REFTIMES_PER_SEC is 10,000,000 (100 nanoseconds per second)
|
|
32
|
+
#ifndef REFTIMES_PER_SEC
|
|
33
|
+
#define REFTIMES_PER_SEC 10000000
|
|
34
|
+
#endif
|
|
35
|
+
|
|
36
|
+
namespace audio_capture {
|
|
37
|
+
|
|
38
|
+
// Custom StreamHandler implementation
|
|
39
|
+
template <typename T>
|
|
40
|
+
class StreamHandlerFunctions : public flutter::StreamHandler<T> {
|
|
41
|
+
public:
|
|
42
|
+
using OnListenHandler = std::function<std::unique_ptr<flutter::StreamHandlerError<T>>(
|
|
43
|
+
const T* arguments,
|
|
44
|
+
std::unique_ptr<flutter::EventSink<T>>&& events)>;
|
|
45
|
+
using OnCancelHandler = std::function<std::unique_ptr<flutter::StreamHandlerError<T>>(
|
|
46
|
+
const T* arguments)>;
|
|
47
|
+
|
|
48
|
+
StreamHandlerFunctions(OnListenHandler on_listen, OnCancelHandler on_cancel)
|
|
49
|
+
: on_listen_(std::move(on_listen)), on_cancel_(std::move(on_cancel)) {}
|
|
50
|
+
|
|
51
|
+
virtual ~StreamHandlerFunctions() = default;
|
|
52
|
+
|
|
53
|
+
protected:
|
|
54
|
+
std::unique_ptr<flutter::StreamHandlerError<T>> OnListenInternal(
|
|
55
|
+
const T* arguments,
|
|
56
|
+
std::unique_ptr<flutter::EventSink<T>>&& events) override {
|
|
57
|
+
return on_listen_(arguments, std::move(events));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
std::unique_ptr<flutter::StreamHandlerError<T>> OnCancelInternal(
|
|
61
|
+
const T* arguments) override {
|
|
62
|
+
return on_cancel_(arguments);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
private:
|
|
66
|
+
OnListenHandler on_listen_;
|
|
67
|
+
OnCancelHandler on_cancel_;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
namespace {
|
|
71
|
+
|
|
72
|
+
constexpr char kMethodChannelName[] = "com.system_audio_transcriber/audio_capture";
|
|
73
|
+
constexpr char kEventChannelName[] = "com.system_audio_transcriber/audio_stream";
|
|
74
|
+
constexpr char kStatusEventChannelName[] = "com.system_audio_transcriber/audio_status";
|
|
75
|
+
constexpr char kDecibelEventChannelName[] = "com.system_audio_transcriber/audio_decibel";
|
|
76
|
+
|
|
77
|
+
constexpr int kDefaultSampleRate = 16000;
|
|
78
|
+
constexpr int kDefaultChannels = 1;
|
|
79
|
+
constexpr int kDefaultBitsPerSample = 16;
|
|
80
|
+
constexpr int kDefaultChunkDurationMs = 1000;
|
|
81
|
+
constexpr float kDefaultGainBoost = 2.5f;
|
|
82
|
+
constexpr float kDefaultInputVolume = 1.0f;
|
|
83
|
+
|
|
84
|
+
} // namespace
|
|
85
|
+
|
|
86
|
+
// static
|
|
87
|
+
void SystemAudioCapturePlugin::RegisterWithRegistrar(
|
|
88
|
+
flutter::PluginRegistrarWindows *registrar) {
|
|
89
|
+
auto plugin = std::make_unique<SystemAudioCapturePlugin>(registrar);
|
|
90
|
+
registrar->AddPlugin(std::move(plugin));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
SystemAudioCapturePlugin::SystemAudioCapturePlugin(
|
|
94
|
+
flutter::PluginRegistrarWindows *registrar)
|
|
95
|
+
: registrar_(registrar),
|
|
96
|
+
is_capturing_(false),
|
|
97
|
+
should_stop_(false),
|
|
98
|
+
sample_rate_(kDefaultSampleRate),
|
|
99
|
+
channels_(kDefaultChannels),
|
|
100
|
+
bits_per_sample_(kDefaultBitsPerSample),
|
|
101
|
+
chunk_duration_ms_(kDefaultChunkDurationMs),
|
|
102
|
+
gain_boost_(kDefaultGainBoost),
|
|
103
|
+
input_volume_(kDefaultInputVolume),
|
|
104
|
+
audio_client_(nullptr),
|
|
105
|
+
capture_client_(nullptr),
|
|
106
|
+
device_(nullptr),
|
|
107
|
+
mix_format_(nullptr),
|
|
108
|
+
buffer_frame_count_(0),
|
|
109
|
+
com_initialized_(false) {
|
|
110
|
+
// Create method channel
|
|
111
|
+
method_channel_ =
|
|
112
|
+
std::make_unique<flutter::MethodChannel<flutter::EncodableValue>>(
|
|
113
|
+
registrar->messenger(), kMethodChannelName,
|
|
114
|
+
&flutter::StandardMethodCodec::GetInstance());
|
|
115
|
+
|
|
116
|
+
method_channel_->SetMethodCallHandler(
|
|
117
|
+
[this](const auto &call, auto result) {
|
|
118
|
+
this->HandleMethodCall(call, std::move(result));
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
// Create event channels
|
|
122
|
+
event_channel_ =
|
|
123
|
+
std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
|
|
124
|
+
registrar->messenger(), kEventChannelName,
|
|
125
|
+
&flutter::StandardMethodCodec::GetInstance());
|
|
126
|
+
|
|
127
|
+
event_channel_->SetStreamHandler(
|
|
128
|
+
std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
|
|
129
|
+
[this](const flutter::EncodableValue* arguments,
|
|
130
|
+
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
|
|
131
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
132
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
133
|
+
event_sink_ = std::move(events);
|
|
134
|
+
return nullptr;
|
|
135
|
+
},
|
|
136
|
+
[this](const flutter::EncodableValue* arguments)
|
|
137
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
138
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
139
|
+
event_sink_.reset();
|
|
140
|
+
return nullptr;
|
|
141
|
+
}));
|
|
142
|
+
|
|
143
|
+
status_event_channel_ =
|
|
144
|
+
std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
|
|
145
|
+
registrar->messenger(), kStatusEventChannelName,
|
|
146
|
+
&flutter::StandardMethodCodec::GetInstance());
|
|
147
|
+
|
|
148
|
+
status_event_channel_->SetStreamHandler(
|
|
149
|
+
std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
|
|
150
|
+
[this](const flutter::EncodableValue* arguments,
|
|
151
|
+
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
|
|
152
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
153
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
154
|
+
status_event_sink_ = std::move(events);
|
|
155
|
+
SendStatusUpdate(is_capturing_);
|
|
156
|
+
return nullptr;
|
|
157
|
+
},
|
|
158
|
+
[this](const flutter::EncodableValue* arguments)
|
|
159
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
160
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
161
|
+
status_event_sink_.reset();
|
|
162
|
+
return nullptr;
|
|
163
|
+
}));
|
|
164
|
+
|
|
165
|
+
decibel_event_channel_ =
|
|
166
|
+
std::make_unique<flutter::EventChannel<flutter::EncodableValue>>(
|
|
167
|
+
registrar->messenger(), kDecibelEventChannelName,
|
|
168
|
+
&flutter::StandardMethodCodec::GetInstance());
|
|
169
|
+
|
|
170
|
+
decibel_event_channel_->SetStreamHandler(
|
|
171
|
+
std::make_unique<StreamHandlerFunctions<flutter::EncodableValue>>(
|
|
172
|
+
[this](const flutter::EncodableValue* arguments,
|
|
173
|
+
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& events)
|
|
174
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
175
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
176
|
+
decibel_event_sink_ = std::move(events);
|
|
177
|
+
return nullptr;
|
|
178
|
+
},
|
|
179
|
+
[this](const flutter::EncodableValue* arguments)
|
|
180
|
+
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
|
181
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
182
|
+
decibel_event_sink_.reset();
|
|
183
|
+
return nullptr;
|
|
184
|
+
}));
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
SystemAudioCapturePlugin::~SystemAudioCapturePlugin() {
|
|
188
|
+
StopCapture();
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
void SystemAudioCapturePlugin::HandleMethodCall(
|
|
192
|
+
const flutter::MethodCall<flutter::EncodableValue> &method_call,
|
|
193
|
+
std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result) {
|
|
194
|
+
if (method_call.method_name().compare("requestPermissions") == 0) {
|
|
195
|
+
// On Windows, permissions are typically handled by the system
|
|
196
|
+
result->Success(flutter::EncodableValue(true));
|
|
197
|
+
} else if (method_call.method_name().compare("startCapture") == 0) {
|
|
198
|
+
const flutter::EncodableMap* args = nullptr;
|
|
199
|
+
if (method_call.arguments() &&
|
|
200
|
+
std::holds_alternative<flutter::EncodableMap>(*method_call.arguments())) {
|
|
201
|
+
args = &std::get<flutter::EncodableMap>(*method_call.arguments());
|
|
202
|
+
}
|
|
203
|
+
bool started = StartCapture(args);
|
|
204
|
+
result->Success(flutter::EncodableValue(started));
|
|
205
|
+
} else if (method_call.method_name().compare("stopCapture") == 0) {
|
|
206
|
+
bool stopped = StopCapture();
|
|
207
|
+
result->Success(flutter::EncodableValue(stopped));
|
|
208
|
+
} else {
|
|
209
|
+
result->NotImplemented();
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
void SystemAudioCapturePlugin::ApplyGainBoostAndConvertToMono(
|
|
214
|
+
const int16_t* input, int16_t* output, size_t frame_count,
|
|
215
|
+
int input_channels, float gain_boost) {
|
|
216
|
+
const float max_value = 32767.0f;
|
|
217
|
+
const float min_value = -32768.0f;
|
|
218
|
+
|
|
219
|
+
if (input_channels == 1) {
|
|
220
|
+
// Mono: just apply gain boost
|
|
221
|
+
for (size_t i = 0; i < frame_count; ++i) {
|
|
222
|
+
float sample = static_cast<float>(input[i]) * gain_boost;
|
|
223
|
+
float clamped_sample = (std::min)(max_value, sample);
|
|
224
|
+
sample = (std::max)(min_value, clamped_sample);
|
|
225
|
+
output[i] = static_cast<int16_t>(sample);
|
|
226
|
+
}
|
|
227
|
+
} else {
|
|
228
|
+
// Stereo: convert to mono and apply gain boost
|
|
229
|
+
for (size_t i = 0; i < frame_count; ++i) {
|
|
230
|
+
float left = static_cast<float>(input[i * 2]);
|
|
231
|
+
float right = static_cast<float>(input[i * 2 + 1]);
|
|
232
|
+
float mono = (left + right) / 2.0f * gain_boost;
|
|
233
|
+
float clamped_mono = (std::min)(max_value, mono);
|
|
234
|
+
mono = (std::max)(min_value, clamped_mono);
|
|
235
|
+
output[i] = static_cast<int16_t>(mono);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
double SystemAudioCapturePlugin::CalculateDecibel(const int16_t* samples,
|
|
241
|
+
size_t sample_count) {
|
|
242
|
+
if (sample_count == 0) {
|
|
243
|
+
return -120.0;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Calculate RMS (Root Mean Square)
|
|
247
|
+
double sum_of_squares = 0.0;
|
|
248
|
+
for (size_t i = 0; i < sample_count; ++i) {
|
|
249
|
+
double value = static_cast<double>(samples[i]);
|
|
250
|
+
sum_of_squares += value * value;
|
|
251
|
+
}
|
|
252
|
+
double mean_square = sum_of_squares / static_cast<double>(sample_count);
|
|
253
|
+
double rms = sqrt(mean_square);
|
|
254
|
+
|
|
255
|
+
// Calculate decibel: dB = 20 * log10(RMS / max_value)
|
|
256
|
+
const double max_value = 32767.0;
|
|
257
|
+
if (rms <= 0.0) {
|
|
258
|
+
return -120.0; // Avoid log(0)
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
double decibel = 20.0 * log10(rms / max_value);
|
|
262
|
+
|
|
263
|
+
// Clamp to reasonable range (-120 dB to 0 dB)
|
|
264
|
+
double clamped_decibel = (std::min)(0.0, decibel);
|
|
265
|
+
return (std::max)(-120.0, clamped_decibel);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
void SystemAudioCapturePlugin::SendStatusUpdate(bool is_active) {
|
|
269
|
+
try {
|
|
270
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
271
|
+
if (status_event_sink_) {
|
|
272
|
+
flutter::EncodableMap status_map;
|
|
273
|
+
status_map[flutter::EncodableValue("isActive")] = flutter::EncodableValue(is_active);
|
|
274
|
+
|
|
275
|
+
// Get current timestamp in seconds
|
|
276
|
+
auto now = std::chrono::system_clock::now();
|
|
277
|
+
auto duration = now.time_since_epoch();
|
|
278
|
+
auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
|
|
279
|
+
double timestamp = static_cast<double>(milliseconds) / 1000.0;
|
|
280
|
+
|
|
281
|
+
status_map[flutter::EncodableValue("timestamp")] = flutter::EncodableValue(timestamp);
|
|
282
|
+
status_event_sink_->Success(flutter::EncodableValue(status_map));
|
|
283
|
+
}
|
|
284
|
+
} catch (...) {
|
|
285
|
+
// Silently ignore errors to prevent crash
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
void SystemAudioCapturePlugin::SendDecibelUpdate(double decibel) {
|
|
290
|
+
try {
|
|
291
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
292
|
+
if (decibel_event_sink_) {
|
|
293
|
+
flutter::EncodableMap decibel_map;
|
|
294
|
+
decibel_map[flutter::EncodableValue("decibel")] = flutter::EncodableValue(decibel);
|
|
295
|
+
|
|
296
|
+
// Get current timestamp in seconds
|
|
297
|
+
auto now = std::chrono::system_clock::now();
|
|
298
|
+
auto duration = now.time_since_epoch();
|
|
299
|
+
auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
|
|
300
|
+
double timestamp = static_cast<double>(milliseconds) / 1000.0;
|
|
301
|
+
|
|
302
|
+
decibel_map[flutter::EncodableValue("timestamp")] = flutter::EncodableValue(timestamp);
|
|
303
|
+
decibel_event_sink_->Success(flutter::EncodableValue(decibel_map));
|
|
304
|
+
}
|
|
305
|
+
} catch (...) {
|
|
306
|
+
// Silently ignore errors to prevent crash
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
bool SystemAudioCapturePlugin::StartCapture(const flutter::EncodableMap* args) {
|
|
311
|
+
// Always cleanup any existing capture first
|
|
312
|
+
{
|
|
313
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
314
|
+
if (is_capturing_) {
|
|
315
|
+
should_stop_ = true;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// Wait for existing capture thread to finish
|
|
320
|
+
if (capture_thread_.joinable()) {
|
|
321
|
+
capture_thread_.join();
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// Cleanup any remaining resources (without lock to avoid deadlock)
|
|
325
|
+
if (audio_client_) {
|
|
326
|
+
audio_client_->Stop();
|
|
327
|
+
audio_client_->Release();
|
|
328
|
+
audio_client_ = nullptr;
|
|
329
|
+
}
|
|
330
|
+
if (capture_client_) {
|
|
331
|
+
capture_client_->Release();
|
|
332
|
+
capture_client_ = nullptr;
|
|
333
|
+
}
|
|
334
|
+
if (mix_format_) {
|
|
335
|
+
CoTaskMemFree(mix_format_);
|
|
336
|
+
mix_format_ = nullptr;
|
|
337
|
+
}
|
|
338
|
+
if (device_) {
|
|
339
|
+
device_->Release();
|
|
340
|
+
device_ = nullptr;
|
|
341
|
+
}
|
|
342
|
+
if (com_initialized_) {
|
|
343
|
+
CoUninitialize();
|
|
344
|
+
com_initialized_ = false;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// Parse arguments
|
|
348
|
+
if (args) {
|
|
349
|
+
auto it = args->find(flutter::EncodableValue("sampleRate"));
|
|
350
|
+
if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
|
|
351
|
+
sample_rate_ = std::get<int32_t>(it->second);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
it = args->find(flutter::EncodableValue("channels"));
|
|
355
|
+
if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
|
|
356
|
+
channels_ = std::get<int32_t>(it->second);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
it = args->find(flutter::EncodableValue("bitsPerSample"));
|
|
360
|
+
if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
|
|
361
|
+
bits_per_sample_ = std::get<int32_t>(it->second);
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
it = args->find(flutter::EncodableValue("chunkDurationMs"));
|
|
365
|
+
if (it != args->end() && std::holds_alternative<int32_t>(it->second)) {
|
|
366
|
+
chunk_duration_ms_ = std::get<int32_t>(it->second);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
it = args->find(flutter::EncodableValue("gainBoost"));
|
|
370
|
+
if (it != args->end() && std::holds_alternative<double>(it->second)) {
|
|
371
|
+
gain_boost_ = static_cast<float>(std::get<double>(it->second));
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
it = args->find(flutter::EncodableValue("inputVolume"));
|
|
375
|
+
if (it != args->end() && std::holds_alternative<double>(it->second)) {
|
|
376
|
+
input_volume_ = static_cast<float>(std::get<double>(it->second));
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
// Clamp values
|
|
381
|
+
sample_rate_ = (std::max)(sample_rate_, 8000);
|
|
382
|
+
int min_channels = (std::min)(channels_, 2);
|
|
383
|
+
channels_ = (std::max)(1, min_channels);
|
|
384
|
+
bits_per_sample_ = 16;
|
|
385
|
+
chunk_duration_ms_ = (std::max)(chunk_duration_ms_, 10);
|
|
386
|
+
float min_gain = (std::min)(10.0f, gain_boost_);
|
|
387
|
+
gain_boost_ = (std::max)(0.1f, min_gain);
|
|
388
|
+
float min_volume = (std::min)(1.0f, input_volume_);
|
|
389
|
+
input_volume_ = (std::max)(0.0f, min_volume);
|
|
390
|
+
|
|
391
|
+
// Initialize COM (allow RPC_E_CHANGED_MODE if already initialized)
|
|
392
|
+
HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
|
|
393
|
+
if (FAILED(hr) && hr != RPC_E_CHANGED_MODE) {
|
|
394
|
+
return false;
|
|
395
|
+
}
|
|
396
|
+
com_initialized_ = (hr == S_OK); // Only true if we initialized it
|
|
397
|
+
|
|
398
|
+
// Get default audio endpoint (eRender for loopback)
|
|
399
|
+
IMMDeviceEnumerator* enumerator = nullptr;
|
|
400
|
+
hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL,
|
|
401
|
+
__uuidof(IMMDeviceEnumerator),
|
|
402
|
+
reinterpret_cast<void**>(&enumerator));
|
|
403
|
+
if (FAILED(hr)) {
|
|
404
|
+
if (com_initialized_) {
|
|
405
|
+
CoUninitialize();
|
|
406
|
+
}
|
|
407
|
+
return false;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
hr = enumerator->GetDefaultAudioEndpoint(eRender, eConsole, &device_);
|
|
411
|
+
enumerator->Release();
|
|
412
|
+
if (FAILED(hr)) {
|
|
413
|
+
if (com_initialized_) {
|
|
414
|
+
CoUninitialize();
|
|
415
|
+
}
|
|
416
|
+
return false;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// Activate IAudioClient
|
|
420
|
+
hr = device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr,
|
|
421
|
+
reinterpret_cast<void**>(&audio_client_));
|
|
422
|
+
if (FAILED(hr)) {
|
|
423
|
+
device_->Release();
|
|
424
|
+
device_ = nullptr;
|
|
425
|
+
if (com_initialized_) {
|
|
426
|
+
CoUninitialize();
|
|
427
|
+
}
|
|
428
|
+
return false;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
// Get mix format - use original format for initialization
|
|
432
|
+
WAVEFORMATEX* device_format = nullptr;
|
|
433
|
+
hr = audio_client_->GetMixFormat(&device_format);
|
|
434
|
+
if (FAILED(hr)) {
|
|
435
|
+
audio_client_->Release();
|
|
436
|
+
audio_client_ = nullptr;
|
|
437
|
+
device_->Release();
|
|
438
|
+
device_ = nullptr;
|
|
439
|
+
if (com_initialized_) {
|
|
440
|
+
CoUninitialize();
|
|
441
|
+
}
|
|
442
|
+
return false;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
// Store device format for use in capture thread
|
|
446
|
+
// We'll use the device's native format and convert in the thread
|
|
447
|
+
mix_format_ = device_format;
|
|
448
|
+
|
|
449
|
+
// Initialize audio client for loopback with device's native format
|
|
450
|
+
// FIX LATENCY: Giảm buffer duration xuống 100ms để giảm latency
|
|
451
|
+
REFERENCE_TIME hnsRequestedDuration = REFTIMES_PER_SEC / 10; // 100ms instead of 1 second
|
|
452
|
+
hr = audio_client_->Initialize(AUDCLNT_SHAREMODE_SHARED,
|
|
453
|
+
AUDCLNT_STREAMFLAGS_LOOPBACK,
|
|
454
|
+
hnsRequestedDuration, 0, mix_format_, nullptr);
|
|
455
|
+
if (FAILED(hr)) {
|
|
456
|
+
CoTaskMemFree(mix_format_);
|
|
457
|
+
mix_format_ = nullptr;
|
|
458
|
+
audio_client_->Release();
|
|
459
|
+
audio_client_ = nullptr;
|
|
460
|
+
device_->Release();
|
|
461
|
+
device_ = nullptr;
|
|
462
|
+
if (com_initialized_) {
|
|
463
|
+
CoUninitialize();
|
|
464
|
+
}
|
|
465
|
+
return false;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// Get buffer size
|
|
469
|
+
hr = audio_client_->GetBufferSize(&buffer_frame_count_);
|
|
470
|
+
if (FAILED(hr)) {
|
|
471
|
+
CoTaskMemFree(mix_format_);
|
|
472
|
+
mix_format_ = nullptr;
|
|
473
|
+
audio_client_->Release();
|
|
474
|
+
audio_client_ = nullptr;
|
|
475
|
+
device_->Release();
|
|
476
|
+
device_ = nullptr;
|
|
477
|
+
if (com_initialized_) {
|
|
478
|
+
CoUninitialize();
|
|
479
|
+
}
|
|
480
|
+
return false;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// Get IAudioCaptureClient
|
|
484
|
+
hr = audio_client_->GetService(__uuidof(IAudioCaptureClient),
|
|
485
|
+
reinterpret_cast<void**>(&capture_client_));
|
|
486
|
+
if (FAILED(hr)) {
|
|
487
|
+
CoTaskMemFree(mix_format_);
|
|
488
|
+
mix_format_ = nullptr;
|
|
489
|
+
audio_client_->Release();
|
|
490
|
+
audio_client_ = nullptr;
|
|
491
|
+
device_->Release();
|
|
492
|
+
device_ = nullptr;
|
|
493
|
+
if (com_initialized_) {
|
|
494
|
+
CoUninitialize();
|
|
495
|
+
}
|
|
496
|
+
return false;
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
// Start capture
|
|
500
|
+
hr = audio_client_->Start();
|
|
501
|
+
if (FAILED(hr)) {
|
|
502
|
+
capture_client_->Release();
|
|
503
|
+
capture_client_ = nullptr;
|
|
504
|
+
CoTaskMemFree(mix_format_);
|
|
505
|
+
mix_format_ = nullptr;
|
|
506
|
+
audio_client_->Release();
|
|
507
|
+
audio_client_ = nullptr;
|
|
508
|
+
device_->Release();
|
|
509
|
+
device_ = nullptr;
|
|
510
|
+
if (com_initialized_) {
|
|
511
|
+
CoUninitialize();
|
|
512
|
+
}
|
|
513
|
+
return false;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
{
|
|
517
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
518
|
+
should_stop_ = false;
|
|
519
|
+
is_capturing_ = true;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// Start capture thread
|
|
523
|
+
capture_thread_ = std::thread(&SystemAudioCapturePlugin::CaptureThread, this);
|
|
524
|
+
|
|
525
|
+
// Wait a bit to ensure thread has started
|
|
526
|
+
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
|
527
|
+
|
|
528
|
+
SendStatusUpdate(true);
|
|
529
|
+
|
|
530
|
+
return true;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
bool SystemAudioCapturePlugin::StopCapture() {
|
|
534
|
+
{
|
|
535
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
536
|
+
if (!is_capturing_) {
|
|
537
|
+
return false;
|
|
538
|
+
}
|
|
539
|
+
should_stop_ = true;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
if (capture_thread_.joinable()) {
|
|
543
|
+
capture_thread_.join();
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
{
|
|
547
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
548
|
+
is_capturing_ = false;
|
|
549
|
+
|
|
550
|
+
// Cleanup WASAPI resources
|
|
551
|
+
if (audio_client_) {
|
|
552
|
+
audio_client_->Stop();
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
if (capture_client_) {
|
|
556
|
+
capture_client_->Release();
|
|
557
|
+
capture_client_ = nullptr;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
if (mix_format_) {
|
|
561
|
+
CoTaskMemFree(mix_format_);
|
|
562
|
+
mix_format_ = nullptr;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
if (audio_client_) {
|
|
566
|
+
audio_client_->Release();
|
|
567
|
+
audio_client_ = nullptr;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
if (device_) {
|
|
571
|
+
device_->Release();
|
|
572
|
+
device_ = nullptr;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// Only uninitialize COM if we initialized it
|
|
576
|
+
if (com_initialized_) {
|
|
577
|
+
CoUninitialize();
|
|
578
|
+
com_initialized_ = false;
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
SendStatusUpdate(false);
|
|
583
|
+
|
|
584
|
+
return true;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
void SystemAudioCapturePlugin::CaptureThread() {
|
|
588
|
+
try {
|
|
589
|
+
// Set thread priority to reduce latency
|
|
590
|
+
SetThreadPriority();
|
|
591
|
+
|
|
592
|
+
if (!mix_format_ || !capture_client_) {
|
|
593
|
+
return;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
// Use actual format from device
|
|
597
|
+
const UINT32 frame_size = mix_format_->nBlockAlign;
|
|
598
|
+
const UINT32 actual_sample_rate = mix_format_->nSamplesPerSec;
|
|
599
|
+
const WORD actual_channels = mix_format_->nChannels;
|
|
600
|
+
const WORD actual_bits_per_sample = mix_format_->wBitsPerSample;
|
|
601
|
+
|
|
602
|
+
// FIX LATENCY 1: Giảm chunk size xuống tối đa 50ms để giảm delay
|
|
603
|
+
// Sử dụng chunk nhỏ hơn để gửi data nhanh hơn
|
|
604
|
+
const int effective_chunk_ms = (std::max)(20, (std::min)(chunk_duration_ms_, 50));
|
|
605
|
+
|
|
606
|
+
// Calculate smaller chunk size for lower latency
|
|
607
|
+
const size_t chunk_frames = (actual_sample_rate * effective_chunk_ms / 1000);
|
|
608
|
+
const size_t chunk_size_bytes = chunk_frames * frame_size;
|
|
609
|
+
const size_t output_frame_count = (sample_rate_ * effective_chunk_ms / 1000);
|
|
610
|
+
|
|
611
|
+
std::vector<uint8_t> raw_buffer(chunk_size_bytes * 2); // Double buffer for safety
|
|
612
|
+
std::vector<int16_t> output_buffer(output_frame_count);
|
|
613
|
+
size_t raw_buffer_pos = 0;
|
|
614
|
+
|
|
615
|
+
// FIX LATENCY 2: Giảm sleep time xuống tối thiểu để giảm delay
|
|
616
|
+
const int sleep_time_ms = 1; // Giảm xuống 1ms để phản hồi nhanh hơn
|
|
617
|
+
|
|
618
|
+
while (!should_stop_) {
|
|
619
|
+
UINT32 num_frames_available = 0;
|
|
620
|
+
HRESULT hr = capture_client_->GetNextPacketSize(&num_frames_available);
|
|
621
|
+
|
|
622
|
+
if (FAILED(hr)) {
|
|
623
|
+
break;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// FIX LATENCY 3: Process ngay khi có data, không đợi nhiều packets
|
|
627
|
+
if (num_frames_available == 0) {
|
|
628
|
+
std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time_ms));
|
|
629
|
+
continue;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
while (num_frames_available > 0 && !should_stop_) {
|
|
633
|
+
BYTE* data = nullptr;
|
|
634
|
+
UINT32 num_frames = 0;
|
|
635
|
+
DWORD flags = 0;
|
|
636
|
+
UINT64 device_position = 0;
|
|
637
|
+
UINT64 qpc_position = 0;
|
|
638
|
+
|
|
639
|
+
hr = capture_client_->GetBuffer(&data, &num_frames, &flags,
|
|
640
|
+
&device_position, &qpc_position);
|
|
641
|
+
|
|
642
|
+
if (FAILED(hr)) {
|
|
643
|
+
break;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
bool is_silent = (flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0;
|
|
647
|
+
|
|
648
|
+
if (!is_silent && data != nullptr && num_frames > 0) {
|
|
649
|
+
const size_t data_size = num_frames * frame_size;
|
|
650
|
+
size_t data_offset = 0;
|
|
651
|
+
|
|
652
|
+
while (data_offset < data_size && !should_stop_) {
|
|
653
|
+
const size_t space_available = raw_buffer.size() - raw_buffer_pos;
|
|
654
|
+
const size_t data_remaining = data_size - data_offset;
|
|
655
|
+
const size_t copy_size = (std::min)(space_available, data_remaining);
|
|
656
|
+
|
|
657
|
+
if (copy_size > 0) {
|
|
658
|
+
memcpy(raw_buffer.data() + raw_buffer_pos,
|
|
659
|
+
reinterpret_cast<const uint8_t*>(data) + data_offset,
|
|
660
|
+
copy_size);
|
|
661
|
+
raw_buffer_pos += copy_size;
|
|
662
|
+
data_offset += copy_size;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
// FIX LATENCY 4: Gửi data ngay khi đủ chunk nhỏ, không đợi buffer đầy
|
|
666
|
+
if (raw_buffer_pos >= chunk_size_bytes) {
|
|
667
|
+
const size_t input_frame_count = chunk_size_bytes / frame_size;
|
|
668
|
+
const size_t total_samples = input_frame_count * actual_channels;
|
|
669
|
+
|
|
670
|
+
std::vector<int16_t> converted_samples(total_samples);
|
|
671
|
+
bool conversion_success = false;
|
|
672
|
+
|
|
673
|
+
// Convert based on format
|
|
674
|
+
if (actual_bits_per_sample == 16 &&
|
|
675
|
+
(mix_format_->wFormatTag == WAVE_FORMAT_PCM ||
|
|
676
|
+
mix_format_->wFormatTag == WAVE_FORMAT_EXTENSIBLE)) {
|
|
677
|
+
const int16_t* raw_samples = reinterpret_cast<const int16_t*>(raw_buffer.data());
|
|
678
|
+
converted_samples.assign(raw_samples, raw_samples + total_samples);
|
|
679
|
+
conversion_success = true;
|
|
680
|
+
} else if (actual_bits_per_sample == 32 &&
|
|
681
|
+
(mix_format_->wFormatTag == WAVE_FORMAT_IEEE_FLOAT ||
|
|
682
|
+
mix_format_->wFormatTag == WAVE_FORMAT_EXTENSIBLE)) {
|
|
683
|
+
const float* float_samples = reinterpret_cast<const float*>(raw_buffer.data());
|
|
684
|
+
for (size_t i = 0; i < total_samples; ++i) {
|
|
685
|
+
float sample = (std::min)(1.0f, (std::max)(-1.0f, float_samples[i]));
|
|
686
|
+
converted_samples[i] = static_cast<int16_t>(sample * 32767.0f);
|
|
687
|
+
}
|
|
688
|
+
conversion_success = true;
|
|
689
|
+
} else if (actual_bits_per_sample == 24) {
|
|
690
|
+
const uint8_t* raw_bytes = raw_buffer.data();
|
|
691
|
+
for (size_t i = 0; i < total_samples; ++i) {
|
|
692
|
+
size_t byte_offset = i * 3;
|
|
693
|
+
int32_t sample24 = (static_cast<int32_t>(raw_bytes[byte_offset]) |
|
|
694
|
+
(static_cast<int32_t>(raw_bytes[byte_offset + 1]) << 8) |
|
|
695
|
+
(static_cast<int32_t>(raw_bytes[byte_offset + 2]) << 16));
|
|
696
|
+
if (sample24 & 0x800000) {
|
|
697
|
+
sample24 |= 0xFF000000;
|
|
698
|
+
}
|
|
699
|
+
converted_samples[i] = static_cast<int16_t>(sample24 >> 8);
|
|
700
|
+
}
|
|
701
|
+
conversion_success = true;
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
if (!conversion_success) {
|
|
705
|
+
raw_buffer_pos = 0;
|
|
706
|
+
continue;
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
// Apply input volume if needed
|
|
710
|
+
if (input_volume_ > 0.0f && input_volume_ < 1.0f) {
|
|
711
|
+
for (size_t i = 0; i < total_samples; ++i) {
|
|
712
|
+
converted_samples[i] = static_cast<int16_t>(
|
|
713
|
+
static_cast<float>(converted_samples[i]) * input_volume_);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
const size_t frames_to_process = converted_samples.size() / actual_channels;
|
|
718
|
+
const size_t output_frames = (std::min)(frames_to_process, output_frame_count);
|
|
719
|
+
|
|
720
|
+
ApplyGainBoostAndConvertToMono(converted_samples.data(), output_buffer.data(),
|
|
721
|
+
output_frames, actual_channels, gain_boost_);
|
|
722
|
+
|
|
723
|
+
double decibel = CalculateDecibel(output_buffer.data(), output_frames);
|
|
724
|
+
|
|
725
|
+
// FIX LATENCY 5: Tối ưu mutex - giữ lock thời gian ngắn nhất
|
|
726
|
+
{
|
|
727
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
728
|
+
if (event_sink_) {
|
|
729
|
+
const size_t output_bytes = output_frames * sizeof(int16_t);
|
|
730
|
+
std::vector<uint8_t> audio_data(
|
|
731
|
+
reinterpret_cast<uint8_t*>(output_buffer.data()),
|
|
732
|
+
reinterpret_cast<uint8_t*>(output_buffer.data()) + output_bytes);
|
|
733
|
+
event_sink_->Success(flutter::EncodableValue(audio_data));
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
SendDecibelUpdate(decibel);
|
|
738
|
+
|
|
739
|
+
// Move remaining data
|
|
740
|
+
if (raw_buffer_pos > chunk_size_bytes) {
|
|
741
|
+
const size_t remaining = raw_buffer_pos - chunk_size_bytes;
|
|
742
|
+
memmove(raw_buffer.data(), raw_buffer.data() + chunk_size_bytes, remaining);
|
|
743
|
+
raw_buffer_pos = remaining;
|
|
744
|
+
} else {
|
|
745
|
+
raw_buffer_pos = 0;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
hr = capture_client_->ReleaseBuffer(num_frames);
|
|
752
|
+
if (FAILED(hr)) {
|
|
753
|
+
break;
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
hr = capture_client_->GetNextPacketSize(&num_frames_available);
|
|
757
|
+
if (FAILED(hr)) {
|
|
758
|
+
break;
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
} catch (...) {
|
|
763
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
764
|
+
is_capturing_ = false;
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
// BONUS: Thêm method để set high priority cho thread để giảm latency
|
|
769
|
+
// Gọi ở đầu CaptureThread() để giảm latency hơn nữa
|
|
770
|
+
void SystemAudioCapturePlugin::SetThreadPriority() {
|
|
771
|
+
HANDLE current_thread = GetCurrentThread();
|
|
772
|
+
// Sử dụng THREAD_PRIORITY_HIGHEST thay vì TIME_CRITICAL để tránh gây vấn đề
|
|
773
|
+
::SetThreadPriority(current_thread, THREAD_PRIORITY_HIGHEST);
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
} // namespace audio_capture
|
|
777
|
+
|