neoagent 3.1.1-beta.9 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -2
- package/README.md +5 -1
- package/docs/architecture.md +1 -1
- package/docs/benchmarking.md +68 -87
- package/docs/capabilities.md +1 -1
- package/docs/clients-and-devices.md +3 -4
- package/docs/configuration.md +1 -1
- package/docs/devices.md +1 -2
- package/docs/health.md +24 -0
- package/docs/index.md +2 -2
- package/docs/integrations-architecture.md +6 -0
- package/docs/integrations.md +15 -1
- package/docs/models.md +1 -1
- package/docs/persistence.md +1 -1
- package/docs/why-neoagent.md +1 -1
- package/flutter_app/README.md +0 -5
- package/flutter_app/android/app/src/main/AndroidManifest.xml +0 -14
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +0 -98
- package/flutter_app/lib/main.dart +0 -6
- package/flutter_app/lib/main_app_shell.dart +0 -2
- package/flutter_app/lib/main_controller.dart +2 -867
- package/flutter_app/lib/main_devices.dart +0 -73
- package/flutter_app/lib/main_integrations.dart +339 -187
- package/flutter_app/lib/main_launcher.dart +2 -212
- package/flutter_app/lib/main_models.dart +0 -259
- package/flutter_app/lib/main_navigation.dart +2 -14
- package/flutter_app/lib/main_operations.dart +322 -364
- package/flutter_app/lib/main_runtime.dart +9 -208
- package/flutter_app/lib/main_settings.dart +17 -199
- package/flutter_app/lib/main_shared.dart +20 -591
- package/flutter_app/lib/main_timeline.dart +26 -122
- package/flutter_app/lib/main_voice_assistant.dart +0 -5
- package/flutter_app/lib/src/backend_client.dart +0 -102
- package/flutter_app/lib/src/desktop_companion_io.dart +1 -65
- package/flutter_app/lib/src/desktop_companion_stub.dart +0 -12
- package/flutter_app/linux/flutter/generated_plugin_registrant.cc +0 -4
- package/flutter_app/linux/flutter/generated_plugins.cmake +0 -1
- package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +0 -2
- package/flutter_app/macos/Podfile.lock +56 -5
- package/flutter_app/pubspec.lock +0 -7
- package/flutter_app/pubspec.yaml +0 -2
- package/flutter_app/web/index.html +174 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.cc +0 -3
- package/flutter_app/windows/flutter/generated_plugins.cmake +0 -1
- package/landing/index.html +0 -2
- package/lib/manager.js +1 -1
- package/lib/schema_migrations.js +75 -0
- package/package.json +1 -4
- package/server/db/database.js +1 -133
- package/server/http/middleware.js +2 -16
- package/server/http/routes.js +0 -2
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/NOTICES +0 -25
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/index.html +174 -0
- package/server/public/main.dart.js +88081 -90036
- package/server/routes/auth.js +11 -2
- package/server/routes/settings.js +0 -5
- package/server/routes/voice_assistant.js +1 -128
- package/server/services/account/erasure.js +0 -3
- package/server/services/ai/deliverables/artifact_helpers.js +1 -1
- package/server/services/ai/loop/conversation_loop.js +6 -2
- package/server/services/ai/loop/tool_dispatch.js +0 -3
- package/server/services/ai/messagingFallback.js +1 -1
- package/server/services/ai/settings.js +0 -6
- package/server/services/ai/toolEvidence.js +1 -4
- package/server/services/ai/tools.js +0 -234
- package/server/services/desktop/auth.js +0 -3
- package/server/services/desktop/registry.js +2 -50
- package/server/services/integrations/neoarchive/provider.js +616 -0
- package/server/services/integrations/neorecall/client.js +140 -0
- package/server/services/integrations/neorecall/constants.js +87 -0
- package/server/services/integrations/neorecall/provider.js +163 -0
- package/server/services/integrations/registry.js +4 -2
- package/server/services/manager.js +0 -22
- package/server/services/memory/manager.js +44 -19
- package/server/services/memory/retrieval_reasoning.js +36 -14
- package/server/services/messaging/whatsapp.js +18 -0
- package/server/services/security/tool_categories.js +0 -5
- package/server/services/social_video/service.js +2 -20
- package/server/services/tasks/adapters/index.js +1 -1
- package/server/services/tasks/integration_runtime.js +3 -67
- package/server/services/tasks/runtime.js +1 -6
- package/server/services/tasks/task_repository.js +6 -13
- package/server/services/timeline/service.js +0 -224
- package/server/services/voice/message.js +0 -24
- package/server/services/voice/providers.js +42 -1
- package/server/services/voice/runtimeManager.js +9 -5
- package/server/services/voice/turnRunner.js +0 -22
- package/server/services/wearable/service.js +0 -5
- package/docs/recordings-and-health.md +0 -41
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +0 -21
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +0 -586
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +0 -78
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +0 -104
- package/flutter_app/lib/main_recordings.dart +0 -920
- package/flutter_app/lib/src/desktop_ocr_bridge.dart +0 -2
- package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +0 -125
- package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +0 -30
- package/flutter_app/lib/src/desktop_passive_history.dart +0 -332
- package/flutter_app/lib/src/recording_bridge.dart +0 -232
- package/flutter_app/lib/src/recording_bridge_io.dart +0 -1027
- package/flutter_app/lib/src/recording_bridge_stub.dart +0 -120
- package/flutter_app/lib/src/recording_bridge_web.dart +0 -702
- package/flutter_app/lib/src/recording_chunk_queue.dart +0 -149
- package/flutter_app/lib/src/recording_chunk_queue_io.dart +0 -182
- package/flutter_app/lib/src/recording_payloads.dart +0 -95
- package/flutter_app/third_party/desktop_audio_capture/LICENSE +0 -21
- package/flutter_app/third_party/desktop_audio_capture/README.md +0 -262
- package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +0 -65
- package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +0 -153
- package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +0 -110
- package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +0 -461
- package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +0 -91
- package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +0 -106
- package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +0 -219
- package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +0 -336
- package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +0 -101
- package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +0 -692
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +0 -35
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +0 -36
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +0 -32
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +0 -32
- package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +0 -878
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +0 -27
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +0 -1172
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +0 -655
- package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +0 -12
- package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +0 -30
- package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +0 -87
- package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +0 -105
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +0 -80
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +0 -31
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +0 -12
- package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +0 -23
- package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +0 -25
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +0 -1117
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +0 -115
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +0 -777
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +0 -87
- package/server/routes/recordings.js +0 -335
- package/server/routes/screenHistory.js +0 -190
- package/server/services/ai/recordingInsights.js +0 -105
- package/server/services/integrations/neomail/provider.js +0 -993
- package/server/services/recordings/deepgram.js +0 -58
- package/server/services/recordings/manager.js +0 -1273
- package/server/services/tasks/adapters/neomail_email_received.js +0 -46
- package/server/services/voice/screenshotContext.js +0 -73
- package/server/services/voice/turnCoordinator.js +0 -51
|
@@ -34,10 +34,7 @@ class _TimelinePanelState extends State<TimelinePanel> {
|
|
|
34
34
|
Widget buildDetail(StateSetter setPopupState) {
|
|
35
35
|
final selectedEvent = items[selectedIndex];
|
|
36
36
|
void selectOffset(int offset) {
|
|
37
|
-
final nextIndex = (selectedIndex + offset).clamp(
|
|
38
|
-
0,
|
|
39
|
-
items.length - 1,
|
|
40
|
-
);
|
|
37
|
+
final nextIndex = (selectedIndex + offset).clamp(0, items.length - 1);
|
|
41
38
|
if (nextIndex == selectedIndex) {
|
|
42
39
|
return;
|
|
43
40
|
}
|
|
@@ -129,13 +126,12 @@ class _TimelinePanelState extends State<TimelinePanel> {
|
|
|
129
126
|
_PageTitle(
|
|
130
127
|
title: 'Timeline',
|
|
131
128
|
subtitle:
|
|
132
|
-
'Emails, AI actions,
|
|
129
|
+
'Emails, AI actions, tasks and run activity in one chronological feed.',
|
|
133
130
|
trailing: Wrap(
|
|
134
131
|
spacing: 10,
|
|
135
132
|
runSpacing: 10,
|
|
136
133
|
children: <Widget>[
|
|
137
134
|
for (final filter in const <({String id, String label})>[
|
|
138
|
-
(id: 'screen', label: 'Screen'),
|
|
139
135
|
(id: 'tasks', label: 'Tasks'),
|
|
140
136
|
(id: 'runs', label: 'Runs'),
|
|
141
137
|
])
|
|
@@ -150,9 +146,7 @@ class _TimelinePanelState extends State<TimelinePanel> {
|
|
|
150
146
|
_timelineLaneIcon(filter.id),
|
|
151
147
|
size: 16,
|
|
152
148
|
color:
|
|
153
|
-
controller.selectedTimelineSources.contains(
|
|
154
|
-
filter.id,
|
|
155
|
-
)
|
|
149
|
+
controller.selectedTimelineSources.contains(filter.id)
|
|
156
150
|
? _bgSecondary
|
|
157
151
|
: _sourceColorForKind(filter.id),
|
|
158
152
|
),
|
|
@@ -915,20 +909,7 @@ List<_TimelineDayGroup> _groupTimelineEvents(List<TimelineEventItem> items) {
|
|
|
915
909
|
|
|
916
910
|
List<_TimelineChip> _timelineEventChips(TimelineEventItem item) {
|
|
917
911
|
final chips = <_TimelineChip>[];
|
|
918
|
-
if (item.sourceKind == '
|
|
919
|
-
final duration = _timelineSpanDuration(item);
|
|
920
|
-
if (duration != null) {
|
|
921
|
-
chips.add(_TimelineChip(label: duration, color: _accent));
|
|
922
|
-
}
|
|
923
|
-
if (item.appName.trim().isNotEmpty) {
|
|
924
|
-
chips.add(_TimelineChip(label: item.appName.trim(), color: _info));
|
|
925
|
-
}
|
|
926
|
-
if (item.windowTitle.trim().isNotEmpty) {
|
|
927
|
-
chips.add(
|
|
928
|
-
_TimelineChip(label: item.windowTitle.trim(), color: _textSecondary),
|
|
929
|
-
);
|
|
930
|
-
}
|
|
931
|
-
} else if (item.sourceKind == 'tasks') {
|
|
912
|
+
if (item.sourceKind == 'tasks') {
|
|
932
913
|
chips.add(
|
|
933
914
|
_TimelineChip(
|
|
934
915
|
label: _titleCase(item.eventKind.replaceAll('_', ' ')),
|
|
@@ -950,7 +931,7 @@ List<_TimelineChip> _timelineEventChips(TimelineEventItem item) {
|
|
|
950
931
|
chips.add(_TimelineChip(label: 'Run linked', color: _accentAlt));
|
|
951
932
|
}
|
|
952
933
|
}
|
|
953
|
-
if (item.deviceLabel.trim().isNotEmpty
|
|
934
|
+
if (item.deviceLabel.trim().isNotEmpty) {
|
|
954
935
|
chips.add(_TimelineChip(label: item.deviceLabel.trim(), color: _info));
|
|
955
936
|
}
|
|
956
937
|
return chips.take(4).toList(growable: false);
|
|
@@ -975,26 +956,7 @@ List<_TimelineDetailCell> _timelineDetailCells(TimelineEventItem item) {
|
|
|
975
956
|
),
|
|
976
957
|
];
|
|
977
958
|
|
|
978
|
-
if (item.sourceKind == '
|
|
979
|
-
cells.add(
|
|
980
|
-
_TimelineDetailCell(
|
|
981
|
-
label: 'DURATION',
|
|
982
|
-
value: _timelineSpanDuration(item) ?? item.screenSpanLabel,
|
|
983
|
-
),
|
|
984
|
-
);
|
|
985
|
-
cells.add(
|
|
986
|
-
_TimelineDetailCell(
|
|
987
|
-
label: 'APP',
|
|
988
|
-
value: item.appName.ifEmpty('Unknown app'),
|
|
989
|
-
),
|
|
990
|
-
);
|
|
991
|
-
cells.add(
|
|
992
|
-
_TimelineDetailCell(
|
|
993
|
-
label: 'DEVICE',
|
|
994
|
-
value: item.deviceLabel.ifEmpty('Desktop'),
|
|
995
|
-
),
|
|
996
|
-
);
|
|
997
|
-
} else if (item.sourceKind == 'tasks') {
|
|
959
|
+
if (item.sourceKind == 'tasks') {
|
|
998
960
|
cells.add(
|
|
999
961
|
_TimelineDetailCell(
|
|
1000
962
|
label: 'TASK',
|
|
@@ -1049,12 +1011,6 @@ List<_TimelineDetailCell> _timelineDetailCells(TimelineEventItem item) {
|
|
|
1049
1011
|
|
|
1050
1012
|
String _timelineCardDescription(TimelineEventItem item) {
|
|
1051
1013
|
switch (item.sourceKind) {
|
|
1052
|
-
case 'screen':
|
|
1053
|
-
final preview = item.previewText.trim();
|
|
1054
|
-
if (preview.isNotEmpty) {
|
|
1055
|
-
return preview;
|
|
1056
|
-
}
|
|
1057
|
-
return '${item.deviceLabel.ifEmpty('Desktop')} · ${item.appName.ifEmpty('Unknown app')}';
|
|
1058
1014
|
case 'tasks':
|
|
1059
1015
|
case 'runs':
|
|
1060
1016
|
return item.summary.trim().ifEmpty(
|
|
@@ -1066,46 +1022,15 @@ String _timelineCardDescription(TimelineEventItem item) {
|
|
|
1066
1022
|
}
|
|
1067
1023
|
|
|
1068
1024
|
String _timelineDetailDescription(TimelineEventItem item) {
|
|
1069
|
-
final body = item.
|
|
1070
|
-
? item.previewText.trim()
|
|
1071
|
-
: item.summary.trim();
|
|
1025
|
+
final body = item.summary.trim();
|
|
1072
1026
|
if (body.isNotEmpty) {
|
|
1073
1027
|
return body;
|
|
1074
1028
|
}
|
|
1075
|
-
if (item.sourceKind == 'screen') {
|
|
1076
|
-
return '${item.appName.ifEmpty('Unknown app')} on ${item.deviceLabel.ifEmpty('Desktop')}';
|
|
1077
|
-
}
|
|
1078
1029
|
return _titleCase(item.eventKind.replaceAll('_', ' '));
|
|
1079
1030
|
}
|
|
1080
1031
|
|
|
1081
|
-
String? _timelineSpanDuration(TimelineEventItem item) {
|
|
1082
|
-
final start = item.startedAt;
|
|
1083
|
-
final end = item.endedAt;
|
|
1084
|
-
if (start == null || end == null) {
|
|
1085
|
-
return null;
|
|
1086
|
-
}
|
|
1087
|
-
final span = end.difference(start);
|
|
1088
|
-
if (span.inSeconds < 60) {
|
|
1089
|
-
return '${span.inSeconds}s';
|
|
1090
|
-
}
|
|
1091
|
-
if (span.inMinutes < 60) {
|
|
1092
|
-
final seconds = span.inSeconds % 60;
|
|
1093
|
-
if (seconds == 0) {
|
|
1094
|
-
return '${span.inMinutes}m';
|
|
1095
|
-
}
|
|
1096
|
-
return '${span.inMinutes}m ${seconds}s';
|
|
1097
|
-
}
|
|
1098
|
-
final minutes = span.inMinutes % 60;
|
|
1099
|
-
if (minutes == 0) {
|
|
1100
|
-
return '${span.inHours}h';
|
|
1101
|
-
}
|
|
1102
|
-
return '${span.inHours}h ${minutes}m';
|
|
1103
|
-
}
|
|
1104
|
-
|
|
1105
1032
|
Color _sourceColorForKind(String kind) {
|
|
1106
1033
|
switch (kind) {
|
|
1107
|
-
case 'screen':
|
|
1108
|
-
return _accent;
|
|
1109
1034
|
case 'tasks':
|
|
1110
1035
|
return _warning;
|
|
1111
1036
|
case 'runs':
|
|
@@ -1166,8 +1091,6 @@ String _weekdayShort(int weekday) {
|
|
|
1166
1091
|
|
|
1167
1092
|
IconData _timelineLaneIcon(String sourceKind) {
|
|
1168
1093
|
switch (sourceKind) {
|
|
1169
|
-
case 'screen':
|
|
1170
|
-
return Icons.desktop_windows_outlined;
|
|
1171
1094
|
case 'tasks':
|
|
1172
1095
|
return Icons.task_alt_outlined;
|
|
1173
1096
|
case 'runs':
|
|
@@ -1183,47 +1106,28 @@ List<Widget> _timelineDetailBody(
|
|
|
1183
1106
|
) {
|
|
1184
1107
|
final content = <Widget>[];
|
|
1185
1108
|
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
icon: Icons.web_asset_outlined,
|
|
1196
|
-
text: item.windowTitle,
|
|
1197
|
-
),
|
|
1198
|
-
_TimelineMetaLine(
|
|
1199
|
-
icon: Icons.schedule_outlined,
|
|
1200
|
-
text: item.screenSpanLabel,
|
|
1201
|
-
),
|
|
1202
|
-
]);
|
|
1203
|
-
} else {
|
|
1109
|
+
content.add(
|
|
1110
|
+
_TimelineMetaLine(
|
|
1111
|
+
icon: item.sourceKind == 'tasks'
|
|
1112
|
+
? Icons.task_alt_outlined
|
|
1113
|
+
: Icons.monitor_heart_outlined,
|
|
1114
|
+
text: _titleCase(item.eventKind.replaceAll('_', ' ')),
|
|
1115
|
+
),
|
|
1116
|
+
);
|
|
1117
|
+
if (item.runId.isNotEmpty && onOpenRun != null) {
|
|
1204
1118
|
content.add(
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
content.add(
|
|
1214
|
-
Padding(
|
|
1215
|
-
padding: const EdgeInsets.only(top: 8),
|
|
1216
|
-
child: Align(
|
|
1217
|
-
alignment: Alignment.centerLeft,
|
|
1218
|
-
child: TextButton.icon(
|
|
1219
|
-
onPressed: onOpenRun,
|
|
1220
|
-
icon: const Icon(Icons.open_in_new_rounded, size: 16),
|
|
1221
|
-
label: const Text('Open run'),
|
|
1222
|
-
),
|
|
1119
|
+
Padding(
|
|
1120
|
+
padding: const EdgeInsets.only(top: 8),
|
|
1121
|
+
child: Align(
|
|
1122
|
+
alignment: Alignment.centerLeft,
|
|
1123
|
+
child: TextButton.icon(
|
|
1124
|
+
onPressed: onOpenRun,
|
|
1125
|
+
icon: const Icon(Icons.open_in_new_rounded, size: 16),
|
|
1126
|
+
label: const Text('Open run'),
|
|
1223
1127
|
),
|
|
1224
1128
|
),
|
|
1225
|
-
)
|
|
1226
|
-
|
|
1129
|
+
),
|
|
1130
|
+
);
|
|
1227
1131
|
}
|
|
1228
1132
|
return content;
|
|
1229
1133
|
}
|
|
@@ -528,7 +528,6 @@ class _VoiceAssistantPanelState extends State<VoiceAssistantPanel> {
|
|
|
528
528
|
.toDouble();
|
|
529
529
|
final assistantUi = _DesktopAssistantControlState.fromController(
|
|
530
530
|
controller,
|
|
531
|
-
blockedHintVisible: false,
|
|
532
531
|
);
|
|
533
532
|
final liveCaptureEngaged = assistantUi.isCapturing;
|
|
534
533
|
final isBusy = _pttPressed || liveCaptureEngaged;
|
|
@@ -698,10 +697,6 @@ class _VoiceAssistantPanelState extends State<VoiceAssistantPanel> {
|
|
|
698
697
|
label: 'Refresh',
|
|
699
698
|
onTap: controller.ensureLiveVoiceSession,
|
|
700
699
|
),
|
|
701
|
-
_VoiceAssistantScreenContextButton(
|
|
702
|
-
controller: controller,
|
|
703
|
-
compact: false,
|
|
704
|
-
),
|
|
705
700
|
],
|
|
706
701
|
),
|
|
707
702
|
const SizedBox(height: 18),
|
|
@@ -1800,81 +1800,6 @@ class BackendClient {
|
|
|
1800
1800
|
return getList(baseUrl, _withAgentQuery('/api/mcp', agentId));
|
|
1801
1801
|
}
|
|
1802
1802
|
|
|
1803
|
-
Future<List<Map<String, dynamic>>> fetchRecordingSessions(
|
|
1804
|
-
String baseUrl, {
|
|
1805
|
-
int limit = 24,
|
|
1806
|
-
}) async {
|
|
1807
|
-
_log(
|
|
1808
|
-
'recording.fetch_sessions.request',
|
|
1809
|
-
data: <String, Object?>{'limit': limit},
|
|
1810
|
-
);
|
|
1811
|
-
return getList(baseUrl, '/api/recordings?limit=$limit');
|
|
1812
|
-
}
|
|
1813
|
-
|
|
1814
|
-
Future<Map<String, dynamic>> fetchRecordingSession(
|
|
1815
|
-
String baseUrl,
|
|
1816
|
-
String sessionId,
|
|
1817
|
-
) async {
|
|
1818
|
-
_log(
|
|
1819
|
-
'recording.fetch_session.request',
|
|
1820
|
-
data: <String, Object?>{'sessionId': sessionId},
|
|
1821
|
-
);
|
|
1822
|
-
return getMap(baseUrl, '/api/recordings/$sessionId');
|
|
1823
|
-
}
|
|
1824
|
-
|
|
1825
|
-
Future<Map<String, dynamic>> createRecordingSession(
|
|
1826
|
-
String baseUrl,
|
|
1827
|
-
Map<String, dynamic> payload,
|
|
1828
|
-
) async {
|
|
1829
|
-
_log(
|
|
1830
|
-
'recording.create.request',
|
|
1831
|
-
data: <String, Object?>{
|
|
1832
|
-
'platform': payload['platform']?.toString(),
|
|
1833
|
-
'sourceCount': (payload['sources'] as List?)?.length ?? 0,
|
|
1834
|
-
},
|
|
1835
|
-
);
|
|
1836
|
-
return postMap(baseUrl, '/api/recordings', payload);
|
|
1837
|
-
}
|
|
1838
|
-
|
|
1839
|
-
Future<Map<String, dynamic>> finalizeRecordingSession(
|
|
1840
|
-
String baseUrl,
|
|
1841
|
-
String sessionId, {
|
|
1842
|
-
String stopReason = 'stopped',
|
|
1843
|
-
}) async {
|
|
1844
|
-
_log(
|
|
1845
|
-
'recording.finalize.request',
|
|
1846
|
-
data: <String, Object?>{'sessionId': sessionId, 'stopReason': stopReason},
|
|
1847
|
-
);
|
|
1848
|
-
return postMap(
|
|
1849
|
-
baseUrl,
|
|
1850
|
-
'/api/recordings/$sessionId/finalize',
|
|
1851
|
-
<String, dynamic>{'stopReason': stopReason},
|
|
1852
|
-
);
|
|
1853
|
-
}
|
|
1854
|
-
|
|
1855
|
-
Future<Map<String, dynamic>> retryRecordingSession(
|
|
1856
|
-
String baseUrl,
|
|
1857
|
-
String sessionId,
|
|
1858
|
-
) async {
|
|
1859
|
-
return postMap(
|
|
1860
|
-
baseUrl,
|
|
1861
|
-
'/api/recordings/$sessionId/retry',
|
|
1862
|
-
const <String, dynamic>{},
|
|
1863
|
-
);
|
|
1864
|
-
}
|
|
1865
|
-
|
|
1866
|
-
Future<Map<String, dynamic>> deleteRecordingTranscriptSegment(
|
|
1867
|
-
String baseUrl,
|
|
1868
|
-
String sessionId,
|
|
1869
|
-
int segmentId,
|
|
1870
|
-
) async {
|
|
1871
|
-
return deleteMap(baseUrl, '/api/recordings/$sessionId/segments/$segmentId');
|
|
1872
|
-
}
|
|
1873
|
-
|
|
1874
|
-
Future<void> deleteRecordingSession(String baseUrl, String sessionId) async {
|
|
1875
|
-
await deleteMap(baseUrl, '/api/recordings/$sessionId');
|
|
1876
|
-
}
|
|
1877
|
-
|
|
1878
1803
|
Future<Map<String, dynamic>> transcribeAudio(
|
|
1879
1804
|
String baseUrl, {
|
|
1880
1805
|
required String audioBase64,
|
|
@@ -1887,33 +1812,6 @@ class BackendClient {
|
|
|
1887
1812
|
);
|
|
1888
1813
|
}
|
|
1889
1814
|
|
|
1890
|
-
Future<Map<String, dynamic>> runVoiceAssistantTurn(
|
|
1891
|
-
String baseUrl, {
|
|
1892
|
-
required String sessionId,
|
|
1893
|
-
String ttsProvider = 'openai',
|
|
1894
|
-
String ttsVoice = 'alloy',
|
|
1895
|
-
String ttsModel = 'gpt-4o-mini-tts',
|
|
1896
|
-
String? agentId,
|
|
1897
|
-
String? screenshotBase64,
|
|
1898
|
-
String? screenshotMimeType,
|
|
1899
|
-
}) async {
|
|
1900
|
-
final payload = <String, dynamic>{
|
|
1901
|
-
'sessionId': sessionId,
|
|
1902
|
-
'ttsProvider': ttsProvider,
|
|
1903
|
-
'ttsVoice': ttsVoice,
|
|
1904
|
-
'ttsModel': ttsModel,
|
|
1905
|
-
if ((screenshotBase64?.trim().isNotEmpty ?? false))
|
|
1906
|
-
'screenshotBase64': screenshotBase64!.trim(),
|
|
1907
|
-
if ((screenshotMimeType?.trim().isNotEmpty ?? false))
|
|
1908
|
-
'screenshotMimeType': screenshotMimeType!.trim(),
|
|
1909
|
-
};
|
|
1910
|
-
return postMap(
|
|
1911
|
-
baseUrl,
|
|
1912
|
-
_withAgentQuery('/api/voice-assistant/respond', agentId),
|
|
1913
|
-
payload,
|
|
1914
|
-
);
|
|
1915
|
-
}
|
|
1916
|
-
|
|
1917
1815
|
Future<Map<String, dynamic>> saveMcpServer(
|
|
1918
1816
|
String baseUrl, {
|
|
1919
1817
|
int? id,
|
|
@@ -7,8 +7,6 @@ import 'package:flutter/foundation.dart';
|
|
|
7
7
|
import 'package:shared_preferences/shared_preferences.dart';
|
|
8
8
|
|
|
9
9
|
import 'desktop_companion_actions.dart';
|
|
10
|
-
import 'desktop_ocr_bridge.dart';
|
|
11
|
-
import 'desktop_passive_history.dart';
|
|
12
10
|
import 'desktop_screen_capture.dart';
|
|
13
11
|
|
|
14
12
|
const String desktopCompanionEnabledPrefsKey = 'desktop.companion.enabled';
|
|
@@ -22,16 +20,9 @@ const String desktopCompanionActiveDisplayPrefsKey =
|
|
|
22
20
|
|
|
23
21
|
class DesktopCompanionManager extends ChangeNotifier {
|
|
24
22
|
DesktopCompanionManager({required DesktopScreenCapture screenCapture})
|
|
25
|
-
: _actions = DesktopCompanionActions(screenCapture: screenCapture)
|
|
26
|
-
_passiveHistory = DesktopPassiveHistoryManager(
|
|
27
|
-
actions: DesktopCompanionActions(screenCapture: screenCapture),
|
|
28
|
-
ocrBridge: createDesktopOcrBridge(),
|
|
29
|
-
) {
|
|
30
|
-
_passiveHistory.addListener(_handlePassiveHistoryChanged);
|
|
31
|
-
}
|
|
23
|
+
: _actions = DesktopCompanionActions(screenCapture: screenCapture);
|
|
32
24
|
|
|
33
25
|
final DesktopCompanionActions _actions;
|
|
34
|
-
final DesktopPassiveHistoryManager _passiveHistory;
|
|
35
26
|
WebSocket? _socket;
|
|
36
27
|
Timer? _reconnectTimer;
|
|
37
28
|
Timer? _connectionWatchdogTimer;
|
|
@@ -63,7 +54,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
63
54
|
Map<String, Object?> _status = const <String, Object?>{};
|
|
64
55
|
|
|
65
56
|
bool get enabled => _enabled;
|
|
66
|
-
bool get passiveHistoryEnabled => _passiveHistory.enabled;
|
|
67
57
|
bool get paused => _paused;
|
|
68
58
|
bool get connecting => _connecting;
|
|
69
59
|
bool get connected => _connected;
|
|
@@ -71,8 +61,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
71
61
|
String get label => _label;
|
|
72
62
|
String get deviceId => _deviceId;
|
|
73
63
|
String get activationId => _activationId;
|
|
74
|
-
String? get passiveHistoryLastUploadedAt => _passiveHistory.lastUploadedAt;
|
|
75
|
-
String? get passiveHistoryLastError => _passiveHistory.lastError;
|
|
76
64
|
Map<String, Object?> get status => _status;
|
|
77
65
|
|
|
78
66
|
Future<void> bootstrap(SharedPreferences prefs) async {
|
|
@@ -93,7 +81,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
93
81
|
'primary';
|
|
94
82
|
await prefs.setString(desktopCompanionDeviceIdPrefsKey, _deviceId);
|
|
95
83
|
await prefs.setString(desktopCompanionActivationIdPrefsKey, _activationId);
|
|
96
|
-
await _passiveHistory.bootstrap(prefs);
|
|
97
84
|
}
|
|
98
85
|
|
|
99
86
|
Future<void> updateSession({
|
|
@@ -104,7 +91,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
104
91
|
_backendUrl = backendUrl.trim();
|
|
105
92
|
_sessionCookie = sessionCookie.trim();
|
|
106
93
|
_authenticated = authenticated;
|
|
107
|
-
_syncPassiveHistoryRuntimeState();
|
|
108
94
|
if (!_authenticated || !_enabled || _sessionCookie.isEmpty) {
|
|
109
95
|
await disconnect();
|
|
110
96
|
return;
|
|
@@ -129,7 +115,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
129
115
|
await disconnect();
|
|
130
116
|
return;
|
|
131
117
|
}
|
|
132
|
-
_syncPassiveHistoryRuntimeState();
|
|
133
118
|
await _ensureConnected();
|
|
134
119
|
}
|
|
135
120
|
|
|
@@ -138,7 +123,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
138
123
|
_label = normalized;
|
|
139
124
|
await prefs.setString(desktopCompanionLabelPrefsKey, normalized);
|
|
140
125
|
notifyListeners();
|
|
141
|
-
_syncPassiveHistoryRuntimeState();
|
|
142
126
|
if (_connected) {
|
|
143
127
|
_status = {..._status, 'label': normalized};
|
|
144
128
|
await _sendEvent('statusChanged', <String, Object?>{'label': normalized});
|
|
@@ -148,24 +132,11 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
148
132
|
Future<void> setPaused(bool value, SharedPreferences prefs) async {
|
|
149
133
|
_paused = value;
|
|
150
134
|
notifyListeners();
|
|
151
|
-
_syncPassiveHistoryRuntimeState();
|
|
152
135
|
if (_connected) {
|
|
153
136
|
await _sendEvent('statusChanged', <String, Object?>{'paused': value});
|
|
154
137
|
}
|
|
155
138
|
}
|
|
156
139
|
|
|
157
|
-
Future<void> setPassiveHistoryEnabled(
|
|
158
|
-
bool value,
|
|
159
|
-
SharedPreferences prefs,
|
|
160
|
-
) async {
|
|
161
|
-
await _passiveHistory.setEnabled(value, prefs);
|
|
162
|
-
_syncPassiveHistoryRuntimeState();
|
|
163
|
-
_mergePassiveHistoryStatus();
|
|
164
|
-
if (_connected) {
|
|
165
|
-
await _sendEvent('statusChanged', _passiveHistory.statusPayload());
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
|
|
169
140
|
Future<void> disconnect() async {
|
|
170
141
|
_reconnectTimer?.cancel();
|
|
171
142
|
_reconnectTimer = null;
|
|
@@ -181,7 +152,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
181
152
|
await socket.close();
|
|
182
153
|
} catch (_) {}
|
|
183
154
|
}
|
|
184
|
-
_syncPassiveHistoryRuntimeState();
|
|
185
155
|
notifyListeners();
|
|
186
156
|
}
|
|
187
157
|
|
|
@@ -210,9 +180,7 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
210
180
|
'hostname': _localHostname(),
|
|
211
181
|
'companionEnabled': _enabled,
|
|
212
182
|
'paused': _paused,
|
|
213
|
-
..._passiveHistory.statusPayload(),
|
|
214
183
|
};
|
|
215
|
-
_syncPassiveHistoryRuntimeState();
|
|
216
184
|
notifyListeners();
|
|
217
185
|
if (_connected) {
|
|
218
186
|
await _sendEvent('statusChanged', <String, Object?>{
|
|
@@ -220,7 +188,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
220
188
|
'capabilities': _status['capabilities'],
|
|
221
189
|
'displays': _status['displays'],
|
|
222
190
|
'activeDisplayId': _status['activeDisplayId'],
|
|
223
|
-
..._passiveHistory.statusPayload(),
|
|
224
191
|
});
|
|
225
192
|
}
|
|
226
193
|
return _status;
|
|
@@ -280,7 +247,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
280
247
|
paused: _paused,
|
|
281
248
|
activeDisplayId: _activeDisplayId,
|
|
282
249
|
);
|
|
283
|
-
hello.addAll(_passiveHistory.statusPayload());
|
|
284
250
|
socket.add(
|
|
285
251
|
jsonEncode(<String, Object?>{'type': 'hello', 'device': hello}),
|
|
286
252
|
);
|
|
@@ -315,8 +281,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
315
281
|
_status = device is Map
|
|
316
282
|
? device.map((key, value) => MapEntry(key.toString(), value))
|
|
317
283
|
: const <String, Object?>{};
|
|
318
|
-
_mergePassiveHistoryStatus();
|
|
319
|
-
_syncPassiveHistoryRuntimeState();
|
|
320
284
|
_activeDisplayId =
|
|
321
285
|
_status['activeDisplayId']?.toString() ?? _activeDisplayId;
|
|
322
286
|
notifyListeners();
|
|
@@ -503,7 +467,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
503
467
|
_socket = null;
|
|
504
468
|
_connecting = false;
|
|
505
469
|
_connected = false;
|
|
506
|
-
_syncPassiveHistoryRuntimeState();
|
|
507
470
|
notifyListeners();
|
|
508
471
|
_scheduleReconnect();
|
|
509
472
|
}
|
|
@@ -518,8 +481,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
518
481
|
_connecting = false;
|
|
519
482
|
_connected = false;
|
|
520
483
|
_enabled = false;
|
|
521
|
-
_passiveHistory.removeListener(_handlePassiveHistoryChanged);
|
|
522
|
-
_passiveHistory.dispose();
|
|
523
484
|
final socket = _socket;
|
|
524
485
|
_socket = null;
|
|
525
486
|
if (socket != null) {
|
|
@@ -566,31 +527,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
566
527
|
);
|
|
567
528
|
}
|
|
568
529
|
|
|
569
|
-
void _syncPassiveHistoryRuntimeState() {
|
|
570
|
-
_passiveHistory.updateRuntimeState(
|
|
571
|
-
backendUrl: _backendUrl,
|
|
572
|
-
sessionCookie: _sessionCookie,
|
|
573
|
-
authenticated: _authenticated,
|
|
574
|
-
connected: _connected,
|
|
575
|
-
paused: _paused,
|
|
576
|
-
deviceId: _deviceId,
|
|
577
|
-
activationId: _activationId,
|
|
578
|
-
label: _label,
|
|
579
|
-
);
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
void _mergePassiveHistoryStatus() {
|
|
583
|
-
_status = <String, Object?>{..._status, ..._passiveHistory.statusPayload()};
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
void _handlePassiveHistoryChanged() {
|
|
587
|
-
_mergePassiveHistoryStatus();
|
|
588
|
-
if (_connected) {
|
|
589
|
-
unawaited(_sendEvent('statusChanged', _passiveHistory.statusPayload()));
|
|
590
|
-
}
|
|
591
|
-
notifyListeners();
|
|
592
|
-
}
|
|
593
|
-
|
|
594
530
|
Future<Map<String, Object?>> _startStreaming(
|
|
595
531
|
Map<String, Object?> payload,
|
|
596
532
|
) async {
|
|
@@ -9,14 +9,11 @@ const String desktopCompanionDeviceIdPrefsKey = 'desktop.companion.deviceId';
|
|
|
9
9
|
const String desktopCompanionActivationIdPrefsKey =
|
|
10
10
|
'desktop.companion.activationId';
|
|
11
11
|
const String desktopCompanionPausedPrefsKey = 'desktop.companion.paused';
|
|
12
|
-
const String desktopPassiveHistoryEnabledPrefsKey =
|
|
13
|
-
'desktop.passiveHistory.enabled';
|
|
14
12
|
|
|
15
13
|
class DesktopCompanionManager extends ChangeNotifier {
|
|
16
14
|
DesktopCompanionManager({required DesktopScreenCapture screenCapture});
|
|
17
15
|
|
|
18
16
|
bool get enabled => false;
|
|
19
|
-
bool get passiveHistoryEnabled => false;
|
|
20
17
|
bool get paused => false;
|
|
21
18
|
bool get connecting => false;
|
|
22
19
|
bool get connected => false;
|
|
@@ -24,8 +21,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
24
21
|
String get label => 'Desktop';
|
|
25
22
|
String get deviceId => '';
|
|
26
23
|
String get activationId => '';
|
|
27
|
-
String? get passiveHistoryLastUploadedAt => null;
|
|
28
|
-
String? get passiveHistoryLastError => null;
|
|
29
24
|
Map<String, Object?> get status => const <String, Object?>{};
|
|
30
25
|
|
|
31
26
|
Future<void> bootstrap(SharedPreferences prefs) async {}
|
|
@@ -48,13 +43,6 @@ class DesktopCompanionManager extends ChangeNotifier {
|
|
|
48
43
|
throw UnsupportedError('Desktop companion is not available here.');
|
|
49
44
|
}
|
|
50
45
|
|
|
51
|
-
Future<void> setPassiveHistoryEnabled(
|
|
52
|
-
bool value,
|
|
53
|
-
SharedPreferences prefs,
|
|
54
|
-
) async {
|
|
55
|
-
throw UnsupportedError('Desktop companion is not available here.');
|
|
56
|
-
}
|
|
57
|
-
|
|
58
46
|
Future<void> disconnect() async {}
|
|
59
47
|
|
|
60
48
|
Future<void> rotateIdentity(SharedPreferences prefs) async {
|
|
@@ -7,7 +7,6 @@
|
|
|
7
7
|
#include "generated_plugin_registrant.h"
|
|
8
8
|
|
|
9
9
|
#include <audioplayers_linux/audioplayers_linux_plugin.h>
|
|
10
|
-
#include <desktop_audio_capture/audio_capture_plugin.h>
|
|
11
10
|
#include <flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h>
|
|
12
11
|
#include <hotkey_manager_linux/hotkey_manager_linux_plugin.h>
|
|
13
12
|
#include <record_linux/record_linux_plugin.h>
|
|
@@ -20,9 +19,6 @@ void fl_register_plugins(FlPluginRegistry* registry) {
|
|
|
20
19
|
g_autoptr(FlPluginRegistrar) audioplayers_linux_registrar =
|
|
21
20
|
fl_plugin_registry_get_registrar_for_plugin(registry, "AudioplayersLinuxPlugin");
|
|
22
21
|
audioplayers_linux_plugin_register_with_registrar(audioplayers_linux_registrar);
|
|
23
|
-
g_autoptr(FlPluginRegistrar) desktop_audio_capture_registrar =
|
|
24
|
-
fl_plugin_registry_get_registrar_for_plugin(registry, "AudioCapturePlugin");
|
|
25
|
-
audio_capture_plugin_register_with_registrar(desktop_audio_capture_registrar);
|
|
26
22
|
g_autoptr(FlPluginRegistrar) flutter_secure_storage_linux_registrar =
|
|
27
23
|
fl_plugin_registry_get_registrar_for_plugin(registry, "FlutterSecureStorageLinuxPlugin");
|
|
28
24
|
flutter_secure_storage_linux_plugin_register_with_registrar(flutter_secure_storage_linux_registrar);
|
|
@@ -7,7 +7,6 @@ import Foundation
|
|
|
7
7
|
|
|
8
8
|
import audioplayers_darwin
|
|
9
9
|
import connectivity_plus
|
|
10
|
-
import desktop_audio_capture
|
|
11
10
|
import file_picker
|
|
12
11
|
import flutter_local_notifications
|
|
13
12
|
import flutter_secure_storage_macos
|
|
@@ -28,7 +27,6 @@ import window_manager
|
|
|
28
27
|
func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) {
|
|
29
28
|
AudioplayersDarwinPlugin.register(with: registry.registrar(forPlugin: "AudioplayersDarwinPlugin"))
|
|
30
29
|
ConnectivityPlusPlugin.register(with: registry.registrar(forPlugin: "ConnectivityPlusPlugin"))
|
|
31
|
-
AudioCapturePlugin.register(with: registry.registrar(forPlugin: "AudioCapturePlugin"))
|
|
32
30
|
FilePickerPlugin.register(with: registry.registrar(forPlugin: "FilePickerPlugin"))
|
|
33
31
|
FlutterLocalNotificationsPlugin.register(with: registry.registrar(forPlugin: "FlutterLocalNotificationsPlugin"))
|
|
34
32
|
FlutterSecureStoragePlugin.register(with: registry.registrar(forPlugin: "FlutterSecureStoragePlugin"))
|