neoagent 3.1.1-beta.8 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -2
- package/README.md +5 -1
- package/docs/architecture.md +1 -1
- package/docs/benchmarking.md +68 -87
- package/docs/capabilities.md +1 -1
- package/docs/clients-and-devices.md +3 -4
- package/docs/configuration.md +1 -1
- package/docs/devices.md +1 -2
- package/docs/health.md +24 -0
- package/docs/index.md +2 -2
- package/docs/integrations-architecture.md +6 -0
- package/docs/integrations.md +15 -1
- package/docs/models.md +1 -1
- package/docs/persistence.md +1 -1
- package/docs/why-neoagent.md +1 -1
- package/flutter_app/README.md +0 -5
- package/flutter_app/android/app/src/main/AndroidManifest.xml +0 -14
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +0 -98
- package/flutter_app/lib/main.dart +0 -6
- package/flutter_app/lib/main_app_shell.dart +0 -2
- package/flutter_app/lib/main_controller.dart +2 -867
- package/flutter_app/lib/main_devices.dart +0 -73
- package/flutter_app/lib/main_integrations.dart +339 -187
- package/flutter_app/lib/main_launcher.dart +2 -212
- package/flutter_app/lib/main_models.dart +0 -259
- package/flutter_app/lib/main_navigation.dart +2 -14
- package/flutter_app/lib/main_operations.dart +322 -364
- package/flutter_app/lib/main_runtime.dart +9 -208
- package/flutter_app/lib/main_settings.dart +17 -199
- package/flutter_app/lib/main_shared.dart +20 -591
- package/flutter_app/lib/main_timeline.dart +149 -438
- package/flutter_app/lib/main_voice_assistant.dart +0 -5
- package/flutter_app/lib/src/backend_client.dart +0 -102
- package/flutter_app/lib/src/desktop_companion_io.dart +1 -65
- package/flutter_app/lib/src/desktop_companion_stub.dart +0 -12
- package/flutter_app/linux/flutter/generated_plugin_registrant.cc +0 -4
- package/flutter_app/linux/flutter/generated_plugins.cmake +0 -1
- package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +0 -2
- package/flutter_app/macos/Podfile.lock +56 -5
- package/flutter_app/pubspec.lock +0 -7
- package/flutter_app/pubspec.yaml +0 -2
- package/flutter_app/web/index.html +174 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.cc +0 -3
- package/flutter_app/windows/flutter/generated_plugins.cmake +0 -1
- package/landing/index.html +0 -2
- package/lib/manager.js +1 -1
- package/lib/schema_migrations.js +75 -0
- package/package.json +1 -4
- package/server/db/database.js +1 -133
- package/server/http/middleware.js +2 -16
- package/server/http/routes.js +0 -2
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/NOTICES +0 -25
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/index.html +174 -0
- package/server/public/main.dart.js +88124 -90164
- package/server/routes/auth.js +11 -2
- package/server/routes/settings.js +0 -5
- package/server/routes/voice_assistant.js +1 -128
- package/server/services/account/erasure.js +0 -3
- package/server/services/ai/deliverables/artifact_helpers.js +1 -1
- package/server/services/ai/loop/conversation_loop.js +6 -2
- package/server/services/ai/loop/tool_dispatch.js +0 -3
- package/server/services/ai/messagingFallback.js +1 -1
- package/server/services/ai/settings.js +0 -6
- package/server/services/ai/toolEvidence.js +1 -4
- package/server/services/ai/tools.js +0 -234
- package/server/services/desktop/auth.js +0 -3
- package/server/services/desktop/registry.js +2 -50
- package/server/services/integrations/neoarchive/provider.js +616 -0
- package/server/services/integrations/neorecall/client.js +140 -0
- package/server/services/integrations/neorecall/constants.js +87 -0
- package/server/services/integrations/neorecall/provider.js +163 -0
- package/server/services/integrations/registry.js +4 -2
- package/server/services/manager.js +0 -22
- package/server/services/memory/manager.js +44 -19
- package/server/services/memory/retrieval_reasoning.js +36 -14
- package/server/services/messaging/whatsapp.js +18 -0
- package/server/services/security/tool_categories.js +0 -5
- package/server/services/social_video/service.js +2 -20
- package/server/services/tasks/adapters/index.js +1 -1
- package/server/services/tasks/integration_runtime.js +3 -67
- package/server/services/tasks/runtime.js +1 -6
- package/server/services/tasks/task_repository.js +6 -13
- package/server/services/timeline/service.js +0 -224
- package/server/services/voice/message.js +0 -24
- package/server/services/voice/providers.js +42 -1
- package/server/services/voice/runtimeManager.js +9 -5
- package/server/services/voice/turnRunner.js +0 -22
- package/server/services/wearable/service.js +0 -5
- package/docs/recordings-and-health.md +0 -41
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +0 -21
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +0 -586
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +0 -78
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +0 -104
- package/flutter_app/lib/main_recordings.dart +0 -920
- package/flutter_app/lib/src/desktop_ocr_bridge.dart +0 -2
- package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +0 -125
- package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +0 -30
- package/flutter_app/lib/src/desktop_passive_history.dart +0 -332
- package/flutter_app/lib/src/recording_bridge.dart +0 -232
- package/flutter_app/lib/src/recording_bridge_io.dart +0 -1027
- package/flutter_app/lib/src/recording_bridge_stub.dart +0 -120
- package/flutter_app/lib/src/recording_bridge_web.dart +0 -702
- package/flutter_app/lib/src/recording_chunk_queue.dart +0 -149
- package/flutter_app/lib/src/recording_chunk_queue_io.dart +0 -182
- package/flutter_app/lib/src/recording_payloads.dart +0 -95
- package/flutter_app/third_party/desktop_audio_capture/LICENSE +0 -21
- package/flutter_app/third_party/desktop_audio_capture/README.md +0 -262
- package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +0 -65
- package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +0 -153
- package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +0 -110
- package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +0 -461
- package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +0 -91
- package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +0 -106
- package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +0 -219
- package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +0 -336
- package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +0 -101
- package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +0 -692
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +0 -35
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +0 -36
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +0 -32
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +0 -32
- package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +0 -878
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +0 -27
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +0 -1172
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +0 -655
- package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +0 -12
- package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +0 -30
- package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +0 -87
- package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +0 -105
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +0 -80
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +0 -31
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +0 -12
- package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +0 -23
- package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +0 -25
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +0 -1117
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +0 -115
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +0 -777
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +0 -87
- package/server/routes/recordings.js +0 -335
- package/server/routes/screenHistory.js +0 -190
- package/server/services/ai/recordingInsights.js +0 -105
- package/server/services/integrations/neomail/provider.js +0 -993
- package/server/services/recordings/deepgram.js +0 -58
- package/server/services/recordings/manager.js +0 -1273
- package/server/services/tasks/adapters/neomail_email_received.js +0 -46
- package/server/services/voice/screenshotContext.js +0 -73
- package/server/services/voice/turnCoordinator.js +0 -51
package/server/routes/auth.js
CHANGED
|
@@ -62,6 +62,8 @@ const passwordResetLimiter = rateLimit({
|
|
|
62
62
|
legacyHeaders: false,
|
|
63
63
|
});
|
|
64
64
|
|
|
65
|
+
const WEARABLE_SESSION_MAX_AGE_MS = 90 * 24 * 60 * 60 * 1000;
|
|
66
|
+
|
|
65
67
|
function getAuthProviderManager(req) {
|
|
66
68
|
return req.app?.locals?.authProviderManager;
|
|
67
69
|
}
|
|
@@ -81,7 +83,7 @@ function toUserPayload(user) {
|
|
|
81
83
|
};
|
|
82
84
|
}
|
|
83
85
|
|
|
84
|
-
function establishSession(req, res, user) {
|
|
86
|
+
function establishSession(req, res, user, options = {}) {
|
|
85
87
|
req.session.regenerate((regenerateError) => {
|
|
86
88
|
if (regenerateError) {
|
|
87
89
|
console.error('Auth session regenerate error:', regenerateError);
|
|
@@ -90,6 +92,9 @@ function establishSession(req, res, user) {
|
|
|
90
92
|
|
|
91
93
|
req.session.userId = user.id;
|
|
92
94
|
req.session.username = user.username;
|
|
95
|
+
if (Number.isFinite(options.maxAgeMs) && options.maxAgeMs > 0) {
|
|
96
|
+
req.session.cookie.maxAge = options.maxAgeMs;
|
|
97
|
+
}
|
|
93
98
|
req.session.save((saveError) => {
|
|
94
99
|
if (saveError) {
|
|
95
100
|
console.error('Auth session save error:', saveError);
|
|
@@ -648,7 +653,11 @@ router.post('/api/auth/qr-login/challenge/:id/claim', qrLoginClaimLimiter, (req,
|
|
|
648
653
|
pollToken,
|
|
649
654
|
});
|
|
650
655
|
updateLastLogin(result.user.id);
|
|
651
|
-
|
|
656
|
+
const requestedPlatform = String(result.challenge?.requestedDevice?.metadata?.platform || '');
|
|
657
|
+
const sessionOptions = requestedPlatform === 'esp32-s3-amoled'
|
|
658
|
+
? { maxAgeMs: WEARABLE_SESSION_MAX_AGE_MS }
|
|
659
|
+
: {};
|
|
660
|
+
return establishSession(req, res, result.user, sessionOptions);
|
|
652
661
|
} catch (error) {
|
|
653
662
|
const statusCode = Number(error?.statusCode || 500);
|
|
654
663
|
return res.status(statusCode).json({
|
|
@@ -44,11 +44,6 @@ const AGENT_SETTING_KEYS = new Set([
|
|
|
44
44
|
'subagent_max_children_per_run',
|
|
45
45
|
'assistant_behavior_notes',
|
|
46
46
|
'auto_skill_learning',
|
|
47
|
-
'auto_recording_insights',
|
|
48
|
-
'default_recording_transcription_provider',
|
|
49
|
-
'default_recording_transcription_model',
|
|
50
|
-
'default_recording_summary_provider',
|
|
51
|
-
'default_recording_summary_model',
|
|
52
47
|
'fallback_model_id',
|
|
53
48
|
'smarter_model_selector',
|
|
54
49
|
'ai_provider_configs',
|
|
@@ -2,140 +2,13 @@ const express = require('express');
|
|
|
2
2
|
|
|
3
3
|
const { requireAuth } = require('../middleware/auth');
|
|
4
4
|
const { sanitizeError } = require('../utils/security');
|
|
5
|
-
const {
|
|
6
|
-
const { TurnCoordinator } = require('../services/voice/turnCoordinator');
|
|
7
|
-
const { normalizeVoiceSynthesisOptions, transcribeVoiceInput } = require('../services/voice/providers');
|
|
5
|
+
const { transcribeVoiceInput } = require('../services/voice/providers');
|
|
8
6
|
const { writeTempAudioFile, removeTempFile } = require('../services/voice/liveAudio');
|
|
9
|
-
const { runVoiceTranscriptTurn } = require('../services/voice/turnRunner');
|
|
10
7
|
|
|
11
8
|
const router = express.Router();
|
|
12
|
-
const turnCoordinator = new TurnCoordinator();
|
|
13
9
|
|
|
14
10
|
router.use(requireAuth);
|
|
15
11
|
|
|
16
|
-
async function resolveCompletedVoiceSession(recordingManager, userId, sessionId) {
|
|
17
|
-
let session = recordingManager.getSession(userId, sessionId);
|
|
18
|
-
if (session.status === 'recording') {
|
|
19
|
-
session = recordingManager.finalizeSession(userId, sessionId, {
|
|
20
|
-
stopReason: 'voice_assistant',
|
|
21
|
-
autoProcess: false,
|
|
22
|
-
includeInsights: false,
|
|
23
|
-
});
|
|
24
|
-
}
|
|
25
|
-
if (session.status === 'processing') {
|
|
26
|
-
await recordingManager.processSession(userId, sessionId, {
|
|
27
|
-
includeInsights: false,
|
|
28
|
-
});
|
|
29
|
-
session = recordingManager.getSession(userId, sessionId);
|
|
30
|
-
}
|
|
31
|
-
if (session.status !== 'completed') {
|
|
32
|
-
throw new Error(`Recording session is not ready for assistant response (status: ${session.status}).`);
|
|
33
|
-
}
|
|
34
|
-
return session;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
router.post('/respond', async (req, res) => {
|
|
38
|
-
try {
|
|
39
|
-
const userId = req.session.userId;
|
|
40
|
-
const recordingManager = req.app?.locals?.recordingManager;
|
|
41
|
-
const agentEngine = req.app?.locals?.agentEngine;
|
|
42
|
-
const memoryManager = req.app?.locals?.memoryManager;
|
|
43
|
-
if (!recordingManager || !agentEngine || !memoryManager) {
|
|
44
|
-
return res.status(500).json({ error: 'Voice assistant service is not initialized.' });
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
const sessionId = String(req.body?.sessionId || '').trim();
|
|
48
|
-
if (!sessionId) {
|
|
49
|
-
return res.status(400).json({ error: 'sessionId is required.' });
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
const voiceOptions = normalizeVoiceSynthesisOptions({
|
|
53
|
-
provider: req.body?.ttsProvider,
|
|
54
|
-
model: req.body?.ttsModel,
|
|
55
|
-
voice: req.body?.ttsVoice,
|
|
56
|
-
});
|
|
57
|
-
const promptHint = String(req.body?.promptHint || '').trim();
|
|
58
|
-
const agentId = resolveAgentId(userId, getAgentIdFromRequest(req));
|
|
59
|
-
|
|
60
|
-
let screenshotData = {};
|
|
61
|
-
if (req.body?.screenshotBase64 || req.body?.screenshotMimeType) {
|
|
62
|
-
if (!req.body?.screenshotBase64 || !req.body?.screenshotMimeType) {
|
|
63
|
-
return res.status(400).json({
|
|
64
|
-
error: 'Both screenshotBase64 and screenshotMimeType must be provided together.',
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
const mimeType = String(req.body.screenshotMimeType || '').trim().toLowerCase();
|
|
69
|
-
const allowedMimeTypes = new Set(['image/png', 'image/jpeg', 'image/jpg', 'image/webp']);
|
|
70
|
-
if (!allowedMimeTypes.has(mimeType)) {
|
|
71
|
-
return res.status(400).json({
|
|
72
|
-
error: `Invalid screenshot MIME type. Allowed types: ${Array.from(allowedMimeTypes).join(', ')}`,
|
|
73
|
-
});
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
const screenshotBase64 = String(req.body.screenshotBase64 || '').trim();
|
|
77
|
-
const base64Regex = /^[A-Za-z0-9+/]+={0,2}$/;
|
|
78
|
-
if (!base64Regex.test(screenshotBase64)) {
|
|
79
|
-
return res.status(400).json({ error: 'Invalid base64 format for screenshot.' });
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
const maxSizeBytes = 5 * 1024 * 1024;
|
|
83
|
-
const approximateSize = (screenshotBase64.length * 3) / 4;
|
|
84
|
-
if (approximateSize > maxSizeBytes) {
|
|
85
|
-
return res.status(400).json({
|
|
86
|
-
error: 'Screenshot size exceeds maximum allowed size of 5MB.',
|
|
87
|
-
});
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
screenshotData = {
|
|
91
|
-
screenshotBase64,
|
|
92
|
-
screenshotMimeType: mimeType,
|
|
93
|
-
};
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
const responsePayload = await turnCoordinator.run(sessionId, async () => {
|
|
97
|
-
const session = await resolveCompletedVoiceSession(
|
|
98
|
-
recordingManager,
|
|
99
|
-
userId,
|
|
100
|
-
sessionId,
|
|
101
|
-
);
|
|
102
|
-
|
|
103
|
-
const transcript = String(session.transcriptText || '').trim();
|
|
104
|
-
if (!transcript) {
|
|
105
|
-
throw new Error('Recording transcript is empty. Please retry transcription or record again.');
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
const turnResult = await runVoiceTranscriptTurn({
|
|
109
|
-
userId,
|
|
110
|
-
agentId,
|
|
111
|
-
transcript,
|
|
112
|
-
promptHint,
|
|
113
|
-
platform: 'voice_assistant',
|
|
114
|
-
metadata: {
|
|
115
|
-
recordingSessionId: sessionId,
|
|
116
|
-
...screenshotData,
|
|
117
|
-
},
|
|
118
|
-
agentEngine,
|
|
119
|
-
memoryManager,
|
|
120
|
-
ttsProvider: voiceOptions.provider,
|
|
121
|
-
ttsModel: voiceOptions.model,
|
|
122
|
-
ttsVoice: voiceOptions.voice,
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
return {
|
|
126
|
-
session,
|
|
127
|
-
...turnResult,
|
|
128
|
-
};
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
return res.json(responsePayload);
|
|
132
|
-
} catch (err) {
|
|
133
|
-
const message = sanitizeError(err);
|
|
134
|
-
const statusCode = /not ready|empty/i.test(message) ? 400 : 500;
|
|
135
|
-
return res.status(statusCode).json({ error: message });
|
|
136
|
-
}
|
|
137
|
-
});
|
|
138
|
-
|
|
139
12
|
router.post('/transcribe', async (req, res) => {
|
|
140
13
|
try {
|
|
141
14
|
const audioBase64 = String(req.body?.audioBase64 || '').trim();
|
|
@@ -60,9 +60,6 @@ function deleteChildRows(uid) {
|
|
|
60
60
|
'DELETE FROM conversation_messages WHERE conversation_id IN (SELECT id FROM conversations WHERE user_id = ?)',
|
|
61
61
|
'DELETE FROM agent_steps WHERE run_id IN (SELECT id FROM agent_runs WHERE user_id = ?)',
|
|
62
62
|
'DELETE FROM ai_widget_snapshots WHERE widget_id IN (SELECT id FROM ai_widgets WHERE user_id = ?)',
|
|
63
|
-
'DELETE FROM recording_transcript_segments WHERE session_id IN (SELECT id FROM recording_sessions WHERE user_id = ?)',
|
|
64
|
-
'DELETE FROM recording_chunks WHERE source_id IN (SELECT s.id FROM recording_sources s JOIN recording_sessions rs ON s.session_id = rs.id WHERE rs.user_id = ?)',
|
|
65
|
-
'DELETE FROM recording_sources WHERE session_id IN (SELECT id FROM recording_sessions WHERE user_id = ?)',
|
|
66
63
|
'DELETE FROM memory_source_links WHERE memory_id IN (SELECT id FROM memories WHERE user_id = ?)',
|
|
67
64
|
];
|
|
68
65
|
for (const sql of childDeletes) {
|
|
@@ -124,7 +124,7 @@ const ARTIFACT_CONTAINER_KEYS = new Set([
|
|
|
124
124
|
|
|
125
125
|
const CONTAINER_URL_KEYS = new Set(['url', 'urls', 'uri', 'uris', 'href', 'hrefs']);
|
|
126
126
|
|
|
127
|
-
const EVIDENCE_RESULT_TOOLS = /^(execute_command|github_|list_|search_|read_|get_|find_|http_request|web_search|browser_get|browser_read|code_navigate|query_structured_data|memory_|session_search|
|
|
127
|
+
const EVIDENCE_RESULT_TOOLS = /^(execute_command|github_|list_|search_|read_|get_|find_|http_request|web_search|browser_get|browser_read|code_navigate|query_structured_data|memory_|session_search|read_health_data)/;
|
|
128
128
|
|
|
129
129
|
function allowsGenericCandidateKeys(toolName = '') {
|
|
130
130
|
return !EVIDENCE_RESULT_TOOLS.test(String(toolName || ''));
|
|
@@ -440,7 +440,10 @@ async function getProviderForUser(userId, task = '', isSubagent = false, modelOv
|
|
|
440
440
|
|
|
441
441
|
if (modelOverride && typeof modelOverride === 'string') {
|
|
442
442
|
const requested = models.find((m) => m.id === modelOverride.trim());
|
|
443
|
-
|
|
443
|
+
// An explicit override (e.g. a failure fallback picked by getFailureFallbackModelId)
|
|
444
|
+
// may intentionally sit outside the user's enabled_models pool — that's the whole
|
|
445
|
+
// point when no in-pool cross-provider option exists. Only require availability.
|
|
446
|
+
if (requested && requested.available !== false) {
|
|
444
447
|
selectedModelDef = requested;
|
|
445
448
|
return {
|
|
446
449
|
provider: createProviderInstance(selectedModelDef.provider, userId, providerConfig),
|
|
@@ -569,6 +572,7 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
|
|
|
569
572
|
let failedStepCount = 0;
|
|
570
573
|
let toolExecutions = [];
|
|
571
574
|
let deliverableWorkflow = null;
|
|
575
|
+
let runTitle;
|
|
572
576
|
const timelineService = app?.locals?.timelineService || null;
|
|
573
577
|
|
|
574
578
|
const { releaseReservation } = enforceRateLimits(userId, {
|
|
@@ -638,7 +642,7 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
|
|
|
638
642
|
}
|
|
639
643
|
};
|
|
640
644
|
|
|
641
|
-
|
|
645
|
+
runTitle = generateTitle(userMessage);
|
|
642
646
|
const initialRunMetadata = buildInitialRunMetadata(options);
|
|
643
647
|
db.prepare(`INSERT INTO agent_runs(
|
|
644
648
|
id, user_id, agent_id, title, status, trigger_type, trigger_source, model, metadata_json
|
|
@@ -88,7 +88,7 @@ function toolWorkDescription(toolName) {
|
|
|
88
88
|
if (name === 'web_search' || name === 'http_request') return 'looked up supporting information';
|
|
89
89
|
if (name.startsWith('browser_')) return 'checked the browser state';
|
|
90
90
|
if (name.startsWith('android_')) return 'checked the Android state';
|
|
91
|
-
if (name === 'read_health_data'
|
|
91
|
+
if (name === 'read_health_data') return 'checked stored data';
|
|
92
92
|
return '';
|
|
93
93
|
}
|
|
94
94
|
|
|
@@ -152,11 +152,6 @@ function createDefaultAiSettings() {
|
|
|
152
152
|
subagent_max_children_per_run: 10,
|
|
153
153
|
assistant_behavior_notes: '',
|
|
154
154
|
auto_skill_learning: false,
|
|
155
|
-
auto_recording_insights: true,
|
|
156
|
-
default_recording_transcription_provider: 'deepgram',
|
|
157
|
-
default_recording_transcription_model: 'nova-3',
|
|
158
|
-
default_recording_summary_provider: 'auto',
|
|
159
|
-
default_recording_summary_model: 'auto',
|
|
160
155
|
fallback_model_id: 'gpt-5-nano',
|
|
161
156
|
smarter_model_selector: true,
|
|
162
157
|
enabled_models: [],
|
|
@@ -343,7 +338,6 @@ function getAiSettings(userId, agentId = null) {
|
|
|
343
338
|
? settings.assistant_behavior_notes
|
|
344
339
|
: DEFAULT_AI_SETTINGS.assistant_behavior_notes;
|
|
345
340
|
settings.auto_skill_learning = settings.auto_skill_learning !== false && settings.auto_skill_learning !== 'false';
|
|
346
|
-
settings.auto_recording_insights = settings.auto_recording_insights !== false && settings.auto_recording_insights !== 'false';
|
|
347
341
|
settings.smarter_model_selector = settings.smarter_model_selector !== false && settings.smarter_model_selector !== 'false';
|
|
348
342
|
settings.fallback_model_id = typeof settings.fallback_model_id === 'string' ? settings.fallback_model_id : DEFAULT_AI_SETTINGS.fallback_model_id;
|
|
349
343
|
settings.enabled_models = Array.isArray(settings.enabled_models) ? settings.enabled_models : DEFAULT_AI_SETTINGS.enabled_models;
|
|
@@ -29,7 +29,7 @@ const EVIDENCE_SOURCE_RULES = [
|
|
|
29
29
|
{ source: 'skills', match: (name) => name.includes('skill') },
|
|
30
30
|
{ source: 'tasks', match: (name) => name === 'create_task' || name === 'update_task' || name === 'delete_task' || name === 'list_tasks' || name.includes('widget') },
|
|
31
31
|
{ source: 'messaging', match: (name) => name === 'send_message' || name === 'make_call' },
|
|
32
|
-
{ source: 'data', match: (name) => name
|
|
32
|
+
{ source: 'data', match: (name) => name === 'read_health_data' },
|
|
33
33
|
{ source: 'vision', match: (name) => name === 'analyze_image' },
|
|
34
34
|
{ source: 'subagent', match: (name) => name.includes('subagent') },
|
|
35
35
|
];
|
|
@@ -55,9 +55,6 @@ function classifyToolExecution(toolName, toolArgs = {}, result, errorMessage = '
|
|
|
55
55
|
'memory_recall',
|
|
56
56
|
'analyze_image',
|
|
57
57
|
'read_health_data',
|
|
58
|
-
'recordings_list',
|
|
59
|
-
'recordings_get',
|
|
60
|
-
'recordings_search',
|
|
61
58
|
'list_tasks',
|
|
62
59
|
'wait_subagent',
|
|
63
60
|
]);
|
|
@@ -25,46 +25,6 @@ function compactText(text, maxChars = 120) {
|
|
|
25
25
|
return `${trimmed.trim()}...`;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
function compactTranscript(text, maxChars = 1200) {
|
|
29
|
-
const str = String(text || '').replace(/\s+/g, ' ').trim();
|
|
30
|
-
if (!str) return '';
|
|
31
|
-
if (str.length <= maxChars) return str;
|
|
32
|
-
return `${str.slice(0, maxChars).trim()}...`;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function compactRecordingSession(session, options = {}) {
|
|
36
|
-
const includeTranscript = options.includeTranscript === true;
|
|
37
|
-
return {
|
|
38
|
-
id: session.id,
|
|
39
|
-
title: session.title,
|
|
40
|
-
platform: session.platform,
|
|
41
|
-
status: session.status,
|
|
42
|
-
startedAt: session.startedAt,
|
|
43
|
-
endedAt: session.endedAt,
|
|
44
|
-
durationMs: Number(session.durationMs) || 0,
|
|
45
|
-
sourceCount: Number(session.sourceCount) || 0,
|
|
46
|
-
transcriptLanguage: session.transcriptLanguage || null,
|
|
47
|
-
transcriptModel: session.transcriptModel || null,
|
|
48
|
-
hasTranscript: !!String(session.transcriptText || '').trim(),
|
|
49
|
-
transcriptPreview: includeTranscript ? compactTranscript(session.transcriptText || '', 1200) : undefined,
|
|
50
|
-
structuredContent: session.structuredContent || null,
|
|
51
|
-
lastError: session.lastError || null,
|
|
52
|
-
};
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
function mapRecordingSource(source) {
|
|
56
|
-
return {
|
|
57
|
-
id: source.id,
|
|
58
|
-
sourceKey: source.sourceKey,
|
|
59
|
-
sourceKind: source.sourceKind,
|
|
60
|
-
mediaKind: source.mediaKind,
|
|
61
|
-
mimeType: source.mimeType,
|
|
62
|
-
status: source.status,
|
|
63
|
-
chunkCount: source.chunkCount,
|
|
64
|
-
durationMs: source.durationMs,
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
|
|
68
28
|
function compactToolDefinition(tool, options = {}) {
|
|
69
29
|
const compact = {
|
|
70
30
|
name: tool.name,
|
|
@@ -1464,46 +1424,6 @@ function getAvailableTools(app, options = {}) {
|
|
|
1464
1424
|
}
|
|
1465
1425
|
}
|
|
1466
1426
|
},
|
|
1467
|
-
{
|
|
1468
|
-
name: 'recordings_list',
|
|
1469
|
-
description: 'List the user\'s recording sessions with status, timing, and transcript availability.',
|
|
1470
|
-
parameters: {
|
|
1471
|
-
type: 'object',
|
|
1472
|
-
properties: {
|
|
1473
|
-
limit: { type: 'number', description: 'Maximum number of sessions to return (default 12, max 50).' },
|
|
1474
|
-
status: { type: 'string', description: 'Optional status filter: recording, processing, completed, failed, cancelled.' },
|
|
1475
|
-
platform: { type: 'string', description: 'Optional platform filter: wearable, web, android, unknown.' },
|
|
1476
|
-
include_transcript_previews: { type: 'boolean', description: 'Include short transcript previews (default false).' }
|
|
1477
|
-
}
|
|
1478
|
-
}
|
|
1479
|
-
},
|
|
1480
|
-
{
|
|
1481
|
-
name: 'recordings_get',
|
|
1482
|
-
description: 'Get one recording session in detail, including transcript text, sources, and optional transcript segments.',
|
|
1483
|
-
parameters: {
|
|
1484
|
-
type: 'object',
|
|
1485
|
-
properties: {
|
|
1486
|
-
session_id: { type: 'string', description: 'Recording session ID.' },
|
|
1487
|
-
include_segments: { type: 'boolean', description: 'Include transcript segments (default true).' },
|
|
1488
|
-
segment_limit: { type: 'number', description: 'Maximum number of transcript segments to return (default 80, max 300).' },
|
|
1489
|
-
include_full_transcript: { type: 'boolean', description: 'Include full transcript text (default true). If false, returns only preview.' }
|
|
1490
|
-
},
|
|
1491
|
-
required: ['session_id']
|
|
1492
|
-
}
|
|
1493
|
-
},
|
|
1494
|
-
{
|
|
1495
|
-
name: 'recordings_search',
|
|
1496
|
-
description: 'Search recording transcripts by keyword and return matching snippets with session references.',
|
|
1497
|
-
parameters: {
|
|
1498
|
-
type: 'object',
|
|
1499
|
-
properties: {
|
|
1500
|
-
query: { type: 'string', description: 'Search query for transcript text.' },
|
|
1501
|
-
limit: { type: 'number', description: 'Maximum number of matches to return (default 20, max 100).' },
|
|
1502
|
-
status: { type: 'string', description: 'Optional status filter for sessions.' }
|
|
1503
|
-
},
|
|
1504
|
-
required: ['query']
|
|
1505
|
-
}
|
|
1506
|
-
},
|
|
1507
1427
|
{
|
|
1508
1428
|
name: 'list_chats',
|
|
1509
1429
|
description: 'List all known groups and conversations across connected messaging platforms. Use this when the user doesn\'t know the exact group name or chat ID — it returns every chat that has ever sent a message, with platform, chat ID, display name, and whether it\'s a group or DM.',
|
|
@@ -1745,7 +1665,6 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
1745
1665
|
const integrations = () => app?.locals?.integrationManager || null;
|
|
1746
1666
|
const sk = () => app?.locals?.skillRunner || engine.skillRunner;
|
|
1747
1667
|
const taskRuntime = () => app?.locals?.taskRuntime || engine.taskRuntime;
|
|
1748
|
-
const rec = () => app?.locals?.recordingManager || null;
|
|
1749
1668
|
const socialVideo = () => app?.locals?.socialVideoService || null;
|
|
1750
1669
|
const socialReach = () => app?.locals?.socialReachService || null;
|
|
1751
1670
|
const widgets = () => app?.locals?.widgetService || null;
|
|
@@ -1848,7 +1767,6 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
1848
1767
|
case 'browser_extract': {
|
|
1849
1768
|
const { provider, backend } = await bc();
|
|
1850
1769
|
if (!provider) return { error: 'Browser controller not available' };
|
|
1851
|
-
if (!args.selector) return { error: 'browser_extract requires a "selector" argument' };
|
|
1852
1770
|
return { ...await provider.extract(args.selector, args.attribute, args.all), backend };
|
|
1853
1771
|
}
|
|
1854
1772
|
|
|
@@ -2182,158 +2100,6 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2182
2100
|
return result;
|
|
2183
2101
|
}
|
|
2184
2102
|
|
|
2185
|
-
case 'recordings_list': {
|
|
2186
|
-
const manager = rec();
|
|
2187
|
-
if (!manager) return { error: 'Recording manager not available' };
|
|
2188
|
-
|
|
2189
|
-
const limit = Math.max(1, Math.min(Number(args.limit) || 12, 50));
|
|
2190
|
-
const includeTranscript = args.include_transcript_previews === true;
|
|
2191
|
-
const statusFilter = typeof args.status === 'string' ? args.status.trim().toLowerCase() : null;
|
|
2192
|
-
const platformFilter = typeof args.platform === 'string' ? args.platform.trim().toLowerCase() : null;
|
|
2193
|
-
|
|
2194
|
-
let sessions = manager.listSessions(userId, { limit: Math.max(limit * 3, limit) });
|
|
2195
|
-
|
|
2196
|
-
if (statusFilter) {
|
|
2197
|
-
sessions = sessions.filter((session) => String(session.status || '').toLowerCase() === statusFilter);
|
|
2198
|
-
}
|
|
2199
|
-
if (platformFilter) {
|
|
2200
|
-
sessions = sessions.filter((session) => String(session.platform || '').toLowerCase() === platformFilter);
|
|
2201
|
-
}
|
|
2202
|
-
|
|
2203
|
-
const filtered = sessions.slice(0, limit).map((session) => compactRecordingSession(session, {
|
|
2204
|
-
includeTranscript,
|
|
2205
|
-
}));
|
|
2206
|
-
|
|
2207
|
-
return {
|
|
2208
|
-
count: filtered.length,
|
|
2209
|
-
filters: {
|
|
2210
|
-
status: statusFilter || null,
|
|
2211
|
-
platform: platformFilter || null,
|
|
2212
|
-
},
|
|
2213
|
-
sessions: filtered,
|
|
2214
|
-
};
|
|
2215
|
-
}
|
|
2216
|
-
|
|
2217
|
-
case 'recordings_get': {
|
|
2218
|
-
const manager = rec();
|
|
2219
|
-
if (!manager) return { error: 'Recording manager not available' };
|
|
2220
|
-
|
|
2221
|
-
const sessionId = `${args.session_id || ''}`.trim();
|
|
2222
|
-
if (!sessionId) return { error: 'session_id is required' };
|
|
2223
|
-
|
|
2224
|
-
try {
|
|
2225
|
-
const session = manager.getSession(userId, sessionId);
|
|
2226
|
-
const includeSegments = args.include_segments !== false;
|
|
2227
|
-
const includeFullTranscript = args.include_full_transcript !== false;
|
|
2228
|
-
const segmentLimit = Math.max(1, Math.min(Number(args.segment_limit) || 80, 300));
|
|
2229
|
-
|
|
2230
|
-
const result = {
|
|
2231
|
-
session: compactRecordingSession(session, {
|
|
2232
|
-
includeTranscript: !includeFullTranscript,
|
|
2233
|
-
}),
|
|
2234
|
-
transcriptText: includeFullTranscript
|
|
2235
|
-
? String(session.transcriptText || '')
|
|
2236
|
-
: compactTranscript(session.transcriptText || '', 1600),
|
|
2237
|
-
sources: (Array.isArray(session.sources) ? session.sources : []).map(mapRecordingSource),
|
|
2238
|
-
segmentCount: Array.isArray(session.transcriptSegments) ? session.transcriptSegments.length : 0,
|
|
2239
|
-
};
|
|
2240
|
-
|
|
2241
|
-
if (includeSegments) {
|
|
2242
|
-
const segments = Array.isArray(session.transcriptSegments) ? session.transcriptSegments : [];
|
|
2243
|
-
result.segments = segments.slice(0, segmentLimit).map((segment) => ({
|
|
2244
|
-
id: segment.id,
|
|
2245
|
-
speaker: segment.speaker,
|
|
2246
|
-
sourceKey: segment.sourceKey,
|
|
2247
|
-
startMs: segment.startMs,
|
|
2248
|
-
endMs: segment.endMs,
|
|
2249
|
-
confidence: segment.confidence,
|
|
2250
|
-
text: segment.text,
|
|
2251
|
-
}));
|
|
2252
|
-
result.segmentsTruncated = segments.length > segmentLimit;
|
|
2253
|
-
}
|
|
2254
|
-
|
|
2255
|
-
return result;
|
|
2256
|
-
} catch (err) {
|
|
2257
|
-
return { error: err.message };
|
|
2258
|
-
}
|
|
2259
|
-
}
|
|
2260
|
-
|
|
2261
|
-
case 'recordings_search': {
|
|
2262
|
-
const query = `${args.query || ''}`.trim();
|
|
2263
|
-
if (!query) return { error: 'query is required' };
|
|
2264
|
-
|
|
2265
|
-
const limit = Math.max(1, Math.min(Number(args.limit) || 20, 100));
|
|
2266
|
-
const statusFilter = typeof args.status === 'string' ? args.status.trim().toLowerCase() : null;
|
|
2267
|
-
const like = `%${query.replace(/[%_]/g, (m) => `\\${m}`)}%`;
|
|
2268
|
-
|
|
2269
|
-
const statusClause = statusFilter ? 'AND s.status = ?' : '';
|
|
2270
|
-
const params = statusFilter
|
|
2271
|
-
? [userId, like, like, statusFilter, limit]
|
|
2272
|
-
: [userId, like, like, limit];
|
|
2273
|
-
|
|
2274
|
-
const rows = db.prepare(`
|
|
2275
|
-
SELECT
|
|
2276
|
-
s.id AS session_id,
|
|
2277
|
-
s.title,
|
|
2278
|
-
s.platform,
|
|
2279
|
-
s.status,
|
|
2280
|
-
s.started_at,
|
|
2281
|
-
s.ended_at,
|
|
2282
|
-
s.duration_ms,
|
|
2283
|
-
seg.id AS segment_id,
|
|
2284
|
-
seg.start_ms,
|
|
2285
|
-
seg.end_ms,
|
|
2286
|
-
seg.speaker,
|
|
2287
|
-
seg.text AS segment_text,
|
|
2288
|
-
s.transcript_text
|
|
2289
|
-
FROM recording_sessions s
|
|
2290
|
-
LEFT JOIN recording_transcript_segments seg
|
|
2291
|
-
ON seg.session_id = s.id
|
|
2292
|
-
WHERE s.user_id = ?
|
|
2293
|
-
AND (
|
|
2294
|
-
LOWER(COALESCE(seg.text, '')) LIKE LOWER(?) ESCAPE '\\'
|
|
2295
|
-
OR LOWER(COALESCE(s.transcript_text, '')) LIKE LOWER(?) ESCAPE '\\'
|
|
2296
|
-
)
|
|
2297
|
-
${statusClause}
|
|
2298
|
-
ORDER BY datetime(s.created_at) DESC, seg.start_ms ASC, seg.id ASC
|
|
2299
|
-
LIMIT ?
|
|
2300
|
-
`).all(...params);
|
|
2301
|
-
|
|
2302
|
-
const matches = rows.map((row) => {
|
|
2303
|
-
const baseText = row.segment_text || row.transcript_text || '';
|
|
2304
|
-
const idx = baseText.toLowerCase().indexOf(query.toLowerCase());
|
|
2305
|
-
let snippet = baseText;
|
|
2306
|
-
if (idx >= 0) {
|
|
2307
|
-
const left = Math.max(0, idx - 120);
|
|
2308
|
-
const right = Math.min(baseText.length, idx + query.length + 120);
|
|
2309
|
-
snippet = baseText.slice(left, right);
|
|
2310
|
-
if (left > 0) snippet = `...${snippet}`;
|
|
2311
|
-
if (right < baseText.length) snippet = `${snippet}...`;
|
|
2312
|
-
}
|
|
2313
|
-
|
|
2314
|
-
return {
|
|
2315
|
-
sessionId: row.session_id,
|
|
2316
|
-
title: row.title,
|
|
2317
|
-
platform: row.platform,
|
|
2318
|
-
status: row.status,
|
|
2319
|
-
startedAt: row.started_at,
|
|
2320
|
-
endedAt: row.ended_at,
|
|
2321
|
-
durationMs: Number(row.duration_ms) || 0,
|
|
2322
|
-
segmentId: row.segment_id == null ? null : Number(row.segment_id),
|
|
2323
|
-
startMs: row.start_ms == null ? null : Number(row.start_ms),
|
|
2324
|
-
endMs: row.end_ms == null ? null : Number(row.end_ms),
|
|
2325
|
-
speaker: row.speaker || null,
|
|
2326
|
-
snippet: compactTranscript(snippet, 400),
|
|
2327
|
-
};
|
|
2328
|
-
});
|
|
2329
|
-
|
|
2330
|
-
return {
|
|
2331
|
-
query,
|
|
2332
|
-
count: matches.length,
|
|
2333
|
-
matches,
|
|
2334
|
-
};
|
|
2335
|
-
}
|
|
2336
|
-
|
|
2337
2103
|
case 'list_chats': {
|
|
2338
2104
|
const listPlatform = typeof args.platform === 'string' ? args.platform.trim().toLowerCase() : null;
|
|
2339
2105
|
|
|
@@ -29,9 +29,6 @@ function normalizeDesktopHello(message) {
|
|
|
29
29
|
: {},
|
|
30
30
|
displays: Array.isArray(device.displays) ? device.displays : [],
|
|
31
31
|
activeDisplayId: String(device.activeDisplayId || '').trim(),
|
|
32
|
-
passiveHistoryEnabled: device.passiveHistoryEnabled === true,
|
|
33
|
-
passiveHistoryLastUploadedAt: String(device.passiveHistoryLastUploadedAt || '').trim(),
|
|
34
|
-
passiveHistoryLastError: String(device.passiveHistoryLastError || '').trim(),
|
|
35
32
|
metadata: device.metadata && typeof device.metadata === 'object'
|
|
36
33
|
? device.metadata
|
|
37
34
|
: {},
|