neoagent 3.1.1-beta.9 → 3.2.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -2
- package/README.md +5 -1
- package/docs/architecture.md +1 -1
- package/docs/benchmarking.md +68 -87
- package/docs/capabilities.md +1 -1
- package/docs/clients-and-devices.md +3 -4
- package/docs/configuration.md +1 -1
- package/docs/devices.md +1 -2
- package/docs/health.md +24 -0
- package/docs/index.md +2 -2
- package/docs/integrations-architecture.md +6 -0
- package/docs/integrations.md +15 -1
- package/docs/models.md +1 -1
- package/docs/persistence.md +1 -1
- package/docs/why-neoagent.md +1 -1
- package/flutter_app/README.md +0 -5
- package/flutter_app/android/app/src/main/AndroidManifest.xml +0 -14
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +0 -98
- package/flutter_app/lib/main.dart +0 -6
- package/flutter_app/lib/main_app_shell.dart +0 -2
- package/flutter_app/lib/main_controller.dart +2 -867
- package/flutter_app/lib/main_devices.dart +0 -73
- package/flutter_app/lib/main_integrations.dart +339 -187
- package/flutter_app/lib/main_launcher.dart +2 -212
- package/flutter_app/lib/main_models.dart +0 -259
- package/flutter_app/lib/main_navigation.dart +2 -14
- package/flutter_app/lib/main_operations.dart +875 -383
- package/flutter_app/lib/main_runtime.dart +9 -208
- package/flutter_app/lib/main_settings.dart +17 -199
- package/flutter_app/lib/main_shared.dart +20 -591
- package/flutter_app/lib/main_timeline.dart +26 -122
- package/flutter_app/lib/main_voice_assistant.dart +0 -5
- package/flutter_app/lib/src/backend_client.dart +0 -102
- package/flutter_app/lib/src/desktop_companion_io.dart +1 -65
- package/flutter_app/lib/src/desktop_companion_stub.dart +0 -12
- package/flutter_app/linux/flutter/generated_plugin_registrant.cc +0 -4
- package/flutter_app/linux/flutter/generated_plugins.cmake +0 -1
- package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +0 -2
- package/flutter_app/macos/Podfile.lock +56 -5
- package/flutter_app/pubspec.lock +0 -7
- package/flutter_app/pubspec.yaml +0 -2
- package/flutter_app/web/index.html +174 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.cc +0 -3
- package/flutter_app/windows/flutter/generated_plugins.cmake +0 -1
- package/landing/index.html +0 -2
- package/lib/manager.js +1 -1
- package/lib/schema_migrations.js +75 -0
- package/package.json +1 -4
- package/server/db/database.js +1 -133
- package/server/http/middleware.js +2 -16
- package/server/http/routes.js +0 -2
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/NOTICES +0 -25
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +2 -2
- package/server/public/index.html +174 -0
- package/server/public/main.dart.js +92036 -91145
- package/server/routes/auth.js +11 -2
- package/server/routes/settings.js +0 -5
- package/server/routes/voice_assistant.js +1 -128
- package/server/services/account/erasure.js +0 -3
- package/server/services/ai/deliverables/artifact_helpers.js +1 -1
- package/server/services/ai/loop/conversation_loop.js +6 -2
- package/server/services/ai/loop/tool_dispatch.js +0 -3
- package/server/services/ai/messagingFallback.js +1 -1
- package/server/services/ai/settings.js +0 -6
- package/server/services/ai/toolEvidence.js +1 -4
- package/server/services/ai/tools.js +0 -234
- package/server/services/desktop/auth.js +0 -3
- package/server/services/desktop/registry.js +2 -50
- package/server/services/integrations/neoarchive/provider.js +616 -0
- package/server/services/integrations/neorecall/client.js +140 -0
- package/server/services/integrations/neorecall/constants.js +87 -0
- package/server/services/integrations/neorecall/provider.js +163 -0
- package/server/services/integrations/registry.js +4 -2
- package/server/services/manager.js +0 -22
- package/server/services/memory/manager.js +44 -19
- package/server/services/memory/retrieval_reasoning.js +36 -14
- package/server/services/messaging/formatting_guides.js +1 -0
- package/server/services/messaging/whatsapp.js +18 -0
- package/server/services/security/tool_categories.js +0 -5
- package/server/services/social_video/service.js +2 -20
- package/server/services/tasks/adapters/index.js +1 -1
- package/server/services/tasks/integration_runtime.js +3 -67
- package/server/services/tasks/runtime.js +1 -6
- package/server/services/tasks/task_repository.js +6 -13
- package/server/services/timeline/service.js +0 -224
- package/server/services/voice/message.js +0 -24
- package/server/services/voice/providers.js +42 -1
- package/server/services/voice/runtimeManager.js +9 -5
- package/server/services/voice/turnRunner.js +0 -22
- package/server/services/wearable/service.js +0 -5
- package/docs/recordings-and-health.md +0 -41
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +0 -21
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +0 -586
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +0 -78
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +0 -104
- package/flutter_app/lib/main_recordings.dart +0 -920
- package/flutter_app/lib/src/desktop_ocr_bridge.dart +0 -2
- package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +0 -125
- package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +0 -30
- package/flutter_app/lib/src/desktop_passive_history.dart +0 -332
- package/flutter_app/lib/src/recording_bridge.dart +0 -232
- package/flutter_app/lib/src/recording_bridge_io.dart +0 -1027
- package/flutter_app/lib/src/recording_bridge_stub.dart +0 -120
- package/flutter_app/lib/src/recording_bridge_web.dart +0 -702
- package/flutter_app/lib/src/recording_chunk_queue.dart +0 -149
- package/flutter_app/lib/src/recording_chunk_queue_io.dart +0 -182
- package/flutter_app/lib/src/recording_payloads.dart +0 -95
- package/flutter_app/third_party/desktop_audio_capture/LICENSE +0 -21
- package/flutter_app/third_party/desktop_audio_capture/README.md +0 -262
- package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +0 -65
- package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +0 -153
- package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +0 -110
- package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +0 -461
- package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +0 -91
- package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +0 -106
- package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +0 -219
- package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +0 -336
- package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +0 -101
- package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +0 -692
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +0 -35
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +0 -36
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +0 -32
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +0 -32
- package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +0 -878
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +0 -27
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +0 -1172
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +0 -655
- package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +0 -12
- package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +0 -30
- package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +0 -87
- package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +0 -105
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +0 -80
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +0 -31
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +0 -12
- package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +0 -23
- package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +0 -25
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +0 -1117
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +0 -115
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +0 -777
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +0 -87
- package/server/routes/recordings.js +0 -335
- package/server/routes/screenHistory.js +0 -190
- package/server/services/ai/recordingInsights.js +0 -105
- package/server/services/integrations/neomail/provider.js +0 -993
- package/server/services/recordings/deepgram.js +0 -58
- package/server/services/recordings/manager.js +0 -1273
- package/server/services/tasks/adapters/neomail_email_received.js +0 -46
- package/server/services/voice/screenshotContext.js +0 -73
- package/server/services/voice/turnCoordinator.js +0 -51
|
@@ -1,1273 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
const fs = require('fs');
|
|
4
|
-
const path = require('path');
|
|
5
|
-
const { v4: uuidv4 } = require('uuid');
|
|
6
|
-
|
|
7
|
-
const db = require('../../db/database');
|
|
8
|
-
const { DATA_DIR } = require('../../../runtime/paths');
|
|
9
|
-
const { sanitizeError } = require('../../utils/security');
|
|
10
|
-
const {
|
|
11
|
-
DEFAULT_LANGUAGE,
|
|
12
|
-
DEFAULT_MODEL,
|
|
13
|
-
isDeepgramConfigured,
|
|
14
|
-
transcribeChunkWithDeepgram,
|
|
15
|
-
} = require('./deepgram');
|
|
16
|
-
const { extractRecordingInsights } = require('../ai/recordingInsights');
|
|
17
|
-
const { getAiSettings } = require('../ai/settings');
|
|
18
|
-
|
|
19
|
-
const RECORDINGS_DIR = path.join(DATA_DIR, 'recordings');
|
|
20
|
-
const SESSION_STATUS = {
|
|
21
|
-
recording: 'recording',
|
|
22
|
-
processing: 'processing',
|
|
23
|
-
completed: 'completed',
|
|
24
|
-
failed: 'failed',
|
|
25
|
-
cancelled: 'cancelled',
|
|
26
|
-
};
|
|
27
|
-
|
|
28
|
-
function ensureRecordingDirs() {
|
|
29
|
-
fs.mkdirSync(RECORDINGS_DIR, { recursive: true });
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
function validateSourceKey(key) {
|
|
33
|
-
if (!key || !/^[a-z0-9][a-z0-9_-]*$/.test(key)) {
|
|
34
|
-
throw new Error(`Invalid sourceKey "${key}": only lowercase alphanumeric, hyphens, and underscores are allowed.`);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
class RecordingManager {
|
|
39
|
-
constructor(io) {
|
|
40
|
-
this.io = io;
|
|
41
|
-
ensureRecordingDirs();
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
listSessions(userId, { limit = 24 } = {}) {
|
|
45
|
-
const rows = db.prepare(`
|
|
46
|
-
SELECT *
|
|
47
|
-
FROM recording_sessions
|
|
48
|
-
WHERE user_id = ?
|
|
49
|
-
ORDER BY datetime(created_at) DESC
|
|
50
|
-
LIMIT ?
|
|
51
|
-
`).all(userId, Math.max(1, Math.min(Number(limit) || 24, 100)));
|
|
52
|
-
|
|
53
|
-
return rows.map((row) => this.getSession(userId, row.id));
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
getSession(userId, sessionId) {
|
|
57
|
-
const session = this.#requireOwnedSession(userId, sessionId);
|
|
58
|
-
|
|
59
|
-
const sources = this.#getSessionSources(sessionId);
|
|
60
|
-
const segments = this.#getSessionTranscriptSegments(sessionId);
|
|
61
|
-
|
|
62
|
-
return {
|
|
63
|
-
...this.#mapSession(session),
|
|
64
|
-
sources: sources.map((source) => this.#mapSource(source)),
|
|
65
|
-
transcriptSegments: segments.map((segment) => this.#mapSegment(segment)),
|
|
66
|
-
};
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
#requireOwnedSession(userId, sessionId) {
|
|
70
|
-
const session = db.prepare(`
|
|
71
|
-
SELECT *
|
|
72
|
-
FROM recording_sessions
|
|
73
|
-
WHERE id = ? AND user_id = ?
|
|
74
|
-
`).get(sessionId, userId);
|
|
75
|
-
if (!session) {
|
|
76
|
-
throw new Error('Recording session not found.');
|
|
77
|
-
}
|
|
78
|
-
return session;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
#getSessionSources(sessionId) {
|
|
82
|
-
return db.prepare(`
|
|
83
|
-
SELECT *
|
|
84
|
-
FROM recording_sources
|
|
85
|
-
WHERE session_id = ?
|
|
86
|
-
ORDER BY created_at ASC
|
|
87
|
-
`).all(sessionId);
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
#getSessionTranscriptSegments(sessionId) {
|
|
91
|
-
return db.prepare(`
|
|
92
|
-
SELECT *
|
|
93
|
-
FROM recording_transcript_segments
|
|
94
|
-
WHERE session_id = ?
|
|
95
|
-
ORDER BY start_ms ASC, id ASC
|
|
96
|
-
`).all(sessionId);
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
#clearSessionTranscriptSegments(sessionId) {
|
|
100
|
-
db.prepare(`
|
|
101
|
-
DELETE FROM recording_transcript_segments
|
|
102
|
-
WHERE session_id = ?
|
|
103
|
-
`).run(sessionId);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
#getSessionSourceByKey(sessionId, sourceKey) {
|
|
107
|
-
return db.prepare(`
|
|
108
|
-
SELECT *
|
|
109
|
-
FROM recording_sources
|
|
110
|
-
WHERE session_id = ? AND LOWER(source_key) = LOWER(?)
|
|
111
|
-
`).get(sessionId, sourceKey);
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
#touchSessionUpdatedAt(sessionId) {
|
|
115
|
-
db.prepare(`
|
|
116
|
-
UPDATE recording_sessions
|
|
117
|
-
SET updated_at = ?
|
|
118
|
-
WHERE id = ?
|
|
119
|
-
`).run(new Date().toISOString(), sessionId);
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
#countSessionChunks(sessionId) {
|
|
123
|
-
const count = db.prepare(`
|
|
124
|
-
SELECT COUNT(*) AS count
|
|
125
|
-
FROM recording_chunks c
|
|
126
|
-
INNER JOIN recording_sources s ON s.id = c.source_id
|
|
127
|
-
WHERE s.session_id = ?
|
|
128
|
-
`).get(sessionId)?.count;
|
|
129
|
-
return Number(count) || 0;
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
#statusFromChunkCount(sessionId) {
|
|
133
|
-
return this.#countSessionChunks(sessionId) > 0
|
|
134
|
-
? SESSION_STATUS.processing
|
|
135
|
-
: SESSION_STATUS.cancelled;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
#applyPostRecordingStatus(sessionId, nextStatus, options = {}) {
|
|
139
|
-
const now = options.now || new Date().toISOString();
|
|
140
|
-
const metadataJson = Object.prototype.hasOwnProperty.call(options, 'metadataJson')
|
|
141
|
-
? options.metadataJson
|
|
142
|
-
: undefined;
|
|
143
|
-
|
|
144
|
-
db.transaction(() => {
|
|
145
|
-
if (metadataJson === undefined) {
|
|
146
|
-
db.prepare(`
|
|
147
|
-
UPDATE recording_sessions
|
|
148
|
-
SET
|
|
149
|
-
status = ?,
|
|
150
|
-
ended_at = COALESCE(ended_at, ?),
|
|
151
|
-
updated_at = ?
|
|
152
|
-
WHERE id = ?
|
|
153
|
-
`).run(nextStatus, now, now, sessionId);
|
|
154
|
-
} else {
|
|
155
|
-
db.prepare(`
|
|
156
|
-
UPDATE recording_sessions
|
|
157
|
-
SET
|
|
158
|
-
status = ?,
|
|
159
|
-
ended_at = COALESCE(ended_at, ?),
|
|
160
|
-
metadata_json = ?,
|
|
161
|
-
updated_at = ?
|
|
162
|
-
WHERE id = ?
|
|
163
|
-
`).run(nextStatus, now, metadataJson, now, sessionId);
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
db.prepare(`
|
|
167
|
-
UPDATE recording_sources
|
|
168
|
-
SET
|
|
169
|
-
status = CASE WHEN chunk_count > 0 THEN ? ELSE ? END,
|
|
170
|
-
updated_at = ?
|
|
171
|
-
WHERE session_id = ?
|
|
172
|
-
`).run(
|
|
173
|
-
nextStatus,
|
|
174
|
-
nextStatus === SESSION_STATUS.processing ? SESSION_STATUS.cancelled : nextStatus,
|
|
175
|
-
now,
|
|
176
|
-
sessionId,
|
|
177
|
-
);
|
|
178
|
-
})();
|
|
179
|
-
|
|
180
|
-
return now;
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
createSession(userId, payload = {}) {
|
|
184
|
-
const sessionId = uuidv4();
|
|
185
|
-
const now = new Date().toISOString();
|
|
186
|
-
const platform = typeof payload.platform === 'string' && payload.platform.trim()
|
|
187
|
-
? payload.platform.trim()
|
|
188
|
-
: 'unknown';
|
|
189
|
-
const sources = this.#normalizeSources(payload.sources, platform);
|
|
190
|
-
const metadata = {
|
|
191
|
-
capturePlan: payload.capturePlan || 'chunked-dual-source',
|
|
192
|
-
screenAnalysisReady: payload.screenAnalysisReady !== false,
|
|
193
|
-
notes: payload.notes || null,
|
|
194
|
-
};
|
|
195
|
-
|
|
196
|
-
const insertSession = db.prepare(`
|
|
197
|
-
INSERT INTO recording_sessions (
|
|
198
|
-
id, user_id, title, platform, status, metadata_json, started_at, created_at, updated_at
|
|
199
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
200
|
-
`);
|
|
201
|
-
const insertSource = db.prepare(`
|
|
202
|
-
INSERT INTO recording_sources (
|
|
203
|
-
id, session_id, source_key, source_kind, media_kind, mime_type, status, metadata_json, created_at, updated_at
|
|
204
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
205
|
-
`);
|
|
206
|
-
|
|
207
|
-
db.transaction(() => {
|
|
208
|
-
insertSession.run(
|
|
209
|
-
sessionId,
|
|
210
|
-
userId,
|
|
211
|
-
this.#resolveTitle(payload.title, platform, now),
|
|
212
|
-
platform,
|
|
213
|
-
SESSION_STATUS.recording,
|
|
214
|
-
JSON.stringify(metadata),
|
|
215
|
-
now,
|
|
216
|
-
now,
|
|
217
|
-
now,
|
|
218
|
-
);
|
|
219
|
-
|
|
220
|
-
for (const source of sources) {
|
|
221
|
-
insertSource.run(
|
|
222
|
-
uuidv4(),
|
|
223
|
-
sessionId,
|
|
224
|
-
source.sourceKey,
|
|
225
|
-
source.sourceKind,
|
|
226
|
-
source.mediaKind,
|
|
227
|
-
source.mimeType,
|
|
228
|
-
SESSION_STATUS.recording,
|
|
229
|
-
JSON.stringify(source.metadata),
|
|
230
|
-
now,
|
|
231
|
-
now,
|
|
232
|
-
);
|
|
233
|
-
}
|
|
234
|
-
})();
|
|
235
|
-
|
|
236
|
-
this.#emitUpdate(userId, sessionId);
|
|
237
|
-
return this.getSession(userId, sessionId);
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
appendChunk(userId, sessionId, metadata = {}, audioBytes) {
|
|
241
|
-
if (!(audioBytes instanceof Buffer) || audioBytes.length === 0) {
|
|
242
|
-
throw new Error('Recording chunk is empty.');
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
const session = this.#requireOwnedSession(userId, sessionId);
|
|
246
|
-
if (![SESSION_STATUS.recording, SESSION_STATUS.processing].includes(session.status)) {
|
|
247
|
-
throw new Error('Recording session is not accepting more chunks.');
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
const sourceKey = `${metadata.sourceKey || ''}`.trim().toLowerCase();
|
|
251
|
-
validateSourceKey(sourceKey);
|
|
252
|
-
|
|
253
|
-
const source = this.#getSessionSourceByKey(sessionId, sourceKey);
|
|
254
|
-
|
|
255
|
-
if (!source) {
|
|
256
|
-
throw new Error(`Unknown recording source: ${sourceKey}`);
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
const sequenceIndex = Number(metadata.sequenceIndex);
|
|
260
|
-
if (!Number.isInteger(sequenceIndex) || sequenceIndex < 0) {
|
|
261
|
-
throw new Error('sequenceIndex must be a non-negative integer.');
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
// Accept chunks even when an earlier sequence number never arrived (e.g. a
|
|
265
|
-
// chunk upload was permanently dropped by a flaky network). A gap only
|
|
266
|
-
// costs the audio in the missing chunk; every later chunk is still stored
|
|
267
|
-
// and transcribed. Rejecting non-contiguous sequences here would instead
|
|
268
|
-
// cascade a single lost chunk into total loss of the rest of the source.
|
|
269
|
-
const existing = db.prepare(`
|
|
270
|
-
SELECT id
|
|
271
|
-
FROM recording_chunks
|
|
272
|
-
WHERE source_id = ? AND sequence_index = ?
|
|
273
|
-
`).get(source.id, sequenceIndex);
|
|
274
|
-
if (existing) {
|
|
275
|
-
return {
|
|
276
|
-
duplicate: true,
|
|
277
|
-
accepted: false,
|
|
278
|
-
sessionId,
|
|
279
|
-
sourceKey,
|
|
280
|
-
sequenceIndex,
|
|
281
|
-
};
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
const mimeType = `${metadata.mimeType || source.mime_type || 'application/octet-stream'}`.trim();
|
|
285
|
-
const startMs = Math.max(0, Number(metadata.startMs) || 0);
|
|
286
|
-
const endMs = Math.max(startMs, Number(metadata.endMs) || startMs);
|
|
287
|
-
const extension = this.#extensionForMime(mimeType);
|
|
288
|
-
const fileDir = path.join(RECORDINGS_DIR, `user-${userId}`, sessionId, sourceKey);
|
|
289
|
-
const sessionRoot = path.resolve(path.join(RECORDINGS_DIR, `user-${userId}`, sessionId));
|
|
290
|
-
if (!path.resolve(fileDir).startsWith(sessionRoot + path.sep)) {
|
|
291
|
-
throw new Error('Invalid recording path');
|
|
292
|
-
}
|
|
293
|
-
fs.mkdirSync(fileDir, { recursive: true });
|
|
294
|
-
const filePath = path.join(fileDir, `${String(sequenceIndex).padStart(6, '0')}${extension}`);
|
|
295
|
-
const tempPath = `${filePath}.tmp-${process.pid}-${Date.now()}`;
|
|
296
|
-
fs.writeFileSync(tempPath, audioBytes);
|
|
297
|
-
fs.renameSync(tempPath, filePath);
|
|
298
|
-
|
|
299
|
-
db.transaction(() => {
|
|
300
|
-
db.prepare(`
|
|
301
|
-
INSERT INTO recording_chunks (
|
|
302
|
-
source_id, sequence_index, start_ms, end_ms, byte_count, mime_type, file_path
|
|
303
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
304
|
-
`).run(
|
|
305
|
-
source.id,
|
|
306
|
-
sequenceIndex,
|
|
307
|
-
startMs,
|
|
308
|
-
endMs,
|
|
309
|
-
audioBytes.length,
|
|
310
|
-
mimeType,
|
|
311
|
-
filePath,
|
|
312
|
-
);
|
|
313
|
-
|
|
314
|
-
db.prepare(`
|
|
315
|
-
UPDATE recording_sources
|
|
316
|
-
SET
|
|
317
|
-
mime_type = COALESCE(?, mime_type),
|
|
318
|
-
chunk_count = chunk_count + 1,
|
|
319
|
-
bytes_received = bytes_received + ?,
|
|
320
|
-
duration_ms = MAX(duration_ms, ?),
|
|
321
|
-
updated_at = ?
|
|
322
|
-
WHERE id = ?
|
|
323
|
-
`).run(
|
|
324
|
-
mimeType,
|
|
325
|
-
audioBytes.length,
|
|
326
|
-
endMs,
|
|
327
|
-
new Date().toISOString(),
|
|
328
|
-
source.id,
|
|
329
|
-
);
|
|
330
|
-
|
|
331
|
-
this.#touchSessionUpdatedAt(sessionId);
|
|
332
|
-
})();
|
|
333
|
-
|
|
334
|
-
return {
|
|
335
|
-
duplicate: false,
|
|
336
|
-
accepted: true,
|
|
337
|
-
sessionId,
|
|
338
|
-
sourceKey,
|
|
339
|
-
sequenceIndex,
|
|
340
|
-
bytesReceived: audioBytes.length,
|
|
341
|
-
};
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
finalizeSession(userId, sessionId, options = {}) {
|
|
345
|
-
const session = this.#requireOwnedSession(userId, sessionId);
|
|
346
|
-
|
|
347
|
-
const stopReason = `${options.stopReason || 'stopped'}`.trim();
|
|
348
|
-
const mergedMetadata = {
|
|
349
|
-
...this.#parseJson(session.metadata_json, {}),
|
|
350
|
-
stopReason,
|
|
351
|
-
};
|
|
352
|
-
const nextStatus = this.#statusFromChunkCount(sessionId);
|
|
353
|
-
this.#applyPostRecordingStatus(sessionId, nextStatus, {
|
|
354
|
-
metadataJson: JSON.stringify(mergedMetadata),
|
|
355
|
-
});
|
|
356
|
-
|
|
357
|
-
this.#emitUpdate(userId, sessionId);
|
|
358
|
-
|
|
359
|
-
const stopReasonLower = String(stopReason || '').toLowerCase();
|
|
360
|
-
const isVoiceAssistantStop = stopReasonLower === 'voice_assistant' || stopReasonLower === 'voice';
|
|
361
|
-
const includeInsights = options.includeInsights === undefined
|
|
362
|
-
? !isVoiceAssistantStop
|
|
363
|
-
: options.includeInsights !== false;
|
|
364
|
-
const autoProcess = options.autoProcess !== false;
|
|
365
|
-
|
|
366
|
-
if (nextStatus === SESSION_STATUS.processing && autoProcess) {
|
|
367
|
-
this.processSession(userId, sessionId, {
|
|
368
|
-
includeInsights,
|
|
369
|
-
}).catch((error) => {
|
|
370
|
-
console.error('[Recordings] Processing failed:', sanitizeError(error));
|
|
371
|
-
});
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
return this.getSession(userId, sessionId);
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
async retrySession(userId, sessionId) {
|
|
378
|
-
this.#requireOwnedSession(userId, sessionId);
|
|
379
|
-
if (!isDeepgramConfigured()) {
|
|
380
|
-
throw new Error('DEEPGRAM_API_KEY is not configured.');
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
db.prepare(`
|
|
384
|
-
UPDATE recording_sessions
|
|
385
|
-
SET status = ?, last_error = NULL, updated_at = ?
|
|
386
|
-
WHERE id = ?
|
|
387
|
-
`).run(SESSION_STATUS.processing, new Date().toISOString(), sessionId);
|
|
388
|
-
this.#emitUpdate(userId, sessionId);
|
|
389
|
-
|
|
390
|
-
await this.processSession(userId, sessionId);
|
|
391
|
-
return this.getSession(userId, sessionId);
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
async resumePendingSessions() {
|
|
395
|
-
this.reconcileStaleRecordingSessions();
|
|
396
|
-
|
|
397
|
-
const rows = db.prepare(`
|
|
398
|
-
SELECT id, user_id
|
|
399
|
-
FROM recording_sessions
|
|
400
|
-
WHERE status = ?
|
|
401
|
-
ORDER BY created_at ASC
|
|
402
|
-
`).all(SESSION_STATUS.processing);
|
|
403
|
-
|
|
404
|
-
for (const row of rows) {
|
|
405
|
-
try {
|
|
406
|
-
await this.processSession(row.user_id, row.id);
|
|
407
|
-
} catch (error) {
|
|
408
|
-
console.error('[Recordings] Resume failed:', sanitizeError(error));
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
reconcileStaleRecordingSessions() {
|
|
414
|
-
const staleRows = db.prepare(`
|
|
415
|
-
SELECT id, user_id
|
|
416
|
-
FROM recording_sessions
|
|
417
|
-
WHERE status = ?
|
|
418
|
-
ORDER BY created_at ASC
|
|
419
|
-
`).all(SESSION_STATUS.recording);
|
|
420
|
-
|
|
421
|
-
for (const row of staleRows) {
|
|
422
|
-
const nextStatus = this.#statusFromChunkCount(row.id);
|
|
423
|
-
this.#applyPostRecordingStatus(row.id, nextStatus);
|
|
424
|
-
|
|
425
|
-
this.#emitUpdate(row.user_id, row.id);
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
async processSession(userId, sessionId, options = {}) {
|
|
430
|
-
this.#requireOwnedSession(userId, sessionId);
|
|
431
|
-
|
|
432
|
-
const sources = this.#getSessionSources(sessionId);
|
|
433
|
-
if (sources.length == 0) {
|
|
434
|
-
throw new Error('Recording session has no sources.');
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
db.transaction(() => {
|
|
438
|
-
this.#clearSessionTranscriptSegments(sessionId);
|
|
439
|
-
db.prepare(`
|
|
440
|
-
UPDATE recording_sources
|
|
441
|
-
SET status = ?, updated_at = ?
|
|
442
|
-
WHERE session_id = ?
|
|
443
|
-
`).run(SESSION_STATUS.processing, new Date().toISOString(), sessionId);
|
|
444
|
-
})();
|
|
445
|
-
|
|
446
|
-
const collectedSegments = [];
|
|
447
|
-
let maxDuration = 0;
|
|
448
|
-
const aiSettings = getAiSettings(userId);
|
|
449
|
-
const recordingTranscriptionProvider = `${aiSettings.default_recording_transcription_provider || 'deepgram'}`
|
|
450
|
-
.trim()
|
|
451
|
-
.toLowerCase() || 'deepgram';
|
|
452
|
-
const recordingTranscriptionModel = `${aiSettings.default_recording_transcription_model || DEFAULT_MODEL}`.trim() || DEFAULT_MODEL;
|
|
453
|
-
const recordingSummaryProvider = `${aiSettings.default_recording_summary_provider || 'auto'}`.trim() || 'auto';
|
|
454
|
-
const recordingSummaryModel = `${aiSettings.default_recording_summary_model || 'auto'}`.trim() || 'auto';
|
|
455
|
-
|
|
456
|
-
try {
|
|
457
|
-
if (!isDeepgramConfigured()) {
|
|
458
|
-
throw new Error('DEEPGRAM_API_KEY is not configured.');
|
|
459
|
-
}
|
|
460
|
-
if (recordingTranscriptionProvider !== 'deepgram') {
|
|
461
|
-
throw new Error(`Recording transcription provider "${recordingTranscriptionProvider}" is not supported yet.`);
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
for (const source of sources) {
|
|
465
|
-
const sourceMetadata = this.#parseJson(source.metadata_json, {});
|
|
466
|
-
const chunks = db.prepare(`
|
|
467
|
-
SELECT *
|
|
468
|
-
FROM recording_chunks
|
|
469
|
-
WHERE source_id = ?
|
|
470
|
-
ORDER BY sequence_index ASC
|
|
471
|
-
`).all(source.id);
|
|
472
|
-
|
|
473
|
-
if (chunks.length === 0) {
|
|
474
|
-
continue;
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
this.#warnOnChunkGaps(source.source_key, chunks);
|
|
478
|
-
const sourceDuration = this.#maxOf(
|
|
479
|
-
chunks.map((chunk) => Number(chunk.end_ms) || 0),
|
|
480
|
-
Number(source.duration_ms) || 0,
|
|
481
|
-
);
|
|
482
|
-
maxDuration = Math.max(maxDuration, sourceDuration);
|
|
483
|
-
|
|
484
|
-
if (!this.#shouldTranscribeSource(source, sourceMetadata)) {
|
|
485
|
-
db.prepare(`
|
|
486
|
-
UPDATE recording_sources
|
|
487
|
-
SET status = ?, duration_ms = ?, updated_at = ?
|
|
488
|
-
WHERE id = ?
|
|
489
|
-
`).run(
|
|
490
|
-
SESSION_STATUS.completed,
|
|
491
|
-
sourceDuration,
|
|
492
|
-
new Date().toISOString(),
|
|
493
|
-
source.id,
|
|
494
|
-
);
|
|
495
|
-
continue;
|
|
496
|
-
}
|
|
497
|
-
|
|
498
|
-
const sourceSegments = await this.#transcribeSourceChunks(source, chunks, {
|
|
499
|
-
transcriptionModel: recordingTranscriptionModel,
|
|
500
|
-
});
|
|
501
|
-
const sourceSegmentEnd = this.#maxOf(
|
|
502
|
-
sourceSegments.map((segment) => Number(segment.endMs) || 0),
|
|
503
|
-
sourceDuration,
|
|
504
|
-
);
|
|
505
|
-
maxDuration = Math.max(maxDuration, sourceSegmentEnd);
|
|
506
|
-
collectedSegments.push(...sourceSegments);
|
|
507
|
-
|
|
508
|
-
db.prepare(`
|
|
509
|
-
UPDATE recording_sources
|
|
510
|
-
SET status = ?, duration_ms = ?, updated_at = ?
|
|
511
|
-
WHERE id = ?
|
|
512
|
-
`).run(
|
|
513
|
-
SESSION_STATUS.completed,
|
|
514
|
-
sourceSegmentEnd,
|
|
515
|
-
new Date().toISOString(),
|
|
516
|
-
source.id,
|
|
517
|
-
);
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
const insertSegment = db.prepare(`
|
|
521
|
-
INSERT INTO recording_transcript_segments (
|
|
522
|
-
session_id, source_id, source_key, speaker, text, start_ms, end_ms, confidence, words_json
|
|
523
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
524
|
-
`);
|
|
525
|
-
const ordered = collectedSegments.sort((a, b) => {
|
|
526
|
-
if (a.startMs !== b.startMs) {
|
|
527
|
-
return a.startMs - b.startMs;
|
|
528
|
-
}
|
|
529
|
-
return a.sourceKey.localeCompare(b.sourceKey);
|
|
530
|
-
});
|
|
531
|
-
|
|
532
|
-
db.transaction(() => {
|
|
533
|
-
for (const segment of ordered) {
|
|
534
|
-
insertSegment.run(
|
|
535
|
-
sessionId,
|
|
536
|
-
segment.sourceId,
|
|
537
|
-
segment.sourceKey,
|
|
538
|
-
segment.speaker,
|
|
539
|
-
segment.text,
|
|
540
|
-
segment.startMs,
|
|
541
|
-
segment.endMs,
|
|
542
|
-
segment.confidence,
|
|
543
|
-
JSON.stringify(segment.words),
|
|
544
|
-
);
|
|
545
|
-
}
|
|
546
|
-
})();
|
|
547
|
-
|
|
548
|
-
const transcriptText = this.#composeTranscriptText(ordered);
|
|
549
|
-
|
|
550
|
-
const includeInsights = options.includeInsights !== false;
|
|
551
|
-
let structuredInsights = null;
|
|
552
|
-
if (includeInsights && transcriptText && aiSettings.auto_recording_insights) {
|
|
553
|
-
try {
|
|
554
|
-
structuredInsights = await extractRecordingInsights(userId, transcriptText, {
|
|
555
|
-
provider: recordingSummaryProvider,
|
|
556
|
-
model: recordingSummaryModel,
|
|
557
|
-
});
|
|
558
|
-
} catch (err) {
|
|
559
|
-
console.error(`[Recordings] Failed to extract insights for session ${sessionId}:`, sanitizeError(err));
|
|
560
|
-
}
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
db.prepare(`
|
|
564
|
-
UPDATE recording_sessions
|
|
565
|
-
SET
|
|
566
|
-
status = ?,
|
|
567
|
-
transcript_text = ?,
|
|
568
|
-
transcript_language = ?,
|
|
569
|
-
transcript_model = ?,
|
|
570
|
-
structured_content_json = ?,
|
|
571
|
-
duration_ms = ?,
|
|
572
|
-
last_error = NULL,
|
|
573
|
-
updated_at = ?,
|
|
574
|
-
ended_at = COALESCE(ended_at, ?)
|
|
575
|
-
WHERE id = ?
|
|
576
|
-
`).run(
|
|
577
|
-
SESSION_STATUS.completed,
|
|
578
|
-
transcriptText,
|
|
579
|
-
DEFAULT_LANGUAGE,
|
|
580
|
-
recordingTranscriptionModel,
|
|
581
|
-
structuredInsights ? JSON.stringify(structuredInsights) : null,
|
|
582
|
-
maxDuration,
|
|
583
|
-
new Date().toISOString(),
|
|
584
|
-
new Date().toISOString(),
|
|
585
|
-
sessionId,
|
|
586
|
-
);
|
|
587
|
-
} catch (error) {
|
|
588
|
-
db.prepare(`
|
|
589
|
-
UPDATE recording_sessions
|
|
590
|
-
SET status = ?, last_error = ?, updated_at = ?
|
|
591
|
-
WHERE id = ?
|
|
592
|
-
`).run(
|
|
593
|
-
SESSION_STATUS.failed,
|
|
594
|
-
sanitizeError(error),
|
|
595
|
-
new Date().toISOString(),
|
|
596
|
-
sessionId,
|
|
597
|
-
);
|
|
598
|
-
db.prepare(`
|
|
599
|
-
UPDATE recording_sources
|
|
600
|
-
SET status = ?, updated_at = ?
|
|
601
|
-
WHERE session_id = ?
|
|
602
|
-
`).run(SESSION_STATUS.failed, new Date().toISOString(), sessionId);
|
|
603
|
-
this.#emitUpdate(userId, sessionId);
|
|
604
|
-
throw error;
|
|
605
|
-
}
|
|
606
|
-
|
|
607
|
-
this.#emitUpdate(userId, sessionId);
|
|
608
|
-
return this.getSession(userId, sessionId);
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
deleteTranscriptSegment(userId, sessionId, segmentId) {
|
|
612
|
-
this.#requireOwnedSession(userId, sessionId);
|
|
613
|
-
|
|
614
|
-
const normalizedSegmentId = Number(segmentId);
|
|
615
|
-
if (!Number.isInteger(normalizedSegmentId) || normalizedSegmentId <= 0) {
|
|
616
|
-
throw new Error('segmentId must be a positive integer.');
|
|
617
|
-
}
|
|
618
|
-
|
|
619
|
-
const segment = db.prepare(`
|
|
620
|
-
SELECT id
|
|
621
|
-
FROM recording_transcript_segments
|
|
622
|
-
WHERE session_id = ? AND id = ?
|
|
623
|
-
`).get(sessionId, normalizedSegmentId);
|
|
624
|
-
if (!segment) {
|
|
625
|
-
throw new Error('Transcript segment not found.');
|
|
626
|
-
}
|
|
627
|
-
|
|
628
|
-
const now = new Date().toISOString();
|
|
629
|
-
let transcriptText = '';
|
|
630
|
-
db.transaction(() => {
|
|
631
|
-
db.prepare(`
|
|
632
|
-
DELETE FROM recording_transcript_segments
|
|
633
|
-
WHERE session_id = ? AND id = ?
|
|
634
|
-
`).run(sessionId, normalizedSegmentId);
|
|
635
|
-
|
|
636
|
-
const remainingSegments = db.prepare(`
|
|
637
|
-
SELECT start_ms, text
|
|
638
|
-
FROM recording_transcript_segments
|
|
639
|
-
WHERE session_id = ?
|
|
640
|
-
ORDER BY start_ms ASC, id ASC
|
|
641
|
-
`).all(sessionId);
|
|
642
|
-
transcriptText = this.#composeTranscriptText(remainingSegments);
|
|
643
|
-
|
|
644
|
-
db.prepare(`
|
|
645
|
-
UPDATE recording_sessions
|
|
646
|
-
SET transcript_text = ?, updated_at = ?
|
|
647
|
-
WHERE id = ?
|
|
648
|
-
`).run(transcriptText, now, sessionId);
|
|
649
|
-
})();
|
|
650
|
-
|
|
651
|
-
this.#emitUpdate(userId, sessionId);
|
|
652
|
-
return this.getSession(userId, sessionId);
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
deleteSession(userId, sessionId) {
|
|
656
|
-
this.#requireOwnedSession(userId, sessionId);
|
|
657
|
-
|
|
658
|
-
const chunkRows = db.prepare(`
|
|
659
|
-
SELECT c.file_path AS filePath
|
|
660
|
-
FROM recording_chunks c
|
|
661
|
-
INNER JOIN recording_sources s ON s.id = c.source_id
|
|
662
|
-
WHERE s.session_id = ?
|
|
663
|
-
`).all(sessionId);
|
|
664
|
-
db.prepare(`
|
|
665
|
-
DELETE FROM recording_sessions
|
|
666
|
-
WHERE id = ?
|
|
667
|
-
`).run(sessionId);
|
|
668
|
-
|
|
669
|
-
for (const row of chunkRows) {
|
|
670
|
-
const filePath = `${row?.filePath || ''}`.trim();
|
|
671
|
-
if (!filePath) {
|
|
672
|
-
continue;
|
|
673
|
-
}
|
|
674
|
-
try {
|
|
675
|
-
const recordingsRoot = path.resolve(RECORDINGS_DIR);
|
|
676
|
-
const recordingsPrefix = recordingsRoot.endsWith(path.sep)
|
|
677
|
-
? recordingsRoot
|
|
678
|
-
: `${recordingsRoot}${path.sep}`;
|
|
679
|
-
const resolvedPath = path.resolve(path.normalize(filePath));
|
|
680
|
-
if (resolvedPath !== recordingsRoot && !resolvedPath.startsWith(recordingsPrefix)) {
|
|
681
|
-
console.warn(`[Recordings] Skipping deletion outside recordings dir: ${sanitizeError(resolvedPath)}`);
|
|
682
|
-
continue;
|
|
683
|
-
}
|
|
684
|
-
if (fs.existsSync(resolvedPath)) {
|
|
685
|
-
fs.unlinkSync(resolvedPath);
|
|
686
|
-
}
|
|
687
|
-
} catch (error) {
|
|
688
|
-
console.warn(`[Recordings] Failed to delete recording chunk file: ${sanitizeError(error)}`);
|
|
689
|
-
}
|
|
690
|
-
}
|
|
691
|
-
|
|
692
|
-
const sessionDir = path.join(RECORDINGS_DIR, `user-${userId}`, sessionId);
|
|
693
|
-
try {
|
|
694
|
-
if (fs.existsSync(sessionDir)) {
|
|
695
|
-
fs.rmSync(sessionDir, { recursive: true, force: true });
|
|
696
|
-
}
|
|
697
|
-
} catch (error) {
|
|
698
|
-
console.warn(`[Recordings] Failed to delete recording session directory: ${sanitizeError(error)}`);
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
this.#emitUpdate(userId, sessionId);
|
|
702
|
-
}
|
|
703
|
-
|
|
704
|
-
async #transcribeSourceChunks(source, chunks, { transcriptionModel = DEFAULT_MODEL } = {}) {
|
|
705
|
-
if (this.#canTranscribeAsSingleWav(source, chunks)) {
|
|
706
|
-
return this.#transcribeMergedWavSource(source, chunks, { transcriptionModel });
|
|
707
|
-
}
|
|
708
|
-
|
|
709
|
-
if (this.#canTranscribeAsMergedBinary(source, chunks)) {
|
|
710
|
-
try {
|
|
711
|
-
return await this.#transcribeMergedBinarySource(source, chunks, { transcriptionModel });
|
|
712
|
-
} catch (error) {
|
|
713
|
-
console.warn(
|
|
714
|
-
`[Recordings] Merged transcription failed for source ${source.source_key}; falling back to per-chunk mode: ${sanitizeError(error)}`,
|
|
715
|
-
);
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
const segments = [];
|
|
720
|
-
|
|
721
|
-
for (const chunk of chunks) {
|
|
722
|
-
const audioBytes = fs.readFileSync(chunk.file_path);
|
|
723
|
-
if (!audioBytes || audioBytes.length === 0) {
|
|
724
|
-
console.warn(
|
|
725
|
-
`[Recordings] Skipping empty chunk for source ${source.source_key} ` +
|
|
726
|
-
`(seq=${chunk.sequence_index}, bytes=${audioBytes?.length || 0})`,
|
|
727
|
-
);
|
|
728
|
-
continue;
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
try {
|
|
732
|
-
const payload = await transcribeChunkWithDeepgram({
|
|
733
|
-
audioBytes,
|
|
734
|
-
mimeType: chunk.mime_type || source.mime_type,
|
|
735
|
-
detectLanguage: DEFAULT_LANGUAGE,
|
|
736
|
-
model: transcriptionModel,
|
|
737
|
-
});
|
|
738
|
-
segments.push(...this.#extractSegments(source, chunk, payload));
|
|
739
|
-
} catch (error) {
|
|
740
|
-
if (this.#isSkippableTranscriptionError(error)) {
|
|
741
|
-
console.warn(
|
|
742
|
-
`[Recordings] Skipping unsupported/corrupt chunk for source ${source.source_key} ` +
|
|
743
|
-
`(seq=${chunk.sequence_index}): ${sanitizeError(error)}`,
|
|
744
|
-
);
|
|
745
|
-
continue;
|
|
746
|
-
}
|
|
747
|
-
throw error;
|
|
748
|
-
}
|
|
749
|
-
}
|
|
750
|
-
|
|
751
|
-
return segments;
|
|
752
|
-
}
|
|
753
|
-
|
|
754
|
-
#isSkippableTranscriptionError(error) {
|
|
755
|
-
const message = `${error?.message || error || ''}`.toLowerCase();
|
|
756
|
-
if (!message) {
|
|
757
|
-
return false;
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
const isDeepgramRequestError =
|
|
761
|
-
message.includes('deepgram request failed') || message.includes('bad request');
|
|
762
|
-
const likelyCorruptPayload =
|
|
763
|
-
message.includes('audio corrupt') ||
|
|
764
|
-
message.includes('unsupported') ||
|
|
765
|
-
message.includes('cannot decode') ||
|
|
766
|
-
message.includes('invalid audio') ||
|
|
767
|
-
message.includes('encoding');
|
|
768
|
-
|
|
769
|
-
return isDeepgramRequestError && likelyCorruptPayload;
|
|
770
|
-
}
|
|
771
|
-
|
|
772
|
-
async #transcribeMergedWavSource(source, chunks, { transcriptionModel = DEFAULT_MODEL } = {}) {
|
|
773
|
-
const wavParts = chunks.map((chunk) => {
|
|
774
|
-
const audioBytes = fs.readFileSync(chunk.file_path);
|
|
775
|
-
return {
|
|
776
|
-
chunk,
|
|
777
|
-
wav: this.#parseWav(audioBytes),
|
|
778
|
-
};
|
|
779
|
-
});
|
|
780
|
-
|
|
781
|
-
const reference = wavParts[0]?.wav;
|
|
782
|
-
if (!reference) {
|
|
783
|
-
return [];
|
|
784
|
-
}
|
|
785
|
-
|
|
786
|
-
for (const part of wavParts.slice(1)) {
|
|
787
|
-
if (
|
|
788
|
-
part.wav.audioFormat !== reference.audioFormat
|
|
789
|
-
|| part.wav.channelCount !== reference.channelCount
|
|
790
|
-
|| part.wav.sampleRate !== reference.sampleRate
|
|
791
|
-
|| part.wav.bitsPerSample !== reference.bitsPerSample
|
|
792
|
-
) {
|
|
793
|
-
throw new Error(`Recording source "${source.source_key}" changed WAV format mid-session.`);
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
|
|
797
|
-
const spans = [];
|
|
798
|
-
let mergedCursorMs = 0;
|
|
799
|
-
for (const part of wavParts) {
|
|
800
|
-
const actualDurationMs = this.#pcmDurationMs(part.wav.data.length, part.wav.sampleRate, part.wav.blockAlign);
|
|
801
|
-
const sessionStartMs = Math.max(0, Number(part.chunk.start_ms) || 0);
|
|
802
|
-
const sessionEndMs = Math.max(sessionStartMs, Number(part.chunk.end_ms) || (sessionStartMs + actualDurationMs));
|
|
803
|
-
spans.push({
|
|
804
|
-
mergedStartMs: mergedCursorMs,
|
|
805
|
-
mergedEndMs: mergedCursorMs + actualDurationMs,
|
|
806
|
-
sessionStartMs,
|
|
807
|
-
sessionEndMs,
|
|
808
|
-
});
|
|
809
|
-
mergedCursorMs += actualDurationMs;
|
|
810
|
-
}
|
|
811
|
-
|
|
812
|
-
const mergedAudioBytes = this.#buildWavBuffer({
|
|
813
|
-
audioFormat: reference.audioFormat,
|
|
814
|
-
channelCount: reference.channelCount,
|
|
815
|
-
sampleRate: reference.sampleRate,
|
|
816
|
-
bitsPerSample: reference.bitsPerSample,
|
|
817
|
-
data: Buffer.concat(wavParts.map((part) => part.wav.data)),
|
|
818
|
-
});
|
|
819
|
-
|
|
820
|
-
const payload = await transcribeChunkWithDeepgram({
|
|
821
|
-
audioBytes: mergedAudioBytes,
|
|
822
|
-
mimeType: chunks[0].mime_type || source.mime_type || 'audio/wav',
|
|
823
|
-
detectLanguage: DEFAULT_LANGUAGE,
|
|
824
|
-
model: transcriptionModel,
|
|
825
|
-
});
|
|
826
|
-
|
|
827
|
-
const lastSpan = spans[spans.length - 1];
|
|
828
|
-
return this.#extractSegmentsFromPayload(source, payload, {
|
|
829
|
-
mapMs: (mergedMs) => this.#mapMergedTimelineMs(mergedMs, spans),
|
|
830
|
-
defaultStartMs: spans[0]?.sessionStartMs || 0,
|
|
831
|
-
defaultEndMs: lastSpan?.sessionEndMs || spans[0]?.sessionStartMs || 0,
|
|
832
|
-
});
|
|
833
|
-
}
|
|
834
|
-
|
|
835
|
-
// Streaming recorders (browser MediaRecorder, native MPEG encoders) split a
|
|
836
|
-
// single media stream across many timeslice chunks where only the first
|
|
837
|
-
// chunk carries the container header (EBML for WebM/Ogg, ftyp/moov for MP4,
|
|
838
|
-
// frame sync for MPEG). The later chunks cannot be decoded on their own, so
|
|
839
|
-
// they MUST be concatenated back into one container before transcription.
|
|
840
|
-
// Sending them to Deepgram individually only ever decodes the first ~chunk.
|
|
841
|
-
#isConcatenableContainerMime(mime) {
|
|
842
|
-
const value = `${mime || ''}`.toLowerCase();
|
|
843
|
-
return (
|
|
844
|
-
value.includes('mpeg') ||
|
|
845
|
-
value.includes('mp3') ||
|
|
846
|
-
value.includes('webm') ||
|
|
847
|
-
value.includes('ogg') ||
|
|
848
|
-
value.includes('opus') ||
|
|
849
|
-
value.includes('mp4') ||
|
|
850
|
-
value.includes('m4a') ||
|
|
851
|
-
value.includes('aac')
|
|
852
|
-
);
|
|
853
|
-
}
|
|
854
|
-
|
|
855
|
-
#canTranscribeAsMergedBinary(source, chunks) {
|
|
856
|
-
if (!Array.isArray(chunks) || chunks.length === 0) {
|
|
857
|
-
return false;
|
|
858
|
-
}
|
|
859
|
-
|
|
860
|
-
if (!this.#isConcatenableContainerMime(source.mime_type)) {
|
|
861
|
-
return false;
|
|
862
|
-
}
|
|
863
|
-
|
|
864
|
-
return chunks.every((chunk) => {
|
|
865
|
-
const mime = `${chunk.mime_type || source.mime_type || ''}`.toLowerCase();
|
|
866
|
-
return this.#isConcatenableContainerMime(mime) || mime.includes('octet-stream');
|
|
867
|
-
});
|
|
868
|
-
}
|
|
869
|
-
|
|
870
|
-
async #transcribeMergedBinarySource(source, chunks, { transcriptionModel = DEFAULT_MODEL } = {}) {
|
|
871
|
-
const nonEmptyParts = chunks
|
|
872
|
-
.map((chunk) => ({
|
|
873
|
-
chunk,
|
|
874
|
-
audioBytes: fs.readFileSync(chunk.file_path),
|
|
875
|
-
}))
|
|
876
|
-
.filter((part) => Buffer.isBuffer(part.audioBytes) && part.audioBytes.length > 0);
|
|
877
|
-
|
|
878
|
-
if (nonEmptyParts.length === 0) {
|
|
879
|
-
return [];
|
|
880
|
-
}
|
|
881
|
-
|
|
882
|
-
const mergedAudioBytes = Buffer.concat(nonEmptyParts.map((part) => part.audioBytes));
|
|
883
|
-
const firstChunk = nonEmptyParts[0].chunk;
|
|
884
|
-
const lastChunk = nonEmptyParts[nonEmptyParts.length - 1].chunk;
|
|
885
|
-
const baseStartMs = Math.max(0, Number(firstChunk.start_ms) || 0);
|
|
886
|
-
const fallbackEndMs = Math.max(baseStartMs, Number(lastChunk.end_ms) || baseStartMs);
|
|
887
|
-
|
|
888
|
-
const payload = await transcribeChunkWithDeepgram({
|
|
889
|
-
audioBytes: mergedAudioBytes,
|
|
890
|
-
mimeType: firstChunk.mime_type || source.mime_type || 'audio/mpeg',
|
|
891
|
-
detectLanguage: DEFAULT_LANGUAGE,
|
|
892
|
-
model: transcriptionModel,
|
|
893
|
-
});
|
|
894
|
-
|
|
895
|
-
return this.#extractSegmentsFromPayload(source, payload, {
|
|
896
|
-
mapMs: (mergedMs) => baseStartMs + Math.max(0, Number(mergedMs) || 0),
|
|
897
|
-
defaultStartMs: baseStartMs,
|
|
898
|
-
defaultEndMs: fallbackEndMs,
|
|
899
|
-
});
|
|
900
|
-
}
|
|
901
|
-
|
|
902
|
-
#extractSegments(source, chunk, payload) {
|
|
903
|
-
const chunkStartMs = Number(chunk.start_ms) || 0;
|
|
904
|
-
const chunkEndMs = Number(chunk.end_ms) || chunkStartMs;
|
|
905
|
-
return this.#extractSegmentsFromPayload(source, payload, {
|
|
906
|
-
mapMs: (relativeMs) => chunkStartMs + Math.max(0, relativeMs),
|
|
907
|
-
defaultStartMs: chunkStartMs,
|
|
908
|
-
defaultEndMs: Math.max(chunkStartMs, chunkEndMs),
|
|
909
|
-
});
|
|
910
|
-
}
|
|
911
|
-
|
|
912
|
-
#extractSegmentsFromPayload(source, payload, { mapMs, defaultStartMs, defaultEndMs }) {
|
|
913
|
-
const results = payload?.results || {};
|
|
914
|
-
const channels = Array.isArray(results.channels) ? results.channels : [];
|
|
915
|
-
const alternative = channels[0]?.alternatives?.[0] || {};
|
|
916
|
-
const utterances = Array.isArray(results.utterances) ? results.utterances : [];
|
|
917
|
-
const words = Array.isArray(alternative.words) ? alternative.words : [];
|
|
918
|
-
|
|
919
|
-
if (utterances.length > 0) {
|
|
920
|
-
return utterances
|
|
921
|
-
.map((utterance, index) => {
|
|
922
|
-
const startMs = mapMs(Math.max(0, Math.round((Number(utterance.start) || 0) * 1000)));
|
|
923
|
-
const endMs = mapMs(Math.max(0, Math.round((Number(utterance.end) || 0) * 1000)));
|
|
924
|
-
return {
|
|
925
|
-
sourceId: source.id,
|
|
926
|
-
sourceKey: source.source_key,
|
|
927
|
-
speaker: source.source_kind,
|
|
928
|
-
text: `${utterance.transcript || ''}`.trim(),
|
|
929
|
-
startMs,
|
|
930
|
-
endMs: Math.max(startMs, endMs),
|
|
931
|
-
confidence: Number(utterance.confidence) || null,
|
|
932
|
-
words: this.#mapWordsTimeline(Array.isArray(utterance.words) ? utterance.words : [], mapMs),
|
|
933
|
-
index,
|
|
934
|
-
};
|
|
935
|
-
})
|
|
936
|
-
.filter((item) => item.text.length > 0);
|
|
937
|
-
}
|
|
938
|
-
|
|
939
|
-
const transcript = `${alternative.transcript || ''}`.trim();
|
|
940
|
-
if (!transcript) {
|
|
941
|
-
return [];
|
|
942
|
-
}
|
|
943
|
-
|
|
944
|
-
return [
|
|
945
|
-
{
|
|
946
|
-
sourceId: source.id,
|
|
947
|
-
sourceKey: source.source_key,
|
|
948
|
-
speaker: source.source_kind,
|
|
949
|
-
text: transcript,
|
|
950
|
-
startMs: defaultStartMs,
|
|
951
|
-
endMs: Math.max(defaultStartMs, defaultEndMs),
|
|
952
|
-
confidence: Number(alternative.confidence) || null,
|
|
953
|
-
words: this.#mapWordsTimeline(words, mapMs),
|
|
954
|
-
},
|
|
955
|
-
];
|
|
956
|
-
}
|
|
957
|
-
|
|
958
|
-
#canTranscribeAsSingleWav(source, chunks) {
|
|
959
|
-
if (!Array.isArray(chunks) || chunks.length === 0) {
|
|
960
|
-
return false;
|
|
961
|
-
}
|
|
962
|
-
const sourceMime = `${source.mime_type || ''}`.toLowerCase();
|
|
963
|
-
if (!/wav/.test(sourceMime)) {
|
|
964
|
-
return false;
|
|
965
|
-
}
|
|
966
|
-
return chunks.every((chunk) => /wav/.test(`${chunk.mime_type || source.mime_type || ''}`.toLowerCase()));
|
|
967
|
-
}
|
|
968
|
-
|
|
969
|
-
#parseWav(buffer) {
|
|
970
|
-
if (!Buffer.isBuffer(buffer) || buffer.length < 44) {
|
|
971
|
-
throw new Error('Invalid WAV chunk.');
|
|
972
|
-
}
|
|
973
|
-
if (buffer.toString('ascii', 0, 4) !== 'RIFF' || buffer.toString('ascii', 8, 12) !== 'WAVE') {
|
|
974
|
-
throw new Error('Unsupported WAV container.');
|
|
975
|
-
}
|
|
976
|
-
|
|
977
|
-
let cursor = 12;
|
|
978
|
-
let format = null;
|
|
979
|
-
let data = null;
|
|
980
|
-
|
|
981
|
-
while (cursor + 8 <= buffer.length) {
|
|
982
|
-
const chunkId = buffer.toString('ascii', cursor, cursor + 4);
|
|
983
|
-
const chunkSize = buffer.readUInt32LE(cursor + 4);
|
|
984
|
-
const chunkStart = cursor + 8;
|
|
985
|
-
const chunkEnd = Math.min(buffer.length, chunkStart + chunkSize);
|
|
986
|
-
|
|
987
|
-
if (chunkId === 'fmt ' && chunkSize >= 16) {
|
|
988
|
-
format = {
|
|
989
|
-
audioFormat: buffer.readUInt16LE(chunkStart),
|
|
990
|
-
channelCount: buffer.readUInt16LE(chunkStart + 2),
|
|
991
|
-
sampleRate: buffer.readUInt32LE(chunkStart + 4),
|
|
992
|
-
byteRate: buffer.readUInt32LE(chunkStart + 8),
|
|
993
|
-
blockAlign: buffer.readUInt16LE(chunkStart + 12),
|
|
994
|
-
bitsPerSample: buffer.readUInt16LE(chunkStart + 14),
|
|
995
|
-
};
|
|
996
|
-
} else if (chunkId === 'data') {
|
|
997
|
-
data = buffer.subarray(chunkStart, chunkEnd);
|
|
998
|
-
}
|
|
999
|
-
|
|
1000
|
-
cursor = chunkStart + chunkSize + (chunkSize % 2);
|
|
1001
|
-
}
|
|
1002
|
-
|
|
1003
|
-
if (!format || !data) {
|
|
1004
|
-
throw new Error('WAV chunk is missing format or audio data.');
|
|
1005
|
-
}
|
|
1006
|
-
|
|
1007
|
-
return {
|
|
1008
|
-
...format,
|
|
1009
|
-
data,
|
|
1010
|
-
};
|
|
1011
|
-
}
|
|
1012
|
-
|
|
1013
|
-
#buildWavBuffer({ audioFormat, channelCount, sampleRate, bitsPerSample, data }) {
|
|
1014
|
-
const blockAlign = channelCount * (bitsPerSample / 8);
|
|
1015
|
-
const byteRate = sampleRate * blockAlign;
|
|
1016
|
-
const header = Buffer.alloc(44);
|
|
1017
|
-
header.write('RIFF', 0, 4, 'ascii');
|
|
1018
|
-
header.writeUInt32LE(36 + data.length, 4);
|
|
1019
|
-
header.write('WAVE', 8, 4, 'ascii');
|
|
1020
|
-
header.write('fmt ', 12, 4, 'ascii');
|
|
1021
|
-
header.writeUInt32LE(16, 16);
|
|
1022
|
-
header.writeUInt16LE(audioFormat, 20);
|
|
1023
|
-
header.writeUInt16LE(channelCount, 22);
|
|
1024
|
-
header.writeUInt32LE(sampleRate, 24);
|
|
1025
|
-
header.writeUInt32LE(byteRate, 28);
|
|
1026
|
-
header.writeUInt16LE(blockAlign, 32);
|
|
1027
|
-
header.writeUInt16LE(bitsPerSample, 34);
|
|
1028
|
-
header.write('data', 36, 4, 'ascii');
|
|
1029
|
-
header.writeUInt32LE(data.length, 40);
|
|
1030
|
-
return Buffer.concat([header, data]);
|
|
1031
|
-
}
|
|
1032
|
-
|
|
1033
|
-
#pcmDurationMs(byteLength, sampleRate, blockAlign) {
|
|
1034
|
-
const safeSampleRate = Number(sampleRate) || 0;
|
|
1035
|
-
const safeBlockAlign = Number(blockAlign) || 0;
|
|
1036
|
-
if (safeSampleRate <= 0 || safeBlockAlign <= 0) {
|
|
1037
|
-
return 0;
|
|
1038
|
-
}
|
|
1039
|
-
return Math.round((byteLength / safeBlockAlign) * 1000 / safeSampleRate);
|
|
1040
|
-
}
|
|
1041
|
-
|
|
1042
|
-
#mapMergedTimelineMs(mergedMs, spans) {
|
|
1043
|
-
const clampedMs = Math.max(0, Number(mergedMs) || 0);
|
|
1044
|
-
for (const span of spans) {
|
|
1045
|
-
if (clampedMs <= span.mergedEndMs || span === spans[spans.length - 1]) {
|
|
1046
|
-
const mergedDuration = Math.max(1, span.mergedEndMs - span.mergedStartMs);
|
|
1047
|
-
const sessionDuration = Math.max(0, span.sessionEndMs - span.sessionStartMs);
|
|
1048
|
-
const relative = Math.max(0, clampedMs - span.mergedStartMs);
|
|
1049
|
-
if (sessionDuration === 0) {
|
|
1050
|
-
return span.sessionStartMs;
|
|
1051
|
-
}
|
|
1052
|
-
return span.sessionStartMs + Math.round((relative / mergedDuration) * sessionDuration);
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
return 0;
|
|
1056
|
-
}
|
|
1057
|
-
|
|
1058
|
-
#mapWordsTimeline(words, mapMs) {
|
|
1059
|
-
return (Array.isArray(words) ? words : []).map((word) => {
|
|
1060
|
-
const start = Math.max(0, Math.round((Number(word?.start) || 0) * 1000));
|
|
1061
|
-
const end = Math.max(start, Math.round((Number(word?.end) || 0) * 1000));
|
|
1062
|
-
return {
|
|
1063
|
-
...word,
|
|
1064
|
-
start: mapMs(start) / 1000,
|
|
1065
|
-
end: mapMs(end) / 1000,
|
|
1066
|
-
};
|
|
1067
|
-
});
|
|
1068
|
-
}
|
|
1069
|
-
|
|
1070
|
-
#warnOnChunkGaps(sourceKey, chunks) {
|
|
1071
|
-
const missing = [];
|
|
1072
|
-
let expected = 0;
|
|
1073
|
-
for (const chunk of chunks) {
|
|
1074
|
-
const sequence = Number(chunk.sequence_index) || 0;
|
|
1075
|
-
while (expected < sequence) {
|
|
1076
|
-
missing.push(expected);
|
|
1077
|
-
expected += 1;
|
|
1078
|
-
}
|
|
1079
|
-
expected = sequence + 1;
|
|
1080
|
-
}
|
|
1081
|
-
if (missing.length > 0) {
|
|
1082
|
-
console.warn(
|
|
1083
|
-
`[Recordings] Source "${sourceKey}" is missing chunk(s) ${missing.slice(0, 20).join(', ')}` +
|
|
1084
|
-
`${missing.length > 20 ? ', …' : ''}; transcribing the audio that did arrive.`,
|
|
1085
|
-
);
|
|
1086
|
-
}
|
|
1087
|
-
}
|
|
1088
|
-
|
|
1089
|
-
#maxOf(values, seed = 0) {
|
|
1090
|
-
let max = Number(seed) || 0;
|
|
1091
|
-
for (const value of (Array.isArray(values) ? values : [])) {
|
|
1092
|
-
const num = Number(value) || 0;
|
|
1093
|
-
if (num > max) {
|
|
1094
|
-
max = num;
|
|
1095
|
-
}
|
|
1096
|
-
}
|
|
1097
|
-
return max;
|
|
1098
|
-
}
|
|
1099
|
-
|
|
1100
|
-
#shouldTranscribeSource(source, metadata = {}) {
|
|
1101
|
-
if (metadata.transcribe === false) {
|
|
1102
|
-
return false;
|
|
1103
|
-
}
|
|
1104
|
-
if (metadata.transcribe === true) {
|
|
1105
|
-
return true;
|
|
1106
|
-
}
|
|
1107
|
-
return `${source.media_kind || ''}`.trim().toLowerCase() === 'audio';
|
|
1108
|
-
}
|
|
1109
|
-
|
|
1110
|
-
#resolveTitle(title, platform, nowIso) {
|
|
1111
|
-
if (typeof title === 'string' && title.trim()) {
|
|
1112
|
-
return title.trim().slice(0, 160);
|
|
1113
|
-
}
|
|
1114
|
-
const stamp = new Date(nowIso).toISOString().replace('T', ' ').slice(0, 16);
|
|
1115
|
-
if (platform === 'web') {
|
|
1116
|
-
return `Screen + mic recording ${stamp}`;
|
|
1117
|
-
}
|
|
1118
|
-
if (platform === 'android') {
|
|
1119
|
-
return `Background microphone recording ${stamp}`;
|
|
1120
|
-
}
|
|
1121
|
-
return `Recording ${stamp}`;
|
|
1122
|
-
}
|
|
1123
|
-
|
|
1124
|
-
#normalizeSources(rawSources, platform) {
|
|
1125
|
-
const fallback = platform === 'android'
|
|
1126
|
-
? [
|
|
1127
|
-
{
|
|
1128
|
-
sourceKey: 'microphone',
|
|
1129
|
-
sourceKind: 'microphone',
|
|
1130
|
-
mediaKind: 'audio',
|
|
1131
|
-
mimeType: 'audio/wav',
|
|
1132
|
-
metadata: { backgroundCapable: true },
|
|
1133
|
-
},
|
|
1134
|
-
]
|
|
1135
|
-
: [
|
|
1136
|
-
{
|
|
1137
|
-
sourceKey: 'screen',
|
|
1138
|
-
sourceKind: 'screen-share',
|
|
1139
|
-
mediaKind: 'video',
|
|
1140
|
-
mimeType: 'video/webm',
|
|
1141
|
-
metadata: { analysisReady: true, transcribe: false },
|
|
1142
|
-
},
|
|
1143
|
-
{
|
|
1144
|
-
sourceKey: 'microphone',
|
|
1145
|
-
sourceKind: 'microphone',
|
|
1146
|
-
mediaKind: 'audio',
|
|
1147
|
-
mimeType: 'audio/webm',
|
|
1148
|
-
metadata: {},
|
|
1149
|
-
},
|
|
1150
|
-
];
|
|
1151
|
-
|
|
1152
|
-
const inputs = Array.isArray(rawSources) && rawSources.length > 0 ? rawSources : fallback;
|
|
1153
|
-
const seen = new Set();
|
|
1154
|
-
|
|
1155
|
-
return inputs.map((item, index) => {
|
|
1156
|
-
const sourceKey = `${item?.sourceKey || item?.id || `source-${index}`}`.trim().toLowerCase();
|
|
1157
|
-
validateSourceKey(sourceKey);
|
|
1158
|
-
if (seen.has(sourceKey)) {
|
|
1159
|
-
throw new Error(`Duplicate recording source: ${sourceKey}`);
|
|
1160
|
-
}
|
|
1161
|
-
seen.add(sourceKey);
|
|
1162
|
-
return {
|
|
1163
|
-
sourceKey,
|
|
1164
|
-
sourceKind: `${item?.sourceKind || sourceKey}`.trim().toLowerCase(),
|
|
1165
|
-
mediaKind: `${item?.mediaKind || 'audio'}`.trim().toLowerCase(),
|
|
1166
|
-
mimeType: `${item?.mimeType || 'application/octet-stream'}`.trim().toLowerCase(),
|
|
1167
|
-
metadata: item?.metadata && typeof item.metadata === 'object'
|
|
1168
|
-
? item.metadata
|
|
1169
|
-
: {},
|
|
1170
|
-
};
|
|
1171
|
-
});
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
#mapSession(row) {
|
|
1175
|
-
const metadata = this.#parseJson(row.metadata_json, {});
|
|
1176
|
-
return {
|
|
1177
|
-
id: row.id,
|
|
1178
|
-
title: row.title || 'Recording',
|
|
1179
|
-
platform: row.platform || 'unknown',
|
|
1180
|
-
status: row.status || SESSION_STATUS.recording,
|
|
1181
|
-
transcriptText: row.transcript_text || '',
|
|
1182
|
-
transcriptLanguage: row.transcript_language || DEFAULT_LANGUAGE,
|
|
1183
|
-
transcriptModel: row.transcript_model || DEFAULT_MODEL,
|
|
1184
|
-
startedAt: row.started_at,
|
|
1185
|
-
endedAt: row.ended_at,
|
|
1186
|
-
durationMs: Number(row.duration_ms) || 0,
|
|
1187
|
-
lastError: row.last_error,
|
|
1188
|
-
structuredContent: this.#parseJson(row.structured_content_json, null),
|
|
1189
|
-
metadata,
|
|
1190
|
-
sourceCount: Number(
|
|
1191
|
-
db.prepare('SELECT COUNT(*) AS count FROM recording_sources WHERE session_id = ?').get(row.id).count,
|
|
1192
|
-
) || 0,
|
|
1193
|
-
};
|
|
1194
|
-
}
|
|
1195
|
-
|
|
1196
|
-
#mapSource(row) {
|
|
1197
|
-
return {
|
|
1198
|
-
id: row.id,
|
|
1199
|
-
sourceKey: row.source_key,
|
|
1200
|
-
sourceKind: row.source_kind,
|
|
1201
|
-
mediaKind: row.media_kind,
|
|
1202
|
-
mimeType: row.mime_type,
|
|
1203
|
-
status: row.status,
|
|
1204
|
-
chunkCount: Number(row.chunk_count) || 0,
|
|
1205
|
-
bytesReceived: Number(row.bytes_received) || 0,
|
|
1206
|
-
durationMs: Number(row.duration_ms) || 0,
|
|
1207
|
-
metadata: this.#parseJson(row.metadata_json, {}),
|
|
1208
|
-
};
|
|
1209
|
-
}
|
|
1210
|
-
|
|
1211
|
-
#mapSegment(row) {
|
|
1212
|
-
return {
|
|
1213
|
-
id: Number(row.id),
|
|
1214
|
-
sourceKey: row.source_key,
|
|
1215
|
-
speaker: row.speaker || row.source_key || 'source',
|
|
1216
|
-
text: row.text || '',
|
|
1217
|
-
startMs: Number(row.start_ms) || 0,
|
|
1218
|
-
endMs: Number(row.end_ms) || 0,
|
|
1219
|
-
confidence: row.confidence == null ? null : Number(row.confidence),
|
|
1220
|
-
words: this.#parseJson(row.words_json, []),
|
|
1221
|
-
};
|
|
1222
|
-
}
|
|
1223
|
-
|
|
1224
|
-
#composeTranscriptText(segments) {
|
|
1225
|
-
return (Array.isArray(segments) ? segments : [])
|
|
1226
|
-
.map((segment) => {
|
|
1227
|
-
const startMs = Number(segment.startMs ?? segment.start_ms) || 0;
|
|
1228
|
-
const text = `${segment.text || ''}`.trim();
|
|
1229
|
-
if (!text) {
|
|
1230
|
-
return null;
|
|
1231
|
-
}
|
|
1232
|
-
return `[${this.#formatTimestamp(startMs)}] ${text}`;
|
|
1233
|
-
})
|
|
1234
|
-
.filter((line) => line != null)
|
|
1235
|
-
.join('\n');
|
|
1236
|
-
}
|
|
1237
|
-
|
|
1238
|
-
#emitUpdate(userId, sessionId) {
|
|
1239
|
-
this.io?.to?.(`user:${userId}`)?.emit('recordings:updated', { sessionId });
|
|
1240
|
-
}
|
|
1241
|
-
|
|
1242
|
-
#extensionForMime(mimeType) {
|
|
1243
|
-
if (/wav/i.test(mimeType)) return '.wav';
|
|
1244
|
-
if (/webm/i.test(mimeType)) return '.webm';
|
|
1245
|
-
if (/mp4|m4a/i.test(mimeType)) return '.m4a';
|
|
1246
|
-
if (/ogg|opus/i.test(mimeType)) return '.ogg';
|
|
1247
|
-
return '.bin';
|
|
1248
|
-
}
|
|
1249
|
-
|
|
1250
|
-
#formatTimestamp(ms) {
|
|
1251
|
-
const totalSeconds = Math.max(0, Math.floor((Number(ms) || 0) / 1000));
|
|
1252
|
-
const hours = Math.floor(totalSeconds / 3600);
|
|
1253
|
-
const minutes = Math.floor((totalSeconds % 3600) / 60);
|
|
1254
|
-
const seconds = totalSeconds % 60;
|
|
1255
|
-
if (hours > 0) {
|
|
1256
|
-
return `${String(hours).padStart(2, '0')}:${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}`;
|
|
1257
|
-
}
|
|
1258
|
-
return `${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}`;
|
|
1259
|
-
}
|
|
1260
|
-
|
|
1261
|
-
#parseJson(value, fallback) {
|
|
1262
|
-
try {
|
|
1263
|
-
return value ? JSON.parse(value) : fallback;
|
|
1264
|
-
} catch {
|
|
1265
|
-
return fallback;
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1268
|
-
}
|
|
1269
|
-
|
|
1270
|
-
module.exports = {
|
|
1271
|
-
RecordingManager,
|
|
1272
|
-
SESSION_STATUS,
|
|
1273
|
-
};
|