neoagent 3.0.1-beta.8 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -10
- package/README.md +12 -3
- package/docs/automation.md +37 -0
- package/docs/benchmarking.md +102 -0
- package/docs/billing.md +34 -8
- package/docs/configuration.md +11 -2
- package/docs/getting-started.md +10 -8
- package/docs/operations.md +1 -1
- package/flutter_app/lib/main.dart +3 -0
- package/flutter_app/lib/main_account_settings.dart +138 -0
- package/flutter_app/lib/main_app_shell.dart +38 -13
- package/flutter_app/lib/main_billing.dart +1465 -0
- package/flutter_app/lib/main_chat.dart +1612 -214
- package/flutter_app/lib/main_controller.dart +398 -26
- package/flutter_app/lib/main_devices.dart +293 -207
- package/flutter_app/lib/main_models.dart +142 -0
- package/flutter_app/lib/main_navigation.dart +19 -1
- package/flutter_app/lib/main_operations.dart +288 -9
- package/flutter_app/lib/main_settings.dart +510 -300
- package/flutter_app/lib/main_shared.dart +2 -0
- package/flutter_app/lib/main_timeline.dart +1378 -0
- package/flutter_app/lib/src/backend_client.dart +149 -19
- package/flutter_app/lib/src/desktop_companion_actions.dart +132 -21
- package/flutter_app/lib/src/desktop_companion_io.dart +65 -1
- package/flutter_app/lib/src/desktop_companion_stub.dart +12 -0
- package/flutter_app/lib/src/desktop_ocr_bridge.dart +2 -0
- package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +125 -0
- package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +30 -0
- package/flutter_app/lib/src/desktop_passive_history.dart +332 -0
- package/flutter_app/lib/src/recording_bridge_io.dart +80 -72
- package/flutter_app/lib/src/recording_bridge_web.dart +127 -114
- package/flutter_app/lib/src/recording_chunk_queue.dart +149 -0
- package/flutter_app/lib/src/recording_chunk_queue_io.dart +182 -0
- package/flutter_app/lib/src/recording_payloads.dart +9 -0
- package/flutter_app/macos/Runner/AppDelegate.swift +25 -0
- package/flutter_app/windows/runner/flutter_window.cpp +75 -0
- package/landing/index.html +3 -3
- package/lib/manager.js +184 -66
- package/package.json +4 -1
- package/server/admin/access.js +12 -7
- package/server/admin/admin.css +78 -0
- package/server/admin/admin.js +436 -10
- package/server/admin/billing.js +165 -42
- package/server/admin/index.html +80 -5
- package/server/admin/users.js +15 -15
- package/server/db/database.js +125 -20
- package/server/http/routes.js +1 -0
- package/server/http/static.js +4 -2
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/canvaskit/wimp.js.symbols +8475 -8467
- package/server/public/canvaskit/wimp.wasm +0 -0
- package/server/public/flutter_bootstrap.js +2 -2
- package/server/public/main.dart.js +91077 -87037
- package/server/routes/account.js +53 -0
- package/server/routes/admin.js +345 -64
- package/server/routes/agents.js +203 -21
- package/server/routes/billing.js +5 -0
- package/server/routes/browser.js +8 -1
- package/server/routes/recordings.js +96 -6
- package/server/routes/screenHistory.js +140 -2
- package/server/routes/timeline.js +43 -0
- package/server/services/account/erasure.js +263 -0
- package/server/services/ai/hooks.js +4 -1
- package/server/services/ai/loop/agent_engine_core.js +8 -1
- package/server/services/ai/loop/blank_recovery.js +36 -0
- package/server/services/ai/loop/conversation_loop.js +166 -33
- package/server/services/ai/messagingFallback.js +22 -0
- package/server/services/ai/rate_limits.js +28 -5
- package/server/services/ai/systemPrompt.js +6 -5
- package/server/services/ai/taskAnalysis.js +2 -0
- package/server/services/ai/toolEvidence.js +15 -0
- package/server/services/ai/toolResult.js +40 -0
- package/server/services/ai/tools.js +163 -4
- package/server/services/android/controller.js +6 -2
- package/server/services/billing/plans.js +2 -1
- package/server/services/browser/anti_detection.js +192 -0
- package/server/services/browser/controller.js +180 -54
- package/server/services/desktop/auth.js +3 -0
- package/server/services/desktop/registry.js +50 -2
- package/server/services/integrations/google/calendar.js +22 -14
- package/server/services/manager.js +12 -42
- package/server/services/memory/ingestion_chunking.js +268 -0
- package/server/services/messaging/telnyx.js +9 -8
- package/server/services/recordings/manager.js +60 -27
- package/server/services/runtime/backends/local-vm.js +40 -22
- package/server/services/runtime/docker-vm-manager.js +157 -266
- package/server/services/runtime/guest_bootstrap.js +17 -5
- package/server/services/runtime/guest_image.js +188 -0
- package/server/services/runtime/manager.js +0 -1
- package/server/services/runtime/validation.js +3 -8
- package/server/services/social_video/service.js +60 -10
- package/server/services/tasks/runtime.js +234 -9
- package/server/services/tasks/schedule_utils.js +5 -5
- package/server/services/tasks/task_repository.js +13 -0
- package/server/services/timeline/service.js +558 -0
- package/server/services/wearable/gateway.js +1 -1
- package/server/services/websocket.js +21 -3
- package/server/services/desktop/screenRecorder.js +0 -292
- package/server/services/desktop/screen_recorder_support.js +0 -46
|
@@ -32,10 +32,7 @@ const { registerToolSecurityHooks } = require('./security/tool_security_hook');
|
|
|
32
32
|
const { BrowserExtensionRegistry } = require('./browser/extension/registry');
|
|
33
33
|
const { DesktopCompanionRegistry } = require('./desktop/registry');
|
|
34
34
|
const { DesktopProvider } = require('./desktop/provider');
|
|
35
|
-
const {
|
|
36
|
-
ScreenRecorder,
|
|
37
|
-
hasOpenConnectionForUser,
|
|
38
|
-
} = require('./desktop/screenRecorder');
|
|
35
|
+
const { TimelineService } = require('./timeline/service');
|
|
39
36
|
const { WearableService } = require('./wearable/service');
|
|
40
37
|
const { getRuntimeValidation } = require('./runtime/validation');
|
|
41
38
|
const {
|
|
@@ -96,6 +93,16 @@ function createDesktopCompanionRegistry(app) {
|
|
|
96
93
|
return registry;
|
|
97
94
|
}
|
|
98
95
|
|
|
96
|
+
function createTimelineService(app, io) {
|
|
97
|
+
const timelineService = registerLocal(
|
|
98
|
+
app,
|
|
99
|
+
'timelineService',
|
|
100
|
+
new TimelineService({ io }),
|
|
101
|
+
);
|
|
102
|
+
logServiceReady('Timeline service ready');
|
|
103
|
+
return timelineService;
|
|
104
|
+
}
|
|
105
|
+
|
|
99
106
|
function createMemoryManager(app) {
|
|
100
107
|
const memoryManager = registerLocal(app, 'memoryManager', new MemoryManager());
|
|
101
108
|
memoryManager.startEmbeddingIndexBackfill();
|
|
@@ -406,30 +413,6 @@ function createWearableService(app) {
|
|
|
406
413
|
return wearableService;
|
|
407
414
|
}
|
|
408
415
|
|
|
409
|
-
function createScreenRecorder(app) {
|
|
410
|
-
const hasActiveCaptureSessionForUser = (userId) => {
|
|
411
|
-
return (
|
|
412
|
-
hasOpenConnectionForUser(app.locals.desktopCompanionRegistry, userId)
|
|
413
|
-
|| hasOpenConnectionForUser(app.locals.browserExtensionRegistry, userId)
|
|
414
|
-
);
|
|
415
|
-
};
|
|
416
|
-
|
|
417
|
-
const screenRecorder = registerLocal(
|
|
418
|
-
app,
|
|
419
|
-
'screenRecorder',
|
|
420
|
-
new ScreenRecorder({ hasActiveCaptureSessionForUser }),
|
|
421
|
-
);
|
|
422
|
-
const status = screenRecorder.start();
|
|
423
|
-
if (status.state === 'running') {
|
|
424
|
-
logServiceReady(
|
|
425
|
-
`Screen recorder started for user ${status.ownerUserId} (${status.intervalMs}ms interval)`,
|
|
426
|
-
);
|
|
427
|
-
} else {
|
|
428
|
-
logServiceReady(`Screen recorder ${status.state}: ${status.reason}`);
|
|
429
|
-
}
|
|
430
|
-
return screenRecorder;
|
|
431
|
-
}
|
|
432
|
-
|
|
433
416
|
function restoreMessagingConnections(messagingManager) {
|
|
434
417
|
void runBackgroundTask('[Messaging] Restore error:', () =>
|
|
435
418
|
messagingManager.restoreConnections(),
|
|
@@ -486,6 +469,7 @@ async function startServices(app, io) {
|
|
|
486
469
|
createWorkspaceManager(app);
|
|
487
470
|
createBrowserExtensionRegistry(app);
|
|
488
471
|
createDesktopCompanionRegistry(app);
|
|
472
|
+
createTimelineService(app, io);
|
|
489
473
|
const memoryManager = createMemoryManager(app);
|
|
490
474
|
const mcpClient = createMcpClient(app);
|
|
491
475
|
createAuthProviderManager(app);
|
|
@@ -531,7 +515,6 @@ async function startServices(app, io) {
|
|
|
531
515
|
createSocialVideoService(app);
|
|
532
516
|
createWidgetService(app);
|
|
533
517
|
createWearableService(app);
|
|
534
|
-
createScreenRecorder(app);
|
|
535
518
|
|
|
536
519
|
restoreMessagingConnections(messagingManager);
|
|
537
520
|
restoreMcpClients(mcpClient);
|
|
@@ -710,19 +693,6 @@ async function stopServices(app) {
|
|
|
710
693
|
}
|
|
711
694
|
}
|
|
712
695
|
|
|
713
|
-
if (app.locals.screenRecorder) {
|
|
714
|
-
tasks.push(
|
|
715
|
-
Promise.resolve()
|
|
716
|
-
.then(() => app.locals.screenRecorder.stop())
|
|
717
|
-
.then((status) => {
|
|
718
|
-
logServiceReady(`Screen recorder shutdown complete (${status.state})`);
|
|
719
|
-
})
|
|
720
|
-
.catch((err) => {
|
|
721
|
-
console.error('[ScreenRecorder] Stop error:', getErrorMessage(err));
|
|
722
|
-
}),
|
|
723
|
-
);
|
|
724
|
-
}
|
|
725
|
-
|
|
726
696
|
if (app.locals.browserExtensionRegistry) {
|
|
727
697
|
try {
|
|
728
698
|
app.locals.browserExtensionRegistry.closeAll();
|
|
@@ -4,6 +4,47 @@ const crypto = require('node:crypto');
|
|
|
4
4
|
|
|
5
5
|
const TARGET_CHARS = 1400;
|
|
6
6
|
const MAX_CHARS = 2200;
|
|
7
|
+
const TARGET_SEGMENT_WORDS = 260;
|
|
8
|
+
|
|
9
|
+
const SEGMENT_TYPES = new Set([
|
|
10
|
+
'Caption',
|
|
11
|
+
'Footnote',
|
|
12
|
+
'Formula',
|
|
13
|
+
'ListItem',
|
|
14
|
+
'Page',
|
|
15
|
+
'PageFooter',
|
|
16
|
+
'PageHeader',
|
|
17
|
+
'Picture',
|
|
18
|
+
'SectionHeader',
|
|
19
|
+
'Table',
|
|
20
|
+
'Text',
|
|
21
|
+
'Title',
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
const SEGMENT_ALIASES = Object.freeze({
|
|
25
|
+
caption: 'Caption',
|
|
26
|
+
footnote: 'Footnote',
|
|
27
|
+
formula: 'Formula',
|
|
28
|
+
listitem: 'ListItem',
|
|
29
|
+
list_item: 'ListItem',
|
|
30
|
+
'list item': 'ListItem',
|
|
31
|
+
page: 'Page',
|
|
32
|
+
pagefooter: 'PageFooter',
|
|
33
|
+
page_footer: 'PageFooter',
|
|
34
|
+
'page footer': 'PageFooter',
|
|
35
|
+
pageheader: 'PageHeader',
|
|
36
|
+
page_header: 'PageHeader',
|
|
37
|
+
'page header': 'PageHeader',
|
|
38
|
+
picture: 'Picture',
|
|
39
|
+
image: 'Picture',
|
|
40
|
+
figure: 'Picture',
|
|
41
|
+
sectionheader: 'SectionHeader',
|
|
42
|
+
section_header: 'SectionHeader',
|
|
43
|
+
'section header': 'SectionHeader',
|
|
44
|
+
table: 'Table',
|
|
45
|
+
text: 'Text',
|
|
46
|
+
title: 'Title',
|
|
47
|
+
});
|
|
7
48
|
|
|
8
49
|
function contentHash(content) {
|
|
9
50
|
return crypto.createHash('sha256').update(content).digest('hex');
|
|
@@ -107,10 +148,237 @@ function packBlocksWithPositions(blocksWithPos) {
|
|
|
107
148
|
}));
|
|
108
149
|
}
|
|
109
150
|
|
|
151
|
+
function wordCount(text) {
|
|
152
|
+
return (String(text || '').trim().match(/\S+/g) || []).length;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function normalizeSegmentType(value) {
|
|
156
|
+
const raw = String(value || '').trim();
|
|
157
|
+
if (SEGMENT_TYPES.has(raw)) return raw;
|
|
158
|
+
return SEGMENT_ALIASES[raw.toLowerCase()] || 'Text';
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function normalizeNumber(value) {
|
|
162
|
+
const number = Number(value);
|
|
163
|
+
return Number.isFinite(number) ? number : null;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function normalizeBoundingBox(value) {
|
|
167
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
|
|
168
|
+
const left = normalizeNumber(value.left ?? value.x ?? value.x1);
|
|
169
|
+
const top = normalizeNumber(value.top ?? value.y ?? value.y1);
|
|
170
|
+
const width = normalizeNumber(value.width ?? (
|
|
171
|
+
value.x2 != null && left != null ? Number(value.x2) - left : null
|
|
172
|
+
));
|
|
173
|
+
const height = normalizeNumber(value.height ?? (
|
|
174
|
+
value.y2 != null && top != null ? Number(value.y2) - top : null
|
|
175
|
+
));
|
|
176
|
+
if ([left, top, width, height].some((number) => number == null)) return null;
|
|
177
|
+
return { left, top, width, height };
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function segmentEmbedContent(segment) {
|
|
181
|
+
return [
|
|
182
|
+
segment.llm,
|
|
183
|
+
segment.markdown,
|
|
184
|
+
segment.html,
|
|
185
|
+
segment.text,
|
|
186
|
+
segment.content,
|
|
187
|
+
].map((value) => String(value || '').trim()).find(Boolean) || '';
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function normalizeLayoutSegments(document) {
|
|
191
|
+
const segments = Array.isArray(document?.payload?.segments)
|
|
192
|
+
? document.payload.segments
|
|
193
|
+
: [];
|
|
194
|
+
const normalized = [];
|
|
195
|
+
for (const segment of segments) {
|
|
196
|
+
if (!segment || typeof segment !== 'object' || Array.isArray(segment)) continue;
|
|
197
|
+
const content = segmentEmbedContent(segment);
|
|
198
|
+
if (!content) continue;
|
|
199
|
+
const segmentType = normalizeSegmentType(segment.type || segment.segmentType || segment.segment_type);
|
|
200
|
+
normalized.push({
|
|
201
|
+
id: String(segment.id || segment.segmentId || segment.segment_id || '').trim() || null,
|
|
202
|
+
type: segmentType,
|
|
203
|
+
content,
|
|
204
|
+
wordCount: wordCount(content),
|
|
205
|
+
pageNumber: normalizeNumber(segment.pageNumber ?? segment.page_number),
|
|
206
|
+
pageWidth: normalizeNumber(segment.pageWidth ?? segment.page_width),
|
|
207
|
+
pageHeight: normalizeNumber(segment.pageHeight ?? segment.page_height),
|
|
208
|
+
bbox: normalizeBoundingBox(segment.bbox || segment.boundingBox || segment.bounding_box),
|
|
209
|
+
confidence: normalizeNumber(segment.confidence),
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
return normalized;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function segmentHierarchyLevel(type) {
|
|
216
|
+
if (type === 'Title') return 3;
|
|
217
|
+
if (type === 'SectionHeader') return 2;
|
|
218
|
+
return 1;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function isAssetType(type) {
|
|
222
|
+
return type === 'Picture' || type === 'Table';
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function hasAdjacentAssetCaptionPair(segments) {
|
|
226
|
+
return segments.some((segment, index) => {
|
|
227
|
+
const nextType = segments[index + 1]?.type;
|
|
228
|
+
return (isAssetType(segment.type) && nextType === 'Caption')
|
|
229
|
+
|| (segment.type === 'Caption' && isAssetType(nextType));
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function findSegmentPosition(content, rendered, searchOffset) {
|
|
234
|
+
const pos = content.indexOf(rendered, searchOffset);
|
|
235
|
+
if (pos >= 0) return { charStart: pos, charEnd: pos + rendered.length };
|
|
236
|
+
|
|
237
|
+
const compactRendered = String(rendered || '').replace(/\s+/g, ' ').trim();
|
|
238
|
+
if (compactRendered && compactRendered.length <= 160) {
|
|
239
|
+
const compactContent = content.replace(/\s+/g, ' ');
|
|
240
|
+
const compactPos = compactContent.indexOf(compactRendered);
|
|
241
|
+
if (compactPos >= 0) {
|
|
242
|
+
return {
|
|
243
|
+
charStart: Math.min(compactPos, content.length),
|
|
244
|
+
charEnd: Math.min(compactPos + compactRendered.length, content.length),
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
return {
|
|
250
|
+
charStart: Math.min(searchOffset, content.length),
|
|
251
|
+
charEnd: Math.min(searchOffset + rendered.length, content.length),
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function metadataForSegments(segments, boundary = 'layout_segments') {
|
|
256
|
+
const pageNumbers = [...new Set(
|
|
257
|
+
segments.map((segment) => segment.pageNumber).filter((number) => number != null),
|
|
258
|
+
)];
|
|
259
|
+
const segmentTypes = [...new Set(segments.map((segment) => segment.type))];
|
|
260
|
+
const resolvedBoundary = boundary === 'layout_segments' && hasAdjacentAssetCaptionPair(segments)
|
|
261
|
+
? 'asset_caption_pair'
|
|
262
|
+
: boundary;
|
|
263
|
+
return {
|
|
264
|
+
boundary: resolvedBoundary,
|
|
265
|
+
segmentTypes,
|
|
266
|
+
segmentIds: segments.map((segment) => segment.id).filter(Boolean),
|
|
267
|
+
segments: segments.map((segment) => ({
|
|
268
|
+
id: segment.id,
|
|
269
|
+
type: segment.type,
|
|
270
|
+
pageNumber: segment.pageNumber,
|
|
271
|
+
pageWidth: segment.pageWidth,
|
|
272
|
+
pageHeight: segment.pageHeight,
|
|
273
|
+
confidence: segment.confidence,
|
|
274
|
+
})),
|
|
275
|
+
pageNumbers,
|
|
276
|
+
pages: pageNumbers,
|
|
277
|
+
bboxes: segments
|
|
278
|
+
.map((segment) => segment.bbox && {
|
|
279
|
+
segmentId: segment.id,
|
|
280
|
+
type: segment.type,
|
|
281
|
+
pageNumber: segment.pageNumber,
|
|
282
|
+
bbox: segment.bbox,
|
|
283
|
+
})
|
|
284
|
+
.filter(Boolean),
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function finalizeSegmentChunk(chunks, group, content, searchState, boundary = 'layout_segments') {
|
|
289
|
+
if (!group.length) return;
|
|
290
|
+
const rendered = group.map((segment) => segment.content).join('\n\n');
|
|
291
|
+
const position = findSegmentPosition(content, rendered, searchState.offset);
|
|
292
|
+
searchState.offset = Math.max(searchState.offset, position.charEnd);
|
|
293
|
+
const metadata = metadataForSegments(group, boundary);
|
|
294
|
+
for (const part of splitOversizedWithPos({
|
|
295
|
+
content: rendered,
|
|
296
|
+
charStart: position.charStart,
|
|
297
|
+
})) {
|
|
298
|
+
chunks.push({
|
|
299
|
+
content: part.content,
|
|
300
|
+
charStart: part.charStart,
|
|
301
|
+
charEnd: Math.min(part.charEnd, position.charEnd),
|
|
302
|
+
metadata,
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function chunkLayoutSegments(document, content) {
|
|
308
|
+
const segments = normalizeLayoutSegments(document);
|
|
309
|
+
if (!segments.length) return null;
|
|
310
|
+
|
|
311
|
+
const chunks = [];
|
|
312
|
+
let group = [];
|
|
313
|
+
let groupWordCount = 0;
|
|
314
|
+
let previousHierarchyLevel = 1;
|
|
315
|
+
const searchState = { offset: 0 };
|
|
316
|
+
|
|
317
|
+
const finalize = (boundary = 'layout_segments') => {
|
|
318
|
+
finalizeSegmentChunk(chunks, group, content, searchState, boundary);
|
|
319
|
+
group = [];
|
|
320
|
+
groupWordCount = 0;
|
|
321
|
+
};
|
|
322
|
+
|
|
323
|
+
for (let i = 0; i < segments.length; i += 1) {
|
|
324
|
+
const segment = segments[i];
|
|
325
|
+
if (segment.type === 'PageHeader' || segment.type === 'PageFooter') {
|
|
326
|
+
continue;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
const hierarchyLevel = segmentHierarchyLevel(segment.type);
|
|
330
|
+
if (segment.type === 'Title' || segment.type === 'SectionHeader') {
|
|
331
|
+
if (group.length && hierarchyLevel > previousHierarchyLevel) {
|
|
332
|
+
finalize('section');
|
|
333
|
+
}
|
|
334
|
+
group.push(segment);
|
|
335
|
+
groupWordCount += segment.wordCount;
|
|
336
|
+
previousHierarchyLevel = hierarchyLevel;
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
let keepWithNext = false;
|
|
341
|
+
if (isAssetType(segment.type)) {
|
|
342
|
+
keepWithNext = segments[i + 1]?.type === 'Caption';
|
|
343
|
+
} else if (segment.type === 'Caption') {
|
|
344
|
+
keepWithNext = isAssetType(segments[i + 1]?.type);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
const nextSegment = keepWithNext ? segments[i + 1] : null;
|
|
348
|
+
const pairedWordCount = segment.wordCount + (nextSegment?.wordCount || 0);
|
|
349
|
+
const wouldOverflow = group.length && groupWordCount + pairedWordCount > TARGET_SEGMENT_WORDS;
|
|
350
|
+
|
|
351
|
+
if (wouldOverflow) finalize('layout_segments');
|
|
352
|
+
|
|
353
|
+
group.push(segment);
|
|
354
|
+
groupWordCount += segment.wordCount;
|
|
355
|
+
|
|
356
|
+
if (groupWordCount > TARGET_SEGMENT_WORDS && !keepWithNext) {
|
|
357
|
+
finalize('layout_segments');
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
previousHierarchyLevel = hierarchyLevel;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
finalize('layout_segments');
|
|
364
|
+
|
|
365
|
+
return chunks.map((chunk, chunkIndex) => ({
|
|
366
|
+
chunkIndex,
|
|
367
|
+
charStart: chunk.charStart,
|
|
368
|
+
charEnd: chunk.charEnd,
|
|
369
|
+
content: chunk.content,
|
|
370
|
+
contentHash: contentHash(chunk.content),
|
|
371
|
+
metadata: chunk.metadata,
|
|
372
|
+
}));
|
|
373
|
+
}
|
|
374
|
+
|
|
110
375
|
function chunkDocument(document) {
|
|
111
376
|
const content = String(document?.content || '').trim();
|
|
112
377
|
if (!content) return [];
|
|
113
378
|
|
|
379
|
+
const layoutChunks = chunkLayoutSegments(document, content);
|
|
380
|
+
if (layoutChunks) return layoutChunks;
|
|
381
|
+
|
|
114
382
|
const messages = messageBlocks(document);
|
|
115
383
|
if (messages.length) {
|
|
116
384
|
let searchOffset = 0;
|
|
@@ -78,11 +78,12 @@ class TelnyxVoicePlatform extends BasePlatform {
|
|
|
78
78
|
const token = process.env.TELNYX_WEBHOOK_TOKEN;
|
|
79
79
|
this._webhookToken = token || null;
|
|
80
80
|
const inboundUrl = `${this.webhookUrl}/api/telnyx/webhook${token ? `?token=${token}` : ''}`;
|
|
81
|
-
|
|
81
|
+
// Log only the base path — the token is a secret and must not appear in logs
|
|
82
|
+
console.log(`[TelnyxVoice] Inbound webhook URL (configure this in the Telnyx portal): ${this.webhookUrl}/api/telnyx/webhook`);
|
|
82
83
|
|
|
83
84
|
this.status = 'connected';
|
|
84
85
|
this.emit('connected');
|
|
85
|
-
console.log(
|
|
86
|
+
console.log('[TelnyxVoice] Connected');
|
|
86
87
|
return { status: 'connected', inboundWebhookUrl: inboundUrl };
|
|
87
88
|
}
|
|
88
89
|
|
|
@@ -223,7 +224,7 @@ class TelnyxVoicePlatform extends BasePlatform {
|
|
|
223
224
|
_banNumber(number, durationMs = 10 * 60 * 1000) {
|
|
224
225
|
const key = this._normalizeNumber(number);
|
|
225
226
|
this._bannedNumbers.set(key, Date.now() + durationMs);
|
|
226
|
-
console.log(`[TelnyxVoice] Banned
|
|
227
|
+
console.log(`[TelnyxVoice] Banned a number for ${durationMs / 60000} min`);
|
|
227
228
|
}
|
|
228
229
|
|
|
229
230
|
_startSecretTimer(ccId) {
|
|
@@ -444,18 +445,18 @@ class TelnyxVoicePlatform extends BasePlatform {
|
|
|
444
445
|
}],
|
|
445
446
|
};
|
|
446
447
|
if (this._isBanned(caller)) {
|
|
447
|
-
console.log(
|
|
448
|
+
console.log('[TelnyxVoice] Rejecting banned caller');
|
|
448
449
|
await this._rejectCall(ccId);
|
|
449
450
|
this.emit('blocked_caller', blockedInfo);
|
|
450
451
|
break;
|
|
451
452
|
}
|
|
452
453
|
if (!this.voiceSecret) {
|
|
453
|
-
console.log(
|
|
454
|
+
console.log('[TelnyxVoice] Blocked non-whitelisted caller (no secret set)');
|
|
454
455
|
await this._rejectCall(ccId);
|
|
455
456
|
this.emit('blocked_caller', blockedInfo);
|
|
456
457
|
break;
|
|
457
458
|
}
|
|
458
|
-
console.log(
|
|
459
|
+
console.log('[TelnyxVoice] Non-whitelisted caller — awaiting secret code');
|
|
459
460
|
this._initSession(ccId, caller);
|
|
460
461
|
this._session(ccId).awaitingSecret = true;
|
|
461
462
|
await this._answerCall(ccId);
|
|
@@ -463,7 +464,7 @@ class TelnyxVoicePlatform extends BasePlatform {
|
|
|
463
464
|
}
|
|
464
465
|
this._initSession(ccId, caller);
|
|
465
466
|
await this._answerCall(ccId);
|
|
466
|
-
console.log(`[TelnyxVoice] Answered inbound call
|
|
467
|
+
console.log(`[TelnyxVoice] Answered inbound call (${ccId.slice(-8)})`);
|
|
467
468
|
break;
|
|
468
469
|
}
|
|
469
470
|
case 'call.answered': {
|
|
@@ -542,7 +543,7 @@ class TelnyxVoicePlatform extends BasePlatform {
|
|
|
542
543
|
sess.awaitingUserInput = true;
|
|
543
544
|
await this._sayText(ccId, 'Hello! I am your AI assistant. How can I help you?');
|
|
544
545
|
} else {
|
|
545
|
-
console.log(
|
|
546
|
+
console.log('[TelnyxVoice] Wrong secret, banning caller');
|
|
546
547
|
this._banNumber(sess.callerNumber);
|
|
547
548
|
this._endSession(ccId);
|
|
548
549
|
try { await this._hangupCall(ccId); } catch {}
|
|
@@ -261,18 +261,11 @@ class RecordingManager {
|
|
|
261
261
|
throw new Error('sequenceIndex must be a non-negative integer.');
|
|
262
262
|
}
|
|
263
263
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
const expectedNextSequence = maxSequence == null ? 0 : Number(maxSequence) + 1;
|
|
270
|
-
if (sequenceIndex > expectedNextSequence) {
|
|
271
|
-
throw new Error(
|
|
272
|
-
`Invalid chunk sequence for source "${source.source_key}": got ${sequenceIndex}, expected ${expectedNextSequence}. Sequence must be contiguous per source.`
|
|
273
|
-
);
|
|
274
|
-
}
|
|
275
|
-
|
|
264
|
+
// Accept chunks even when an earlier sequence number never arrived (e.g. a
|
|
265
|
+
// chunk upload was permanently dropped by a flaky network). A gap only
|
|
266
|
+
// costs the audio in the missing chunk; every later chunk is still stored
|
|
267
|
+
// and transcribed. Rejecting non-contiguous sequences here would instead
|
|
268
|
+
// cascade a single lost chunk into total loss of the rest of the source.
|
|
276
269
|
const existing = db.prepare(`
|
|
277
270
|
SELECT id
|
|
278
271
|
FROM recording_chunks
|
|
@@ -481,10 +474,10 @@ class RecordingManager {
|
|
|
481
474
|
continue;
|
|
482
475
|
}
|
|
483
476
|
|
|
484
|
-
this.#
|
|
485
|
-
const sourceDuration =
|
|
477
|
+
this.#warnOnChunkGaps(source.source_key, chunks);
|
|
478
|
+
const sourceDuration = this.#maxOf(
|
|
479
|
+
chunks.map((chunk) => Number(chunk.end_ms) || 0),
|
|
486
480
|
Number(source.duration_ms) || 0,
|
|
487
|
-
...chunks.map((chunk) => Number(chunk.end_ms) || 0),
|
|
488
481
|
);
|
|
489
482
|
maxDuration = Math.max(maxDuration, sourceDuration);
|
|
490
483
|
|
|
@@ -505,11 +498,11 @@ class RecordingManager {
|
|
|
505
498
|
const sourceSegments = await this.#transcribeSourceChunks(source, chunks, {
|
|
506
499
|
transcriptionModel: recordingTranscriptionModel,
|
|
507
500
|
});
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
...sourceSegments.map((segment) => Number(segment.endMs) || 0),
|
|
501
|
+
const sourceSegmentEnd = this.#maxOf(
|
|
502
|
+
sourceSegments.map((segment) => Number(segment.endMs) || 0),
|
|
511
503
|
sourceDuration,
|
|
512
504
|
);
|
|
505
|
+
maxDuration = Math.max(maxDuration, sourceSegmentEnd);
|
|
513
506
|
collectedSegments.push(...sourceSegments);
|
|
514
507
|
|
|
515
508
|
db.prepare(`
|
|
@@ -518,7 +511,7 @@ class RecordingManager {
|
|
|
518
511
|
WHERE id = ?
|
|
519
512
|
`).run(
|
|
520
513
|
SESSION_STATUS.completed,
|
|
521
|
-
|
|
514
|
+
sourceSegmentEnd,
|
|
522
515
|
new Date().toISOString(),
|
|
523
516
|
source.id,
|
|
524
517
|
);
|
|
@@ -839,20 +832,38 @@ class RecordingManager {
|
|
|
839
832
|
});
|
|
840
833
|
}
|
|
841
834
|
|
|
835
|
+
// Streaming recorders (browser MediaRecorder, native MPEG encoders) split a
|
|
836
|
+
// single media stream across many timeslice chunks where only the first
|
|
837
|
+
// chunk carries the container header (EBML for WebM/Ogg, ftyp/moov for MP4,
|
|
838
|
+
// frame sync for MPEG). The later chunks cannot be decoded on their own, so
|
|
839
|
+
// they MUST be concatenated back into one container before transcription.
|
|
840
|
+
// Sending them to Deepgram individually only ever decodes the first ~chunk.
|
|
841
|
+
#isConcatenableContainerMime(mime) {
|
|
842
|
+
const value = `${mime || ''}`.toLowerCase();
|
|
843
|
+
return (
|
|
844
|
+
value.includes('mpeg') ||
|
|
845
|
+
value.includes('mp3') ||
|
|
846
|
+
value.includes('webm') ||
|
|
847
|
+
value.includes('ogg') ||
|
|
848
|
+
value.includes('opus') ||
|
|
849
|
+
value.includes('mp4') ||
|
|
850
|
+
value.includes('m4a') ||
|
|
851
|
+
value.includes('aac')
|
|
852
|
+
);
|
|
853
|
+
}
|
|
854
|
+
|
|
842
855
|
#canTranscribeAsMergedBinary(source, chunks) {
|
|
843
856
|
if (!Array.isArray(chunks) || chunks.length === 0) {
|
|
844
857
|
return false;
|
|
845
858
|
}
|
|
846
859
|
|
|
847
|
-
|
|
848
|
-
const sourceSeemsMpeg = sourceMime.includes('mpeg') || sourceMime.includes('mp3');
|
|
849
|
-
if (!sourceSeemsMpeg) {
|
|
860
|
+
if (!this.#isConcatenableContainerMime(source.mime_type)) {
|
|
850
861
|
return false;
|
|
851
862
|
}
|
|
852
863
|
|
|
853
864
|
return chunks.every((chunk) => {
|
|
854
865
|
const mime = `${chunk.mime_type || source.mime_type || ''}`.toLowerCase();
|
|
855
|
-
return
|
|
866
|
+
return this.#isConcatenableContainerMime(mime) || mime.includes('octet-stream');
|
|
856
867
|
});
|
|
857
868
|
}
|
|
858
869
|
|
|
@@ -1056,12 +1067,34 @@ class RecordingManager {
|
|
|
1056
1067
|
});
|
|
1057
1068
|
}
|
|
1058
1069
|
|
|
1059
|
-
#
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1070
|
+
#warnOnChunkGaps(sourceKey, chunks) {
|
|
1071
|
+
const missing = [];
|
|
1072
|
+
let expected = 0;
|
|
1073
|
+
for (const chunk of chunks) {
|
|
1074
|
+
const sequence = Number(chunk.sequence_index) || 0;
|
|
1075
|
+
while (expected < sequence) {
|
|
1076
|
+
missing.push(expected);
|
|
1077
|
+
expected += 1;
|
|
1078
|
+
}
|
|
1079
|
+
expected = sequence + 1;
|
|
1080
|
+
}
|
|
1081
|
+
if (missing.length > 0) {
|
|
1082
|
+
console.warn(
|
|
1083
|
+
`[Recordings] Source "${sourceKey}" is missing chunk(s) ${missing.slice(0, 20).join(', ')}` +
|
|
1084
|
+
`${missing.length > 20 ? ', …' : ''}; transcribing the audio that did arrive.`,
|
|
1085
|
+
);
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
#maxOf(values, seed = 0) {
|
|
1090
|
+
let max = Number(seed) || 0;
|
|
1091
|
+
for (const value of (Array.isArray(values) ? values : [])) {
|
|
1092
|
+
const num = Number(value) || 0;
|
|
1093
|
+
if (num > max) {
|
|
1094
|
+
max = num;
|
|
1063
1095
|
}
|
|
1064
1096
|
}
|
|
1097
|
+
return max;
|
|
1065
1098
|
}
|
|
1066
1099
|
|
|
1067
1100
|
#shouldTranscribeSource(source, metadata = {}) {
|
|
@@ -131,32 +131,50 @@ class RuntimeHttpClient {
|
|
|
131
131
|
}
|
|
132
132
|
|
|
133
133
|
async requestStream(method, pathname, stream, options = {}) {
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
...(options.contentType ? { 'content-type': options.contentType } : {}),
|
|
138
|
-
...(options.contentLength != null ? { 'content-length': String(options.contentLength) } : {}),
|
|
139
|
-
...(options.headers || {}),
|
|
140
|
-
...(this.token ? { authorization: `Bearer ${this.token}` } : {}),
|
|
141
|
-
},
|
|
142
|
-
body: stream,
|
|
143
|
-
duplex: 'half',
|
|
144
|
-
});
|
|
134
|
+
const retryCount = Math.max(0, Number(options.retryCount ?? 6));
|
|
135
|
+
const retryDelayMs = Math.max(100, Number(options.retryDelayMs ?? 1000));
|
|
136
|
+
let lastError = null;
|
|
145
137
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
138
|
+
for (let attempt = 0; attempt <= retryCount; attempt += 1) {
|
|
139
|
+
try {
|
|
140
|
+
const response = await fetch(`${this.baseUrl}${pathname}`, {
|
|
141
|
+
method,
|
|
142
|
+
headers: {
|
|
143
|
+
...(options.contentType ? { 'content-type': options.contentType } : {}),
|
|
144
|
+
...(options.contentLength != null ? { 'content-length': String(options.contentLength) } : {}),
|
|
145
|
+
...(options.headers || {}),
|
|
146
|
+
...(this.token ? { authorization: `Bearer ${this.token}` } : {}),
|
|
147
|
+
},
|
|
148
|
+
body: stream,
|
|
149
|
+
duplex: 'half',
|
|
150
|
+
});
|
|
149
151
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
152
|
+
if (response.ok && typeof this.onActivity === 'function') {
|
|
153
|
+
this.onActivity();
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const contentType = response.headers.get('content-type') || '';
|
|
157
|
+
const payload = contentType.includes('application/json')
|
|
158
|
+
? await response.json().catch(() => ({}))
|
|
159
|
+
: { text: await response.text().catch(() => '') };
|
|
154
160
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
161
|
+
if (!response.ok) {
|
|
162
|
+
const errorMessage = payload?.error || payload?.text || `Runtime request failed: ${response.status}`;
|
|
163
|
+
throw new Error(errorMessage);
|
|
164
|
+
}
|
|
165
|
+
return payload;
|
|
166
|
+
} catch (error) {
|
|
167
|
+
lastError = error;
|
|
168
|
+
const message = String(error?.message || error);
|
|
169
|
+
const retryable = /fetch failed|ECONNREFUSED|ECONNRESET|socket hang up|timed out/i.test(message);
|
|
170
|
+
if (!retryable || attempt === retryCount) {
|
|
171
|
+
throw error;
|
|
172
|
+
}
|
|
173
|
+
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
|
|
174
|
+
}
|
|
158
175
|
}
|
|
159
|
-
|
|
176
|
+
|
|
177
|
+
throw lastError || new Error('Runtime request failed.');
|
|
160
178
|
}
|
|
161
179
|
}
|
|
162
180
|
|