neoagent 3.0.1-beta.8 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/.env.example +2 -10
  2. package/README.md +12 -3
  3. package/docs/automation.md +37 -0
  4. package/docs/benchmarking.md +102 -0
  5. package/docs/billing.md +34 -8
  6. package/docs/configuration.md +11 -2
  7. package/docs/getting-started.md +10 -8
  8. package/docs/operations.md +1 -1
  9. package/flutter_app/lib/main.dart +3 -0
  10. package/flutter_app/lib/main_account_settings.dart +138 -0
  11. package/flutter_app/lib/main_app_shell.dart +38 -13
  12. package/flutter_app/lib/main_billing.dart +1465 -0
  13. package/flutter_app/lib/main_chat.dart +1612 -214
  14. package/flutter_app/lib/main_controller.dart +398 -26
  15. package/flutter_app/lib/main_devices.dart +293 -207
  16. package/flutter_app/lib/main_models.dart +142 -0
  17. package/flutter_app/lib/main_navigation.dart +19 -1
  18. package/flutter_app/lib/main_operations.dart +288 -9
  19. package/flutter_app/lib/main_settings.dart +510 -300
  20. package/flutter_app/lib/main_shared.dart +2 -0
  21. package/flutter_app/lib/main_timeline.dart +1378 -0
  22. package/flutter_app/lib/src/backend_client.dart +149 -19
  23. package/flutter_app/lib/src/desktop_companion_actions.dart +132 -21
  24. package/flutter_app/lib/src/desktop_companion_io.dart +65 -1
  25. package/flutter_app/lib/src/desktop_companion_stub.dart +12 -0
  26. package/flutter_app/lib/src/desktop_ocr_bridge.dart +2 -0
  27. package/flutter_app/lib/src/desktop_ocr_bridge_io.dart +125 -0
  28. package/flutter_app/lib/src/desktop_ocr_bridge_stub.dart +30 -0
  29. package/flutter_app/lib/src/desktop_passive_history.dart +332 -0
  30. package/flutter_app/lib/src/recording_bridge_io.dart +80 -72
  31. package/flutter_app/lib/src/recording_bridge_web.dart +127 -114
  32. package/flutter_app/lib/src/recording_chunk_queue.dart +149 -0
  33. package/flutter_app/lib/src/recording_chunk_queue_io.dart +182 -0
  34. package/flutter_app/lib/src/recording_payloads.dart +9 -0
  35. package/flutter_app/macos/Runner/AppDelegate.swift +25 -0
  36. package/flutter_app/windows/runner/flutter_window.cpp +75 -0
  37. package/landing/index.html +3 -3
  38. package/lib/manager.js +184 -66
  39. package/package.json +4 -1
  40. package/server/admin/access.js +12 -7
  41. package/server/admin/admin.css +78 -0
  42. package/server/admin/admin.js +436 -10
  43. package/server/admin/billing.js +165 -42
  44. package/server/admin/index.html +80 -5
  45. package/server/admin/users.js +15 -15
  46. package/server/db/database.js +125 -20
  47. package/server/http/routes.js +1 -0
  48. package/server/http/static.js +4 -2
  49. package/server/public/.last_build_id +1 -1
  50. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  51. package/server/public/canvaskit/wimp.js.symbols +8475 -8467
  52. package/server/public/canvaskit/wimp.wasm +0 -0
  53. package/server/public/flutter_bootstrap.js +2 -2
  54. package/server/public/main.dart.js +91077 -87037
  55. package/server/routes/account.js +53 -0
  56. package/server/routes/admin.js +345 -64
  57. package/server/routes/agents.js +203 -21
  58. package/server/routes/billing.js +5 -0
  59. package/server/routes/browser.js +8 -1
  60. package/server/routes/recordings.js +96 -6
  61. package/server/routes/screenHistory.js +140 -2
  62. package/server/routes/timeline.js +43 -0
  63. package/server/services/account/erasure.js +263 -0
  64. package/server/services/ai/hooks.js +4 -1
  65. package/server/services/ai/loop/agent_engine_core.js +8 -1
  66. package/server/services/ai/loop/blank_recovery.js +36 -0
  67. package/server/services/ai/loop/conversation_loop.js +166 -33
  68. package/server/services/ai/messagingFallback.js +22 -0
  69. package/server/services/ai/rate_limits.js +28 -5
  70. package/server/services/ai/systemPrompt.js +6 -5
  71. package/server/services/ai/taskAnalysis.js +2 -0
  72. package/server/services/ai/toolEvidence.js +15 -0
  73. package/server/services/ai/toolResult.js +40 -0
  74. package/server/services/ai/tools.js +163 -4
  75. package/server/services/android/controller.js +6 -2
  76. package/server/services/billing/plans.js +2 -1
  77. package/server/services/browser/anti_detection.js +192 -0
  78. package/server/services/browser/controller.js +180 -54
  79. package/server/services/desktop/auth.js +3 -0
  80. package/server/services/desktop/registry.js +50 -2
  81. package/server/services/integrations/google/calendar.js +22 -14
  82. package/server/services/manager.js +12 -42
  83. package/server/services/memory/ingestion_chunking.js +268 -0
  84. package/server/services/messaging/telnyx.js +9 -8
  85. package/server/services/recordings/manager.js +60 -27
  86. package/server/services/runtime/backends/local-vm.js +40 -22
  87. package/server/services/runtime/docker-vm-manager.js +157 -266
  88. package/server/services/runtime/guest_bootstrap.js +17 -5
  89. package/server/services/runtime/guest_image.js +188 -0
  90. package/server/services/runtime/manager.js +0 -1
  91. package/server/services/runtime/validation.js +3 -8
  92. package/server/services/social_video/service.js +60 -10
  93. package/server/services/tasks/runtime.js +234 -9
  94. package/server/services/tasks/schedule_utils.js +5 -5
  95. package/server/services/tasks/task_repository.js +13 -0
  96. package/server/services/timeline/service.js +558 -0
  97. package/server/services/wearable/gateway.js +1 -1
  98. package/server/services/websocket.js +21 -3
  99. package/server/services/desktop/screenRecorder.js +0 -292
  100. package/server/services/desktop/screen_recorder_support.js +0 -46
@@ -32,10 +32,7 @@ const { registerToolSecurityHooks } = require('./security/tool_security_hook');
32
32
  const { BrowserExtensionRegistry } = require('./browser/extension/registry');
33
33
  const { DesktopCompanionRegistry } = require('./desktop/registry');
34
34
  const { DesktopProvider } = require('./desktop/provider');
35
- const {
36
- ScreenRecorder,
37
- hasOpenConnectionForUser,
38
- } = require('./desktop/screenRecorder');
35
+ const { TimelineService } = require('./timeline/service');
39
36
  const { WearableService } = require('./wearable/service');
40
37
  const { getRuntimeValidation } = require('./runtime/validation');
41
38
  const {
@@ -96,6 +93,16 @@ function createDesktopCompanionRegistry(app) {
96
93
  return registry;
97
94
  }
98
95
 
96
+ function createTimelineService(app, io) {
97
+ const timelineService = registerLocal(
98
+ app,
99
+ 'timelineService',
100
+ new TimelineService({ io }),
101
+ );
102
+ logServiceReady('Timeline service ready');
103
+ return timelineService;
104
+ }
105
+
99
106
  function createMemoryManager(app) {
100
107
  const memoryManager = registerLocal(app, 'memoryManager', new MemoryManager());
101
108
  memoryManager.startEmbeddingIndexBackfill();
@@ -406,30 +413,6 @@ function createWearableService(app) {
406
413
  return wearableService;
407
414
  }
408
415
 
409
- function createScreenRecorder(app) {
410
- const hasActiveCaptureSessionForUser = (userId) => {
411
- return (
412
- hasOpenConnectionForUser(app.locals.desktopCompanionRegistry, userId)
413
- || hasOpenConnectionForUser(app.locals.browserExtensionRegistry, userId)
414
- );
415
- };
416
-
417
- const screenRecorder = registerLocal(
418
- app,
419
- 'screenRecorder',
420
- new ScreenRecorder({ hasActiveCaptureSessionForUser }),
421
- );
422
- const status = screenRecorder.start();
423
- if (status.state === 'running') {
424
- logServiceReady(
425
- `Screen recorder started for user ${status.ownerUserId} (${status.intervalMs}ms interval)`,
426
- );
427
- } else {
428
- logServiceReady(`Screen recorder ${status.state}: ${status.reason}`);
429
- }
430
- return screenRecorder;
431
- }
432
-
433
416
  function restoreMessagingConnections(messagingManager) {
434
417
  void runBackgroundTask('[Messaging] Restore error:', () =>
435
418
  messagingManager.restoreConnections(),
@@ -486,6 +469,7 @@ async function startServices(app, io) {
486
469
  createWorkspaceManager(app);
487
470
  createBrowserExtensionRegistry(app);
488
471
  createDesktopCompanionRegistry(app);
472
+ createTimelineService(app, io);
489
473
  const memoryManager = createMemoryManager(app);
490
474
  const mcpClient = createMcpClient(app);
491
475
  createAuthProviderManager(app);
@@ -531,7 +515,6 @@ async function startServices(app, io) {
531
515
  createSocialVideoService(app);
532
516
  createWidgetService(app);
533
517
  createWearableService(app);
534
- createScreenRecorder(app);
535
518
 
536
519
  restoreMessagingConnections(messagingManager);
537
520
  restoreMcpClients(mcpClient);
@@ -710,19 +693,6 @@ async function stopServices(app) {
710
693
  }
711
694
  }
712
695
 
713
- if (app.locals.screenRecorder) {
714
- tasks.push(
715
- Promise.resolve()
716
- .then(() => app.locals.screenRecorder.stop())
717
- .then((status) => {
718
- logServiceReady(`Screen recorder shutdown complete (${status.state})`);
719
- })
720
- .catch((err) => {
721
- console.error('[ScreenRecorder] Stop error:', getErrorMessage(err));
722
- }),
723
- );
724
- }
725
-
726
696
  if (app.locals.browserExtensionRegistry) {
727
697
  try {
728
698
  app.locals.browserExtensionRegistry.closeAll();
@@ -4,6 +4,47 @@ const crypto = require('node:crypto');
4
4
 
5
5
  const TARGET_CHARS = 1400;
6
6
  const MAX_CHARS = 2200;
7
+ const TARGET_SEGMENT_WORDS = 260;
8
+
9
+ const SEGMENT_TYPES = new Set([
10
+ 'Caption',
11
+ 'Footnote',
12
+ 'Formula',
13
+ 'ListItem',
14
+ 'Page',
15
+ 'PageFooter',
16
+ 'PageHeader',
17
+ 'Picture',
18
+ 'SectionHeader',
19
+ 'Table',
20
+ 'Text',
21
+ 'Title',
22
+ ]);
23
+
24
+ const SEGMENT_ALIASES = Object.freeze({
25
+ caption: 'Caption',
26
+ footnote: 'Footnote',
27
+ formula: 'Formula',
28
+ listitem: 'ListItem',
29
+ list_item: 'ListItem',
30
+ 'list item': 'ListItem',
31
+ page: 'Page',
32
+ pagefooter: 'PageFooter',
33
+ page_footer: 'PageFooter',
34
+ 'page footer': 'PageFooter',
35
+ pageheader: 'PageHeader',
36
+ page_header: 'PageHeader',
37
+ 'page header': 'PageHeader',
38
+ picture: 'Picture',
39
+ image: 'Picture',
40
+ figure: 'Picture',
41
+ sectionheader: 'SectionHeader',
42
+ section_header: 'SectionHeader',
43
+ 'section header': 'SectionHeader',
44
+ table: 'Table',
45
+ text: 'Text',
46
+ title: 'Title',
47
+ });
7
48
 
8
49
  function contentHash(content) {
9
50
  return crypto.createHash('sha256').update(content).digest('hex');
@@ -107,10 +148,237 @@ function packBlocksWithPositions(blocksWithPos) {
107
148
  }));
108
149
  }
109
150
 
151
+ function wordCount(text) {
152
+ return (String(text || '').trim().match(/\S+/g) || []).length;
153
+ }
154
+
155
+ function normalizeSegmentType(value) {
156
+ const raw = String(value || '').trim();
157
+ if (SEGMENT_TYPES.has(raw)) return raw;
158
+ return SEGMENT_ALIASES[raw.toLowerCase()] || 'Text';
159
+ }
160
+
161
+ function normalizeNumber(value) {
162
+ const number = Number(value);
163
+ return Number.isFinite(number) ? number : null;
164
+ }
165
+
166
+ function normalizeBoundingBox(value) {
167
+ if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
168
+ const left = normalizeNumber(value.left ?? value.x ?? value.x1);
169
+ const top = normalizeNumber(value.top ?? value.y ?? value.y1);
170
+ const width = normalizeNumber(value.width ?? (
171
+ value.x2 != null && left != null ? Number(value.x2) - left : null
172
+ ));
173
+ const height = normalizeNumber(value.height ?? (
174
+ value.y2 != null && top != null ? Number(value.y2) - top : null
175
+ ));
176
+ if ([left, top, width, height].some((number) => number == null)) return null;
177
+ return { left, top, width, height };
178
+ }
179
+
180
+ function segmentEmbedContent(segment) {
181
+ return [
182
+ segment.llm,
183
+ segment.markdown,
184
+ segment.html,
185
+ segment.text,
186
+ segment.content,
187
+ ].map((value) => String(value || '').trim()).find(Boolean) || '';
188
+ }
189
+
190
+ function normalizeLayoutSegments(document) {
191
+ const segments = Array.isArray(document?.payload?.segments)
192
+ ? document.payload.segments
193
+ : [];
194
+ const normalized = [];
195
+ for (const segment of segments) {
196
+ if (!segment || typeof segment !== 'object' || Array.isArray(segment)) continue;
197
+ const content = segmentEmbedContent(segment);
198
+ if (!content) continue;
199
+ const segmentType = normalizeSegmentType(segment.type || segment.segmentType || segment.segment_type);
200
+ normalized.push({
201
+ id: String(segment.id || segment.segmentId || segment.segment_id || '').trim() || null,
202
+ type: segmentType,
203
+ content,
204
+ wordCount: wordCount(content),
205
+ pageNumber: normalizeNumber(segment.pageNumber ?? segment.page_number),
206
+ pageWidth: normalizeNumber(segment.pageWidth ?? segment.page_width),
207
+ pageHeight: normalizeNumber(segment.pageHeight ?? segment.page_height),
208
+ bbox: normalizeBoundingBox(segment.bbox || segment.boundingBox || segment.bounding_box),
209
+ confidence: normalizeNumber(segment.confidence),
210
+ });
211
+ }
212
+ return normalized;
213
+ }
214
+
215
+ function segmentHierarchyLevel(type) {
216
+ if (type === 'Title') return 3;
217
+ if (type === 'SectionHeader') return 2;
218
+ return 1;
219
+ }
220
+
221
+ function isAssetType(type) {
222
+ return type === 'Picture' || type === 'Table';
223
+ }
224
+
225
+ function hasAdjacentAssetCaptionPair(segments) {
226
+ return segments.some((segment, index) => {
227
+ const nextType = segments[index + 1]?.type;
228
+ return (isAssetType(segment.type) && nextType === 'Caption')
229
+ || (segment.type === 'Caption' && isAssetType(nextType));
230
+ });
231
+ }
232
+
233
+ function findSegmentPosition(content, rendered, searchOffset) {
234
+ const pos = content.indexOf(rendered, searchOffset);
235
+ if (pos >= 0) return { charStart: pos, charEnd: pos + rendered.length };
236
+
237
+ const compactRendered = String(rendered || '').replace(/\s+/g, ' ').trim();
238
+ if (compactRendered && compactRendered.length <= 160) {
239
+ const compactContent = content.replace(/\s+/g, ' ');
240
+ const compactPos = compactContent.indexOf(compactRendered);
241
+ if (compactPos >= 0) {
242
+ return {
243
+ charStart: Math.min(compactPos, content.length),
244
+ charEnd: Math.min(compactPos + compactRendered.length, content.length),
245
+ };
246
+ }
247
+ }
248
+
249
+ return {
250
+ charStart: Math.min(searchOffset, content.length),
251
+ charEnd: Math.min(searchOffset + rendered.length, content.length),
252
+ };
253
+ }
254
+
255
+ function metadataForSegments(segments, boundary = 'layout_segments') {
256
+ const pageNumbers = [...new Set(
257
+ segments.map((segment) => segment.pageNumber).filter((number) => number != null),
258
+ )];
259
+ const segmentTypes = [...new Set(segments.map((segment) => segment.type))];
260
+ const resolvedBoundary = boundary === 'layout_segments' && hasAdjacentAssetCaptionPair(segments)
261
+ ? 'asset_caption_pair'
262
+ : boundary;
263
+ return {
264
+ boundary: resolvedBoundary,
265
+ segmentTypes,
266
+ segmentIds: segments.map((segment) => segment.id).filter(Boolean),
267
+ segments: segments.map((segment) => ({
268
+ id: segment.id,
269
+ type: segment.type,
270
+ pageNumber: segment.pageNumber,
271
+ pageWidth: segment.pageWidth,
272
+ pageHeight: segment.pageHeight,
273
+ confidence: segment.confidence,
274
+ })),
275
+ pageNumbers,
276
+ pages: pageNumbers,
277
+ bboxes: segments
278
+ .map((segment) => segment.bbox && {
279
+ segmentId: segment.id,
280
+ type: segment.type,
281
+ pageNumber: segment.pageNumber,
282
+ bbox: segment.bbox,
283
+ })
284
+ .filter(Boolean),
285
+ };
286
+ }
287
+
288
+ function finalizeSegmentChunk(chunks, group, content, searchState, boundary = 'layout_segments') {
289
+ if (!group.length) return;
290
+ const rendered = group.map((segment) => segment.content).join('\n\n');
291
+ const position = findSegmentPosition(content, rendered, searchState.offset);
292
+ searchState.offset = Math.max(searchState.offset, position.charEnd);
293
+ const metadata = metadataForSegments(group, boundary);
294
+ for (const part of splitOversizedWithPos({
295
+ content: rendered,
296
+ charStart: position.charStart,
297
+ })) {
298
+ chunks.push({
299
+ content: part.content,
300
+ charStart: part.charStart,
301
+ charEnd: Math.min(part.charEnd, position.charEnd),
302
+ metadata,
303
+ });
304
+ }
305
+ }
306
+
307
+ function chunkLayoutSegments(document, content) {
308
+ const segments = normalizeLayoutSegments(document);
309
+ if (!segments.length) return null;
310
+
311
+ const chunks = [];
312
+ let group = [];
313
+ let groupWordCount = 0;
314
+ let previousHierarchyLevel = 1;
315
+ const searchState = { offset: 0 };
316
+
317
+ const finalize = (boundary = 'layout_segments') => {
318
+ finalizeSegmentChunk(chunks, group, content, searchState, boundary);
319
+ group = [];
320
+ groupWordCount = 0;
321
+ };
322
+
323
+ for (let i = 0; i < segments.length; i += 1) {
324
+ const segment = segments[i];
325
+ if (segment.type === 'PageHeader' || segment.type === 'PageFooter') {
326
+ continue;
327
+ }
328
+
329
+ const hierarchyLevel = segmentHierarchyLevel(segment.type);
330
+ if (segment.type === 'Title' || segment.type === 'SectionHeader') {
331
+ if (group.length && hierarchyLevel > previousHierarchyLevel) {
332
+ finalize('section');
333
+ }
334
+ group.push(segment);
335
+ groupWordCount += segment.wordCount;
336
+ previousHierarchyLevel = hierarchyLevel;
337
+ continue;
338
+ }
339
+
340
+ let keepWithNext = false;
341
+ if (isAssetType(segment.type)) {
342
+ keepWithNext = segments[i + 1]?.type === 'Caption';
343
+ } else if (segment.type === 'Caption') {
344
+ keepWithNext = isAssetType(segments[i + 1]?.type);
345
+ }
346
+
347
+ const nextSegment = keepWithNext ? segments[i + 1] : null;
348
+ const pairedWordCount = segment.wordCount + (nextSegment?.wordCount || 0);
349
+ const wouldOverflow = group.length && groupWordCount + pairedWordCount > TARGET_SEGMENT_WORDS;
350
+
351
+ if (wouldOverflow) finalize('layout_segments');
352
+
353
+ group.push(segment);
354
+ groupWordCount += segment.wordCount;
355
+
356
+ if (groupWordCount > TARGET_SEGMENT_WORDS && !keepWithNext) {
357
+ finalize('layout_segments');
358
+ }
359
+
360
+ previousHierarchyLevel = hierarchyLevel;
361
+ }
362
+
363
+ finalize('layout_segments');
364
+
365
+ return chunks.map((chunk, chunkIndex) => ({
366
+ chunkIndex,
367
+ charStart: chunk.charStart,
368
+ charEnd: chunk.charEnd,
369
+ content: chunk.content,
370
+ contentHash: contentHash(chunk.content),
371
+ metadata: chunk.metadata,
372
+ }));
373
+ }
374
+
110
375
  function chunkDocument(document) {
111
376
  const content = String(document?.content || '').trim();
112
377
  if (!content) return [];
113
378
 
379
+ const layoutChunks = chunkLayoutSegments(document, content);
380
+ if (layoutChunks) return layoutChunks;
381
+
114
382
  const messages = messageBlocks(document);
115
383
  if (messages.length) {
116
384
  let searchOffset = 0;
@@ -78,11 +78,12 @@ class TelnyxVoicePlatform extends BasePlatform {
78
78
  const token = process.env.TELNYX_WEBHOOK_TOKEN;
79
79
  this._webhookToken = token || null;
80
80
  const inboundUrl = `${this.webhookUrl}/api/telnyx/webhook${token ? `?token=${token}` : ''}`;
81
- console.log(`[TelnyxVoice] Inbound webhook URL (configure this in the Telnyx portal): ${inboundUrl}`);
81
+ // Log only the base path the token is a secret and must not appear in logs
82
+ console.log(`[TelnyxVoice] Inbound webhook URL (configure this in the Telnyx portal): ${this.webhookUrl}/api/telnyx/webhook`);
82
83
 
83
84
  this.status = 'connected';
84
85
  this.emit('connected');
85
- console.log(`[TelnyxVoice] Connected — phone: ${this.phoneNumber}`);
86
+ console.log('[TelnyxVoice] Connected');
86
87
  return { status: 'connected', inboundWebhookUrl: inboundUrl };
87
88
  }
88
89
 
@@ -223,7 +224,7 @@ class TelnyxVoicePlatform extends BasePlatform {
223
224
  _banNumber(number, durationMs = 10 * 60 * 1000) {
224
225
  const key = this._normalizeNumber(number);
225
226
  this._bannedNumbers.set(key, Date.now() + durationMs);
226
- console.log(`[TelnyxVoice] Banned ${number} for ${durationMs / 60000} min`);
227
+ console.log(`[TelnyxVoice] Banned a number for ${durationMs / 60000} min`);
227
228
  }
228
229
 
229
230
  _startSecretTimer(ccId) {
@@ -444,18 +445,18 @@ class TelnyxVoicePlatform extends BasePlatform {
444
445
  }],
445
446
  };
446
447
  if (this._isBanned(caller)) {
447
- console.log(`[TelnyxVoice] Rejecting banned caller: ${caller}`);
448
+ console.log('[TelnyxVoice] Rejecting banned caller');
448
449
  await this._rejectCall(ccId);
449
450
  this.emit('blocked_caller', blockedInfo);
450
451
  break;
451
452
  }
452
453
  if (!this.voiceSecret) {
453
- console.log(`[TelnyxVoice] Blocked non-whitelisted caller (no secret set): ${caller}`);
454
+ console.log('[TelnyxVoice] Blocked non-whitelisted caller (no secret set)');
454
455
  await this._rejectCall(ccId);
455
456
  this.emit('blocked_caller', blockedInfo);
456
457
  break;
457
458
  }
458
- console.log(`[TelnyxVoice] Non-whitelisted caller ${caller} — awaiting secret code`);
459
+ console.log('[TelnyxVoice] Non-whitelisted caller — awaiting secret code');
459
460
  this._initSession(ccId, caller);
460
461
  this._session(ccId).awaitingSecret = true;
461
462
  await this._answerCall(ccId);
@@ -463,7 +464,7 @@ class TelnyxVoicePlatform extends BasePlatform {
463
464
  }
464
465
  this._initSession(ccId, caller);
465
466
  await this._answerCall(ccId);
466
- console.log(`[TelnyxVoice] Answered inbound call from ${caller}`);
467
+ console.log(`[TelnyxVoice] Answered inbound call (${ccId.slice(-8)})`);
467
468
  break;
468
469
  }
469
470
  case 'call.answered': {
@@ -542,7 +543,7 @@ class TelnyxVoicePlatform extends BasePlatform {
542
543
  sess.awaitingUserInput = true;
543
544
  await this._sayText(ccId, 'Hello! I am your AI assistant. How can I help you?');
544
545
  } else {
545
- console.log(`[TelnyxVoice] Wrong secret from ${sess.callerNumber}, banning`);
546
+ console.log('[TelnyxVoice] Wrong secret, banning caller');
546
547
  this._banNumber(sess.callerNumber);
547
548
  this._endSession(ccId);
548
549
  try { await this._hangupCall(ccId); } catch {}
@@ -261,18 +261,11 @@ class RecordingManager {
261
261
  throw new Error('sequenceIndex must be a non-negative integer.');
262
262
  }
263
263
 
264
- const maxSequence = db.prepare(`
265
- SELECT MAX(sequence_index) AS maxSequence
266
- FROM recording_chunks
267
- WHERE source_id = ?
268
- `).get(source.id)?.maxSequence;
269
- const expectedNextSequence = maxSequence == null ? 0 : Number(maxSequence) + 1;
270
- if (sequenceIndex > expectedNextSequence) {
271
- throw new Error(
272
- `Invalid chunk sequence for source "${source.source_key}": got ${sequenceIndex}, expected ${expectedNextSequence}. Sequence must be contiguous per source.`
273
- );
274
- }
275
-
264
+ // Accept chunks even when an earlier sequence number never arrived (e.g. a
265
+ // chunk upload was permanently dropped by a flaky network). A gap only
266
+ // costs the audio in the missing chunk; every later chunk is still stored
267
+ // and transcribed. Rejecting non-contiguous sequences here would instead
268
+ // cascade a single lost chunk into total loss of the rest of the source.
276
269
  const existing = db.prepare(`
277
270
  SELECT id
278
271
  FROM recording_chunks
@@ -481,10 +474,10 @@ class RecordingManager {
481
474
  continue;
482
475
  }
483
476
 
484
- this.#assertSequentialChunks(source.source_key, chunks);
485
- const sourceDuration = Math.max(
477
+ this.#warnOnChunkGaps(source.source_key, chunks);
478
+ const sourceDuration = this.#maxOf(
479
+ chunks.map((chunk) => Number(chunk.end_ms) || 0),
486
480
  Number(source.duration_ms) || 0,
487
- ...chunks.map((chunk) => Number(chunk.end_ms) || 0),
488
481
  );
489
482
  maxDuration = Math.max(maxDuration, sourceDuration);
490
483
 
@@ -505,11 +498,11 @@ class RecordingManager {
505
498
  const sourceSegments = await this.#transcribeSourceChunks(source, chunks, {
506
499
  transcriptionModel: recordingTranscriptionModel,
507
500
  });
508
- maxDuration = Math.max(
509
- maxDuration,
510
- ...sourceSegments.map((segment) => Number(segment.endMs) || 0),
501
+ const sourceSegmentEnd = this.#maxOf(
502
+ sourceSegments.map((segment) => Number(segment.endMs) || 0),
511
503
  sourceDuration,
512
504
  );
505
+ maxDuration = Math.max(maxDuration, sourceSegmentEnd);
513
506
  collectedSegments.push(...sourceSegments);
514
507
 
515
508
  db.prepare(`
@@ -518,7 +511,7 @@ class RecordingManager {
518
511
  WHERE id = ?
519
512
  `).run(
520
513
  SESSION_STATUS.completed,
521
- Math.max(sourceDuration, ...sourceSegments.map((segment) => Number(segment.endMs) || 0)),
514
+ sourceSegmentEnd,
522
515
  new Date().toISOString(),
523
516
  source.id,
524
517
  );
@@ -839,20 +832,38 @@ class RecordingManager {
839
832
  });
840
833
  }
841
834
 
835
+ // Streaming recorders (browser MediaRecorder, native MPEG encoders) split a
836
+ // single media stream across many timeslice chunks where only the first
837
+ // chunk carries the container header (EBML for WebM/Ogg, ftyp/moov for MP4,
838
+ // frame sync for MPEG). The later chunks cannot be decoded on their own, so
839
+ // they MUST be concatenated back into one container before transcription.
840
+ // Sending them to Deepgram individually only ever decodes the first ~chunk.
841
+ #isConcatenableContainerMime(mime) {
842
+ const value = `${mime || ''}`.toLowerCase();
843
+ return (
844
+ value.includes('mpeg') ||
845
+ value.includes('mp3') ||
846
+ value.includes('webm') ||
847
+ value.includes('ogg') ||
848
+ value.includes('opus') ||
849
+ value.includes('mp4') ||
850
+ value.includes('m4a') ||
851
+ value.includes('aac')
852
+ );
853
+ }
854
+
842
855
  #canTranscribeAsMergedBinary(source, chunks) {
843
856
  if (!Array.isArray(chunks) || chunks.length === 0) {
844
857
  return false;
845
858
  }
846
859
 
847
- const sourceMime = `${source.mime_type || ''}`.toLowerCase();
848
- const sourceSeemsMpeg = sourceMime.includes('mpeg') || sourceMime.includes('mp3');
849
- if (!sourceSeemsMpeg) {
860
+ if (!this.#isConcatenableContainerMime(source.mime_type)) {
850
861
  return false;
851
862
  }
852
863
 
853
864
  return chunks.every((chunk) => {
854
865
  const mime = `${chunk.mime_type || source.mime_type || ''}`.toLowerCase();
855
- return mime.includes('mpeg') || mime.includes('mp3') || mime.includes('octet-stream');
866
+ return this.#isConcatenableContainerMime(mime) || mime.includes('octet-stream');
856
867
  });
857
868
  }
858
869
 
@@ -1056,12 +1067,34 @@ class RecordingManager {
1056
1067
  });
1057
1068
  }
1058
1069
 
1059
- #assertSequentialChunks(sourceKey, chunks) {
1060
- for (let index = 0; index < chunks.length; index += 1) {
1061
- if (Number(chunks[index].sequence_index) !== index) {
1062
- throw new Error(`Recording source "${sourceKey}" is missing chunk ${index}.`);
1070
+ #warnOnChunkGaps(sourceKey, chunks) {
1071
+ const missing = [];
1072
+ let expected = 0;
1073
+ for (const chunk of chunks) {
1074
+ const sequence = Number(chunk.sequence_index) || 0;
1075
+ while (expected < sequence) {
1076
+ missing.push(expected);
1077
+ expected += 1;
1078
+ }
1079
+ expected = sequence + 1;
1080
+ }
1081
+ if (missing.length > 0) {
1082
+ console.warn(
1083
+ `[Recordings] Source "${sourceKey}" is missing chunk(s) ${missing.slice(0, 20).join(', ')}` +
1084
+ `${missing.length > 20 ? ', …' : ''}; transcribing the audio that did arrive.`,
1085
+ );
1086
+ }
1087
+ }
1088
+
1089
+ #maxOf(values, seed = 0) {
1090
+ let max = Number(seed) || 0;
1091
+ for (const value of (Array.isArray(values) ? values : [])) {
1092
+ const num = Number(value) || 0;
1093
+ if (num > max) {
1094
+ max = num;
1063
1095
  }
1064
1096
  }
1097
+ return max;
1065
1098
  }
1066
1099
 
1067
1100
  #shouldTranscribeSource(source, metadata = {}) {
@@ -131,32 +131,50 @@ class RuntimeHttpClient {
131
131
  }
132
132
 
133
133
  async requestStream(method, pathname, stream, options = {}) {
134
- const response = await fetch(`${this.baseUrl}${pathname}`, {
135
- method,
136
- headers: {
137
- ...(options.contentType ? { 'content-type': options.contentType } : {}),
138
- ...(options.contentLength != null ? { 'content-length': String(options.contentLength) } : {}),
139
- ...(options.headers || {}),
140
- ...(this.token ? { authorization: `Bearer ${this.token}` } : {}),
141
- },
142
- body: stream,
143
- duplex: 'half',
144
- });
134
+ const retryCount = Math.max(0, Number(options.retryCount ?? 6));
135
+ const retryDelayMs = Math.max(100, Number(options.retryDelayMs ?? 1000));
136
+ let lastError = null;
145
137
 
146
- if (response.ok && typeof this.onActivity === 'function') {
147
- this.onActivity();
148
- }
138
+ for (let attempt = 0; attempt <= retryCount; attempt += 1) {
139
+ try {
140
+ const response = await fetch(`${this.baseUrl}${pathname}`, {
141
+ method,
142
+ headers: {
143
+ ...(options.contentType ? { 'content-type': options.contentType } : {}),
144
+ ...(options.contentLength != null ? { 'content-length': String(options.contentLength) } : {}),
145
+ ...(options.headers || {}),
146
+ ...(this.token ? { authorization: `Bearer ${this.token}` } : {}),
147
+ },
148
+ body: stream,
149
+ duplex: 'half',
150
+ });
149
151
 
150
- const contentType = response.headers.get('content-type') || '';
151
- const payload = contentType.includes('application/json')
152
- ? await response.json().catch(() => ({}))
153
- : { text: await response.text().catch(() => '') };
152
+ if (response.ok && typeof this.onActivity === 'function') {
153
+ this.onActivity();
154
+ }
155
+
156
+ const contentType = response.headers.get('content-type') || '';
157
+ const payload = contentType.includes('application/json')
158
+ ? await response.json().catch(() => ({}))
159
+ : { text: await response.text().catch(() => '') };
154
160
 
155
- if (!response.ok) {
156
- const errorMessage = payload?.error || payload?.text || `Runtime request failed: ${response.status}`;
157
- throw new Error(errorMessage);
161
+ if (!response.ok) {
162
+ const errorMessage = payload?.error || payload?.text || `Runtime request failed: ${response.status}`;
163
+ throw new Error(errorMessage);
164
+ }
165
+ return payload;
166
+ } catch (error) {
167
+ lastError = error;
168
+ const message = String(error?.message || error);
169
+ const retryable = /fetch failed|ECONNREFUSED|ECONNRESET|socket hang up|timed out/i.test(message);
170
+ if (!retryable || attempt === retryCount) {
171
+ throw error;
172
+ }
173
+ await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
174
+ }
158
175
  }
159
- return payload;
176
+
177
+ throw lastError || new Error('Runtime request failed.');
160
178
  }
161
179
  }
162
180