@memberjunction/realtime-runtime 6.2.0-edge.4 → 6.2.0-edge.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/dist/hosts/IRealtimeMediaHost.d.ts +25 -30
  2. package/dist/hosts/IRealtimeMediaHost.d.ts.map +1 -1
  3. package/dist/index.d.ts +6 -2
  4. package/dist/index.d.ts.map +1 -1
  5. package/dist/index.js +5 -2
  6. package/dist/index.js.map +1 -1
  7. package/dist/session/RealtimeSessionRuntime.d.ts +372 -73
  8. package/dist/session/RealtimeSessionRuntime.d.ts.map +1 -1
  9. package/dist/session/RealtimeSessionRuntime.js +919 -212
  10. package/dist/session/RealtimeSessionRuntime.js.map +1 -1
  11. package/dist/session/RecordingShardDrain.d.ts +59 -0
  12. package/dist/session/RecordingShardDrain.d.ts.map +1 -0
  13. package/dist/session/RecordingShardDrain.js +135 -0
  14. package/dist/session/RecordingShardDrain.js.map +1 -0
  15. package/dist/session/RecordingShardQueue.d.ts +64 -38
  16. package/dist/session/RecordingShardQueue.d.ts.map +1 -1
  17. package/dist/session/RecordingShardQueue.js +106 -84
  18. package/dist/session/RecordingShardQueue.js.map +1 -1
  19. package/dist/session/RecordingShardStore.d.ts +57 -0
  20. package/dist/session/RecordingShardStore.d.ts.map +1 -0
  21. package/dist/session/RecordingShardStore.js +184 -0
  22. package/dist/session/RecordingShardStore.js.map +1 -0
  23. package/dist/session/RecordingShardWindow.d.ts +42 -0
  24. package/dist/session/RecordingShardWindow.d.ts.map +1 -0
  25. package/dist/session/RecordingShardWindow.js +88 -0
  26. package/dist/session/RecordingShardWindow.js.map +1 -0
  27. package/dist/session/RecordingTransport.d.ts +67 -0
  28. package/dist/session/RecordingTransport.d.ts.map +1 -0
  29. package/dist/session/RecordingTransport.js +61 -0
  30. package/dist/session/RecordingTransport.js.map +1 -0
  31. package/package.json +10 -9
@@ -5,8 +5,8 @@ import { AddRealtimeUsageRecord, AIEngineBase, HasRealtimeUsage } from '@memberj
5
5
  import { GraphQLRealtimeSessionClient } from '@memberjunction/graphql-dataprovider';
6
6
  import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
7
7
  import { DEFAULT_REALTIME_AUDIO_TRACKS, ParseRealtimeAvatarStatus, ParseRealtimeClientTransport } from '@memberjunction/ai';
8
- import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
9
- import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
8
+ import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, RECORDING_TERMINAL_MAX_BYTES, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
9
+ import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadElevenRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
10
10
  import { DefaultRealtimeSessionLauncher, HasClientCredential } from './session-launcher.js';
11
11
  import { RequestsAgentVideo, ResolveAvatarNotice } from './avatar-notice.js';
12
12
  import { ClientSessionDeadline } from './client-session-deadline.js';
@@ -23,6 +23,26 @@ import { AppClientToolRegistry, DEFAULT_APP_TOOL_OWNER } from './app-client-tool
23
23
  import { UserSettingsExposurePreferences } from './channel-exposure-preferences.js';
24
24
  import { BuildChannelCandidate, FindPreparedChannel, MergeToolMetadata, PreparedChannelsShowAgentVideo, ReconcileChannelsWithPolicy, ResolveLocalChannelScope, ToolsByChannelKey } from './channel-session-scope.js';
25
25
  import { RecordingShardQueue } from './RecordingShardQueue.js';
26
+ import { RecordingShardWindow } from './RecordingShardWindow.js';
27
+ import { CreateRecordingShardStore } from './RecordingShardStore.js';
28
+ import { CreateRecordingTransportForProvider, DeleteStoredRecordingShard, DrainStoredRecordingShardsOnce } from './RecordingShardDrain.js';
29
+ /**
30
+ * How long teardown waits for the server to acknowledge the recording's terminal request, measured
31
+ * from after `CloseAgentSession`. The terminal is `keepalive`, so it completes even if the page goes
32
+ * away; the wait only keeps `SavingRecording$` (and so a host's "don't close yet" guard) up while the
33
+ * server has not confirmed. Bounded so a dead server can never hold the End of a call hostage —
34
+ * unacknowledged shards stay in the local store and the server janitor assembles what arrived.
35
+ */
36
+ export const RECORDING_TERMINAL_ACK_TIMEOUT_MS = 3000;
37
+ /**
38
+ * How long teardown waits for the recorder to stop and flush its encoder. `Stop()` runs BEFORE
39
+ * `CloseAgentSession`, so a wedged encoder must not be able to hold the End of a call (and the
40
+ * server session) open; past this the recording ends at the packets already encoded.
41
+ */
42
+ export const RECORDER_STOP_TIMEOUT_MS = 2000;
43
+ function describeRecordingFailure(failure) {
44
+ return failure instanceof Error ? failure.message : String(failure);
45
+ }
26
46
  /**
27
47
  * `MJ: User Settings` key for the per-user "record this voice call" consent toggle. Stored as
28
48
  * the literal string `'true'`/`'false'` (read with `=== 'true'`), cross-device via
@@ -52,6 +72,7 @@ LoadGeminiEnterpriseRealtimeClient();
52
72
  LoadElevenLabsRealtimeClient();
53
73
  LoadAssemblyAIRealtimeClient();
54
74
  LoadxAIRealtimeClient();
75
+ LoadElevenRealtimeClient();
55
76
  LoadHuggingFaceRealtimeClient();
56
77
  /**
57
78
  * A registry row's surface placement. A row whose `UIConfig` is not valid JSON places its surface as a row without one
@@ -149,6 +170,14 @@ function microphoneStartError(reason, message) {
149
170
  error.name = MICROPHONE_ERROR_NAMES[reason];
150
171
  return error;
151
172
  }
173
+ /** Thrown by a start while a call is live: the runtime holds one driver, and never replaces a live one. */
174
+ export class RealtimeSessionAlreadyActiveError extends Error {
175
+ constructor(ActiveAgentSessionId) {
176
+ super(`A realtime session is already active${ActiveAgentSessionId ? ` (${ActiveAgentSessionId})` : ''}; end it before starting another.`);
177
+ this.ActiveAgentSessionId = ActiveAgentSessionId;
178
+ this.name = 'RealtimeSessionAlreadyActiveError';
179
+ }
180
+ }
152
181
  /**
153
182
  * Drives a **client-direct** real-time voice session: the browser mints an ephemeral
154
183
  * token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
@@ -174,9 +203,9 @@ function microphoneStartError(reason, message) {
174
203
  export class RealtimeSessionRuntime {
175
204
  /**
176
205
  * @param mediaHost the platform's media capabilities. The runtime never touches `navigator`,
177
- * `Blob` or `FileReader` itself — microphone acquisition and audio recording
178
- * are the host's, because both are platform-specific product decisions
179
- * (permission UX, container format, where the bytes live).
206
+ * `Blob` or `FileReader` itself — microphone acquisition and audio capture/encoding
207
+ * are the host's, because both are platform-specific (permission UX, which
208
+ * encoder the platform offers).
180
209
  */
181
210
  constructor(mediaHost) {
182
211
  this.mediaHost = mediaHost;
@@ -254,10 +283,11 @@ export class RealtimeSessionRuntime {
254
283
  this.Minimized$ = this._minimized$.asObservable();
255
284
  /**
256
285
  * True while the end-of-call recording is being saved: from the first statement of a teardown
257
- * that holds a recorder until that teardown settles. False at construction and for unrecorded
258
- * sessions. Exists for #5195: the page must stay open until the consolidated upload lands, or
259
- * the recording is lost. The runtime is platform-free, so hosts own the reaction (a browser
260
- * `beforeunload` guard, a "saving" indicator, etc.).
286
+ * that holds a recorder until that teardown settles — which includes waiting (at most
287
+ * {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}) for the server to acknowledge the recording's terminal
288
+ * request. False at construction and for unrecorded sessions. Exists for #5195; the runtime is
289
+ * platform-free, so hosts own the reaction (a "saving" indicator, a disabled navigation button,
290
+ * etc.).
261
291
  */
262
292
  this.SavingRecording$ = this._savingRecording$.asObservable();
263
293
  /**
@@ -341,8 +371,8 @@ export class RealtimeSessionRuntime {
341
371
  * Fired EXACTLY ONCE per session as teardown begins, with the prior
342
372
  * `agentSessionId` (so subscribers can correlate against `SessionStarted$`'s
343
373
  * sessionId) and the client-distinguishable reason — `'explicit'` when the
344
- * user called `EndRealtimeSession`, `'error'` when teardown ran from a catch
345
- * block. Server-side close paths (janitor, shutdown) do NOT propagate here —
374
+ * user called `EndRealtimeSession`, `'error'` when the runtime ended the call over a failure: a
375
+ * failed start, or a fatal error from the provider mid-call. Server-side close paths (janitor, shutdown) do NOT propagate here —
346
376
  * they happen out-of-process and have no client push channel today.
347
377
  */
348
378
  this.SessionEnded$ = this._sessionEnded$.asObservable();
@@ -423,23 +453,40 @@ export class RealtimeSessionRuntime {
423
453
  // ── Browser-side call recording ────────────────────────────────────────────
424
454
  /**
425
455
  * The active session's audio recorder (mic + agent mix), or `null` when the user didn't
426
- * consent or the browser can't record. Created after the client connects; stopped + uploaded
427
- * at teardown.
456
+ * consent or the browser can't record. Created after the client connects; its packets are
457
+ * shipped as 5 s shards during the call, and it is stopped at teardown.
428
458
  */
429
459
  this.recorder = null;
430
460
  /** ISO timestamp of when recording started — sent to the server on session start. */
431
461
  this.recordingStartedAtIso = null;
432
- /** Interval that flushes ~15s crash-recovery shards to the server during a recording. */
433
- this.segmentTimer = null;
462
+ /** Interval that cuts and uploads a shard every {@link recordingShardFlushMs} during a recording. */
463
+ this.shardFlushTimer = null;
464
+ /**
465
+ * The current recording's shard pipeline (window → store → queue → transport), or null when not
466
+ * recording. A fresh pipeline per recording keeps a flush still in flight from a previous session
467
+ * off this session's indexes. Its queue is closed when the recording ends; anything it had not
468
+ * delivered stays in the local store for the next-load drain.
469
+ */
470
+ this.recordingShards = null;
471
+ /**
472
+ * Bytes of this page's recording `keepalive` requests still in flight. The browser's keepalive budget
473
+ * is per page, and closing a tab fires `visibilitychange` and `pagehide` back to back, so each dispatch
474
+ * budgets against what the previous one has not finished sending.
475
+ */
476
+ this.keepaliveBytesInFlight = 0;
477
+ /** `sessionId/index` of shards whose `keepalive` copy is still in flight; never sent twice at once. */
478
+ this.keepaliveShardsInFlight = new Set();
479
+ /** The durable local shard store, created on first use and shared by every recording of this runtime. */
480
+ this.shardStore = null;
434
481
  /**
435
- * Owns shard indexing and retry for the current recording: it assigns each shard its index and
436
- * keeps failed shards to resend under that same index. A fresh queue per recording keeps a flush
437
- * still in flight from a previous session off this session's indexes. Null when not recording.
438
- * A recording that ends discards its queue, and any retained shard with it: the consolidated
439
- * end-of-call upload supersedes them, and if that upload is lost too, server-side recovery
440
- * (#5196) fills the gap with silence.
482
+ * True once a drain ran with a transport; the drain then does nothing until a recording ends and
483
+ * re-arms it ({@link rearmPendingShardDrain}).
441
484
  */
442
- this.shardQueue = null;
485
+ this.pendingShardsDrained = false;
486
+ /** Bumped by every re-arm, so a drain already in flight cannot mark itself done over a re-arm. */
487
+ this.drainGeneration = 0;
488
+ /** The drain in flight, so overlapping callers share it instead of uploading the same shards twice. */
489
+ this.drainInFlight = null;
443
490
  // ── Server-side liveness ───────────────────────────────────────────────────
444
491
  /**
445
492
  * Interval that tells the server this session is still in use, or `null` when no session is
@@ -450,12 +497,40 @@ export class RealtimeSessionRuntime {
450
497
  * Recording-relative ms offset at which the IN-FLIGHT (not-yet-finalized) turn's audio
451
498
  * actually BEGAN — captured the moment that turn's audio/text starts flowing (its first
452
499
  * interim transcript), NOT inherited from the previous turn's end. `null` before the first
453
- * turn / between turns (until the next turn's audio starts). Sent as `utteranceStartMs` on
454
- * the turn's final transcript so per-turn timing lines up with the recording even when a
455
- * tool-call / silence gap sits between turns (the inherit-previous-end model mis-stamped
456
- * the post-gap turn at the pre-gap offset). See {@link markTurnAudioStart}.
500
+ * turn / between turns (until the next turn's audio starts), and it stays `null` for a turn
501
+ * whose start was never measured. Sent as `utteranceStartMs` on the turn's final transcript
502
+ * AS-IS — `null` is relayed as `null`, never replaced with 0 — so per-turn timing lines up
503
+ * with the recording even when a tool-call / silence gap sits between turns (the
504
+ * inherit-previous-end model mis-stamped the post-gap turn at the pre-gap offset). See
505
+ * {@link markTurnAudioStart}. A user speech onset signalled by the client does NOT land here;
506
+ * it has its own slot, {@link userSpeechStartMs}.
457
507
  */
458
508
  this.currentTurnStartMs = null;
509
+ /**
510
+ * Recording-relative ms offset of the first user speech onset the client signalled
511
+ * (`OnUserSpeechStart`) since the last final user transcript, or `null` when none was signalled.
512
+ * It has its own slot, apart from {@link currentTurnStartMs}, so a user onset never claims the
513
+ * assistant turn's start: OpenAI user transcripts arrive asynchronously, often after the
514
+ * assistant's interims (or its final) for the reply have already flowed. See
515
+ * {@link markUserSpeechStart}.
516
+ */
517
+ this.userSpeechStartMs = null;
518
+ /**
519
+ * `true` once an assistant turn has finalized after {@link userSpeechStartMs} was stamped BEFORE
520
+ * that turn began. The held onset may still be claimed by a late user transcript (OpenAI
521
+ * transcribes asynchronously), but the NEXT onset replaces it: an onset no transcript ever claimed
522
+ * (a VAD false positive, a failed transcription) must not become the start of the next real user
523
+ * turn. A barge-in onset (stamped during the assistant turn) is not superseded by that turn's
524
+ * final. An EMPTY user transcript releases the onset outright (see onClientTranscript).
525
+ */
526
+ this.userSpeechStartSuperseded = false;
527
+ /**
528
+ * Recording-relative ms offsets of user speech onsets keyed by the provider's conversation item id
529
+ * (`input_audio_buffer.speech_started.item_id`), paired with the matching final user transcript
530
+ * (`RealtimeClientTranscript.ItemId`) so a late user transcript never claims a later turn's onset
531
+ * (MJ#5902). Unkeyed drivers continue to use {@link userSpeechStartMs}.
532
+ */
533
+ this.userSpeechStartByItemId = new Map();
459
534
  /**
460
535
  * Wall-anchor of the SESSION clock (#3832): `performance.now()` at the moment the call went
461
536
  * live, or `null` before any call has. Read only through {@link nowTurnOffsetMs}.
@@ -540,6 +615,15 @@ export class RealtimeSessionRuntime {
540
615
  * two `SessionEnded$` emissions for one session.
541
616
  */
542
617
  this.teardownInFlight = null;
618
+ /**
619
+ * True from the moment a start finishes (connected, liveness pulse running) until the slot is
620
+ * released. A driver's Fatal ends the call only while this is set: during a start, drivers emit
621
+ * Fatal from their socket handlers while Connect is still pending, and the start's own failure path
622
+ * ({@link failSessionStart}) is what reports that error, whether Connect then rejects or resolves
623
+ * (runMintedSession checks {@link lastSessionError} after Connect). Not {@link isSessionLive}: drivers move the
624
+ * state to 'listening' from inside Connect, and a live call that is resuming reads 'connecting'.
625
+ */
626
+ this.callWentLive = false;
543
627
  /**
544
628
  * The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
545
629
  * back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
@@ -662,6 +746,16 @@ export class RealtimeSessionRuntime {
662
746
  * Owner-keyed — see {@link AppClientToolRegistry}.
663
747
  */
664
748
  this.appToolRegistry = new AppClientToolRegistry();
749
+ // Re-upload shards a previous, interrupted call left in the local store. Deferred a tick so a
750
+ // host can assign `Provider` right after construction; when it still has no transport the drain
751
+ // defers itself and session start tries again. Unref'd so it never holds a Node process open.
752
+ const drainTimer = setTimeout(() => {
753
+ void this.DrainPendingRecordingShards();
754
+ }, 0);
755
+ // Node returns a Timeout object with unref(); browsers return a number.
756
+ if (typeof drainTimer === 'object' && drainTimer !== null && 'unref' in drainTimer && typeof drainTimer.unref === 'function') {
757
+ drainTimer.unref();
758
+ }
665
759
  }
666
760
  /** Synchronous read of {@link SavingRecording$}. */
667
761
  get IsSavingRecording() {
@@ -730,8 +824,20 @@ export class RealtimeSessionRuntime {
730
824
  UpdateAppContext(snapshot) {
731
825
  this._appContext$.next(snapshot);
732
826
  }
733
- /** How often crash-recovery shards are flushed during a recording. */
734
- static { this.segmentFlushMs = 15000; }
827
+ /** How often packets are cut into shards: the 5 s shard window, so each tick seals about one window. */
828
+ static { this.recordingShardFlushMs = 5000; }
829
+ /**
830
+ * Bytes of `keepalive` request bodies one terminal may use, the tail included. Browsers share about
831
+ * 64 KiB of in-flight keepalive bodies per page; the brief reserves 48 KiB of it for the recording,
832
+ * leaving room for a host's own close beacon (Caliber sends one from `pagehide` too).
833
+ */
834
+ static { this.recordingKeepaliveBudgetBytes = RECORDING_TERMINAL_MAX_BYTES; }
835
+ /**
836
+ * Most runs one drain call makes (each run is itself bounded): it runs again when it was re-armed while
837
+ * running, or when a run deleted shards the server refused for good, which frees the page for the
838
+ * shards behind them.
839
+ */
840
+ static { this.maxDrainRunsPerCall = 3; }
735
841
  /**
736
842
  * How often the client asserts liveness. Comfortably under `SessionJanitor`'s
737
843
  * `closeThresholdMinutes` (15 by default) so several pulses must be missed in a row before a
@@ -873,6 +979,25 @@ export class RealtimeSessionRuntime {
873
979
  IsActiveFor(conversationId) {
874
980
  return this.IsActive && conversationId != null && UUIDsEqual(this.sessionConversationId, conversationId);
875
981
  }
982
+ /**
983
+ * Takes the runtime's one driver slot for a new start; the two starts call it after awaiting any
984
+ * teardown in flight, and only when no End arrived during that wait (a changed {@link startGeneration}).
985
+ *
986
+ * A teardown still in flight is awaited rather than read as "busy": ending a call and starting the
987
+ * next one back to back is normal host behaviour (a retry, a re-mounted surface), and the flag it
988
+ * used to read flips only at the END of teardown, so the start was dropped with nothing to show
989
+ * for it — a bare `undefined`, which hosts had to guess at (MJ#4629). A call that is genuinely
990
+ * live is refused with an error the host can show; it is never silently ignored, and never
991
+ * overwritten, since a driver no slot points at can never be stopped again.
992
+ *
993
+ * Deliberately synchronous: the start's state must be set in the tick the host called it in, or an
994
+ * End issued straight after the Start (a user tapping back during the mint) finds nothing to abandon.
995
+ */
996
+ claimSessionSlot() {
997
+ if (this.IsActive) {
998
+ throw new RealtimeSessionAlreadyActiveError(this.agentSessionId);
999
+ }
1000
+ }
876
1001
  /**
877
1002
  * Start a client-direct voice session fronting `targetAgentId`.
878
1003
  *
@@ -903,7 +1028,7 @@ export class RealtimeSessionRuntime {
903
1028
  * pickers, and never synthesize overrides beyond what the user explicitly chose.
904
1029
  * Omit/`null` for the server's defaults (today's behavior).
905
1030
  * @param recordingConsent Optional EXPLICIT "record this call" consent for THIS session. When
906
- * `true`, the browser records a mic + agent-audio mix and uploads it at session end. When
1031
+ * `true`, the browser records a mic + agent-audio mix and uploads it as 5 s Opus shards during the call. When
907
1032
  * omitted/`null`, the per-user persisted preference (`mj.realtimeVoice.recordingConsent.v1`
908
1033
  * via {@link UserInfoEngine}) is read as the default. `false` never records.
909
1034
  * @param mediaCollectionId Optional per-session media-kit override (`MJ: Collections.ID`). When set,
@@ -919,13 +1044,22 @@ export class RealtimeSessionRuntime {
919
1044
  * ({@link RealtimeSessionStartOptions.ConversationBranchID}; omitted is Main, where its transcript is written).
920
1045
  */
921
1046
  async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext, options) {
922
- if (this.IsActive) {
923
- return; // a session is already running — ignore duplicate starts
1047
+ // Captured BEFORE any wait: a teardown from here on bumps it, and the checks below (and in
1048
+ // runMintedSession) abandon the start. Re-capturing after the wait would absorb an End issued during it.
1049
+ const generation = this.startGeneration;
1050
+ // Synchronous unless a teardown is in flight: a start must take the slot in the same tick it was
1051
+ // called in, so a host's End issued right after it finds a start to abandon (see claimSessionSlot).
1052
+ if (this.teardownInFlight) {
1053
+ await this.teardownInFlight;
1054
+ // An End issued during the wait found IsActive still true (the last call was closing), so it only
1055
+ // joined that teardown and bumped the generation. Nothing is acquired yet, so nothing to release.
1056
+ if (this.startGeneration !== generation) {
1057
+ return;
1058
+ }
924
1059
  }
1060
+ this.claimSessionSlot();
925
1061
  const consent = this.beginSessionStart({ agentName, recordingConsent, applicationId, appContext });
926
1062
  this.sessionConversationId = conversationId ?? null;
927
- // A teardown while the channels are prepared or the mint runs bumps this; see the checks below.
928
- const generation = this.startGeneration;
929
1063
  // Captured BEFORE the channels are prepared so the mint carries exactly the snapshot the prologue
930
1064
  // resolved, whatever a channel plugin may push in the meantime.
931
1065
  const effectiveAppContext = this._appContext$.value;
@@ -951,14 +1085,14 @@ export class RealtimeSessionRuntime {
951
1085
  // The host ended the start while the mint ran. That teardown had no session id to close, so
952
1086
  // close the minted row here, and touch no shared state: a newer start may already own it.
953
1087
  if (this.startGeneration !== generation) {
954
- await this.closeServerSession(session.AgentSessionId);
1088
+ await this.closeServerSession(session.AgentSessionId, 'Explicit');
955
1089
  return;
956
1090
  }
957
1091
  if (!this.hostCanUseProvider(session.Provider)) {
958
1092
  await this.abortUnusableSession(session);
959
1093
  return;
960
1094
  }
961
- await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true);
1095
+ await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true, generation);
962
1096
  }
963
1097
  /**
964
1098
  * Declares which provider keys this host can actually carry audio for.
@@ -994,8 +1128,7 @@ export class RealtimeSessionRuntime {
994
1128
  // published them on ActiveChannels$; skipping teardown would leave them undisposed, their tool
995
1129
  // handlers registered, and their subscriptions live until the next start replaced them.
996
1130
  this.agentSessionId = session.AgentSessionId ?? this.agentSessionId;
997
- this.setConnectionState('error');
998
- await this.teardown(true);
1131
+ await this.endOnFailure();
999
1132
  }
1000
1133
  /**
1001
1134
  * Run a session the HOST has already minted itself — the second half of
@@ -1019,22 +1152,28 @@ export class RealtimeSessionRuntime {
1019
1152
  * Every field defaults exactly as its {@link StartRealtimeSession} counterpart does.
1020
1153
  */
1021
1154
  async StartRealtimeSessionFromResult(result, options) {
1022
- if (this.IsActive) {
1023
- return; // a session is already running — ignore duplicate starts
1155
+ // Same wait, same generation check, same reasons as StartRealtimeSession.
1156
+ const generation = this.startGeneration;
1157
+ if (this.teardownInFlight) {
1158
+ await this.teardownInFlight;
1159
+ if (this.startGeneration !== generation) {
1160
+ return;
1161
+ }
1024
1162
  }
1163
+ this.claimSessionSlot();
1025
1164
  const effectiveOptions = options ?? {};
1026
1165
  if (!this.hostCanUseProvider(result.Provider)) {
1027
1166
  await this.abortUnusableSession(result);
1028
1167
  return;
1029
1168
  }
1030
1169
  const consent = this.beginSessionStart(effectiveOptions);
1031
- await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false);
1170
+ await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false, generation);
1032
1171
  }
1033
1172
  /**
1034
1173
  * Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
1035
1174
  * publish the agent name, reset per-session state, and flip the session live (which is ALSO what
1036
- * makes the `IsActive` guard suppress duplicate starts while the mint is still in flight — hence it
1037
- * runs before minting, not after). Returns the resolved recording consent, which the mint half
1175
+ * makes {@link claimSessionSlot} refuse a second start with {@link RealtimeSessionAlreadyActiveError}
1176
+ * while the mint is still in flight — hence it runs before minting, not after). Returns the resolved recording consent, which the mint half
1038
1177
  * reports to the server and the run half uses to decide whether to record.
1039
1178
  */
1040
1179
  beginSessionStart(options) {
@@ -1060,6 +1199,9 @@ export class RealtimeSessionRuntime {
1060
1199
  this.resetState();
1061
1200
  this._active$.next(true);
1062
1201
  this.setConnectionState('connecting');
1202
+ // A drain the constructor deferred (no provider/transport yet) gets its second chance here, and
1203
+ // runs whether or not THIS session records: the pending shards belong to an earlier call.
1204
+ void this.DrainPendingRecordingShards();
1063
1205
  // Resolve recording consent for this session: explicit value wins, else the per-user
1064
1206
  // persisted preference. Computed before mint so it can be reported to the server.
1065
1207
  const consent = options.recordingConsent ?? this.readPersistedRecordingConsent();
@@ -1071,10 +1213,10 @@ export class RealtimeSessionRuntime {
1071
1213
  * the provider connection, and go live. `inputConversationId` is the conversation the START asked
1072
1214
  * for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
1073
1215
  * whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
1216
+ * `generation` is the {@link startGeneration} the entry point captured before its first await; every
1217
+ * await below is a window in which the host can end the session, and so was every await before this.
1074
1218
  */
1075
- async runMintedSession(session, inputConversationId, consent, cameraCheck) {
1076
- // Captured up front: every await below is a window in which the host can end the session.
1077
- const generation = this.startGeneration;
1219
+ async runMintedSession(session, inputConversationId, consent, cameraCheck, generation) {
1078
1220
  try {
1079
1221
  // Mount the channels the resolved policy puts in the session. Deliberately BEFORE the session
1080
1222
  // id is adopted: `ActiveChannels$` consumers (the sessions adapter) treat an emission that
@@ -1123,6 +1265,13 @@ export class RealtimeSessionRuntime {
1123
1265
  if (connectError) {
1124
1266
  throw connectError;
1125
1267
  }
1268
+ // A Fatal the driver sent during this start (the fatal handler leaves it to the start, see
1269
+ // callWentLive) fails the start even when Connect resolved: the real drivers keep awaiting mic
1270
+ // capture after their connect-failure channel settles, then report 'listening' and resolve on a
1271
+ // socket that already closed. lastSessionError is reset per start and set only by the current client.
1272
+ if (this.lastSessionError?.Fatal) {
1273
+ throw new Error(this.lastSessionError.Message);
1274
+ }
1126
1275
  // Tracks are negotiated now, so a capture can tell whether the model takes video, and the call whether it shows
1127
1276
  // the avatar its agent asked for. The driver holds the microphone now, so the user may switch it.
1128
1277
  this.openCaptures(client, cameraCheck);
@@ -1174,6 +1323,7 @@ export class RealtimeSessionRuntime {
1174
1323
  // Same place, same reason: the session is connected and its id is known, which is exactly
1175
1324
  // the window in which the server needs to be told it is alive.
1176
1325
  this.startLivenessPulse();
1326
+ this.callWentLive = true;
1177
1327
  }
1178
1328
  catch (error) {
1179
1329
  await this.failSessionStart(error);
@@ -1235,7 +1385,7 @@ export class RealtimeSessionRuntime {
1235
1385
  // session, and clearing the id would leave that teardown no session to report.
1236
1386
  if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId && !this.teardownInFlight) {
1237
1387
  this.agentSessionId = null;
1238
- await this.closeServerSession(session.AgentSessionId);
1388
+ await this.closeServerSession(session.AgentSessionId, 'Explicit');
1239
1389
  }
1240
1390
  }
1241
1391
  /**
@@ -1546,8 +1696,20 @@ export class RealtimeSessionRuntime {
1546
1696
  async failSessionStart(error) {
1547
1697
  console.error('[RealtimeSession] Failed to start session:', error);
1548
1698
  this.lastStartError = error instanceof Error ? error : new Error(String(error));
1699
+ await this.endOnFailure();
1700
+ }
1701
+ /**
1702
+ * Ends the call over a failure: a failed start, an unusable provider, or a fatal provider error.
1703
+ *
1704
+ * The teardown is claimed BEFORE `'error'` is published. `ConnectionState$` emits synchronously, and a
1705
+ * host may end the call the moment it sees `'error'` (MobileApp's voice screen does); claimed first, that
1706
+ * host's End joins this teardown instead of starting its own as `'explicit'`, so the row still closes as
1707
+ * `Error` and `SessionEnded$` still says `'error'`. Teardown keeps a terminal `'error'` state.
1708
+ */
1709
+ endOnFailure() {
1710
+ const ending = this.teardown('error');
1549
1711
  this.setConnectionState('error');
1550
- await this.teardown(false);
1712
+ return ending;
1551
1713
  }
1552
1714
  /**
1553
1715
  * End the active session: stop the mic, tear down the provider connection, and close
@@ -1557,7 +1719,7 @@ export class RealtimeSessionRuntime {
1557
1719
  if (!this.IsActive && !this.agentSessionId) {
1558
1720
  return;
1559
1721
  }
1560
- await this.teardown(true);
1722
+ await this.teardown('explicit');
1561
1723
  }
1562
1724
  /**
1563
1725
  * Inject a typed message into the live session as a user turn.
@@ -1583,7 +1745,7 @@ export class RealtimeSessionRuntime {
1583
1745
  }
1584
1746
  client.SendText(trimmed);
1585
1747
  // Relay as a user turn — same path spoken input uses (caption + persisted transcript).
1586
- void this.onUserTranscript(trimmed);
1748
+ void this.onUserTranscript(trimmed, 'typed');
1587
1749
  }
1588
1750
  /**
1589
1751
  * Mute / unmute the local microphone track. Returns the new muted state. While the call has no microphone track (a
@@ -1648,7 +1810,17 @@ export class RealtimeSessionRuntime {
1648
1810
  if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
1649
1811
  return false;
1650
1812
  }
1651
- this.requestChannelSpokenResponse(trimmed);
1813
+ // The opening is part of the conversation, so it goes out as a spoken TURN: captioned, relayed
1814
+ // and persisted like any turn the model takes on its own. It used to ride the delegation
1815
+ // narration channel, whose replies are disposable by contract, so the greeting the participant
1816
+ // heard was never in the transcript (MJ#4591). A driver from before the capability existed
1817
+ // falls back to that channel rather than leaving the session silent.
1818
+ if (this.client.RequestSpokenTurn) {
1819
+ this.client.RequestSpokenTurn(trimmed);
1820
+ }
1821
+ else {
1822
+ this.client.RequestSpokenUpdate(trimmed);
1823
+ }
1652
1824
  return true;
1653
1825
  }
1654
1826
  /**
@@ -1728,6 +1900,9 @@ export class RealtimeSessionRuntime {
1728
1900
  // turn one's cue wherever the clocks happen to differ.
1729
1901
  this.currentTurnStartMs = recorder.IsRecording ? 0 : null;
1730
1902
  this.turnAudioStartCaptured = false;
1903
+ this.userSpeechStartMs = null;
1904
+ this.userSpeechStartSuperseded = false;
1905
+ this.userSpeechStartByItemId.clear();
1731
1906
  if (this.recorder) {
1732
1907
  // WebRTC drivers (OpenAI): the agent's track usually lands AFTER Connect() resolves, so
1733
1908
  // `remoteStream` above is null here and this handler attaches it later. PCM-playback
@@ -1735,7 +1910,7 @@ export class RealtimeSessionRuntime {
1735
1910
  // `remoteStream` is already set and the handler fires immediately with the same stream;
1736
1911
  // AttachRemoteStream is idempotent, so it is mixed only once.
1737
1912
  client.OnRemoteMediaStream?.((stream) => this.recorder?.AttachRemoteStream(stream));
1738
- this.startSegmentFlushing();
1913
+ this.startShardFlushing();
1739
1914
  }
1740
1915
  }
1741
1916
  catch (error) {
@@ -1743,16 +1918,161 @@ export class RealtimeSessionRuntime {
1743
1918
  this.recorder = null;
1744
1919
  }
1745
1920
  }
1746
- /** Begins flushing ~15s crash-recovery shards to the server for the duration of the recording. */
1747
- startSegmentFlushing() {
1748
- this.shardQueue = new RecordingShardQueue();
1749
- this.segmentTimer = setInterval(() => { void this.flushRecordingSegment(); }, RealtimeSessionRuntime.segmentFlushMs);
1921
+ // ── Recording shards (MJ#5294) ───────────────────────────────────────────────
1922
+ //
1923
+ // The recorder hands over Opus packets; the runtime cuts them into 5 s `MJOPUS1` shards, writes
1924
+ // each one to the local store BEFORE its upload starts, uploads it as raw bytes, and deletes it from
1925
+ // the store only on the server's ack. The call ends with a `keepalive` terminal request carrying the
1926
+ // final partial shard and the counts the server needs to know it has everything.
1927
+ /**
1928
+ * Builds the transport for shard uploads and terminal requests. The default posts to MJAPI's REST
1929
+ * recording routes, derived from the session's GraphQL provider configuration, with credentials
1930
+ * read per request so a refreshed token is used. Returns null — shards then wait in the local
1931
+ * store — when the provider has no URL (a test double, a provider that is not GraphQL, or one not
1932
+ * configured yet). Override to supply another transport.
1933
+ */
1934
+ createRecordingTransport() {
1935
+ return CreateRecordingTransportForProvider(this.Provider);
1936
+ }
1937
+ /**
1938
+ * Builds the durable local shard store. The default is IndexedDB where the host has it and a no-op
1939
+ * store elsewhere (Node, some React Native runtimes). Called once per runtime. Override to supply
1940
+ * another store.
1941
+ */
1942
+ createRecordingShardStore() {
1943
+ return CreateRecordingShardStore(globalThis.indexedDB);
1944
+ }
1945
+ /** The provider's signed-in user, or null (no provider yet, or an anonymous page). */
1946
+ signedInUserID() {
1947
+ return this.Provider?.CurrentUser?.ID ?? null;
1948
+ }
1949
+ /** The runtime's shard store, created on first use. */
1950
+ ensureShardStore() {
1951
+ this.shardStore ??= this.createRecordingShardStore();
1952
+ return this.shardStore;
1953
+ }
1954
+ /**
1955
+ * Re-uploads shards an earlier, interrupted call left in the local store (crash, closed tab, dead
1956
+ * network), each under its OWN session id and index, oldest first, at most
1957
+ * {@link MAX_DRAINED_SHARDS_PER_LOAD} per runtime. A shard is deleted only on the server's ack.
1958
+ *
1959
+ * Runs once per runtime instance, re-armed whenever a recording ends (which may leave shards behind
1960
+ * for the next session start on the same page). When there is no transport yet (no provider configuration, e.g.
1961
+ * before login) it returns 0 WITHOUT marking itself done, so a later call — session start — drains
1962
+ * instead of the shards being skipped for this page. Concurrent calls share one run. Never rejects.
1963
+ *
1964
+ * @returns the number of shards the server acknowledged in this call.
1965
+ */
1966
+ DrainPendingRecordingShards() {
1967
+ if (this.pendingShardsDrained) {
1968
+ return Promise.resolve(0);
1969
+ }
1970
+ this.drainInFlight ??= this.drainPendingShards().finally(() => {
1971
+ this.drainInFlight = null;
1972
+ });
1973
+ return this.drainInFlight;
1974
+ }
1975
+ /** Makes the next {@link DrainPendingRecordingShards} call run again, even one already in flight. */
1976
+ rearmPendingShardDrain() {
1977
+ this.pendingShardsDrained = false;
1978
+ this.drainGeneration++;
1979
+ }
1980
+ /**
1981
+ * The body of {@link DrainPendingRecordingShards}, up to {@link maxDrainRunsPerCall} runs. It runs
1982
+ * again after a re-arm while a run is in flight (a recovery after an outage gave up on more shards),
1983
+ * because the running pass listed the store before those shards were there; and after a run deleted
1984
+ * refused shards, because those filled the page that the shards behind them now get.
1985
+ */
1986
+ async drainPendingShards() {
1987
+ try {
1988
+ const transport = this.createRecordingTransport();
1989
+ if (!transport) {
1990
+ return 0;
1991
+ }
1992
+ let delivered = 0;
1993
+ for (let run = 1; run <= RealtimeSessionRuntime.maxDrainRunsPerCall; run++) {
1994
+ // A re-arm while this run is in flight must win over this run marking itself done.
1995
+ const generation = this.drainGeneration;
1996
+ const outcome = await this.drainStoredShardsOnce(transport);
1997
+ delivered += outcome.Delivered;
1998
+ // Done only when every listed shard was delivered or refused for good: a drain during an outage
1999
+ // must not stop the next session start or recovery from trying again.
2000
+ this.pendingShardsDrained = generation === this.drainGeneration && outcome.Delivered + outcome.Refused === outcome.Listed;
2001
+ if (generation === this.drainGeneration && outcome.Refused === 0) {
2002
+ break;
2003
+ }
2004
+ }
2005
+ return delivered;
2006
+ }
2007
+ catch (error) {
2008
+ console.warn(`[RealtimeSession] Draining stored recording shards failed: ${describeRecordingFailure(error)}; will retry at the next session start.`, error);
2009
+ return 0;
2010
+ }
2011
+ }
2012
+ /** One drain pass over the signed-in user's stored shards (see {@link DrainStoredRecordingShardsOnce}). */
2013
+ drainStoredShardsOnce(transport) {
2014
+ return DrainStoredRecordingShardsOnce(this.ensureShardStore(), transport, this.signedInUserID());
2015
+ }
2016
+ /** Builds this recording's shard pipeline and starts the 5 s flush. */
2017
+ startShardFlushing() {
2018
+ const sessionId = this.agentSessionId;
2019
+ if (!sessionId) {
2020
+ // Unreachable in practice (the session id is adopted before recording starts), but without
2021
+ // one there is nowhere to send shards, so say so rather than recording into the void.
2022
+ console.warn('[RealtimeSession] Recording started without an agent session id; shards will not be saved.');
2023
+ return;
2024
+ }
2025
+ const transport = this.createRecordingTransport();
2026
+ if (!transport) {
2027
+ console.warn(`[RealtimeSession] No recording transport for session ${sessionId} (provider has no REST configuration); shards are kept in the local store only.`);
2028
+ }
2029
+ // True once the queue gave up on a shard and until an upload succeeds again: that first success
2030
+ // re-sends what the queue dropped meanwhile (see recoverStoredShardsAfterOutage).
2031
+ let shardsGivenUp = false;
2032
+ this.recordingShards = {
2033
+ SessionID: sessionId,
2034
+ UserID: this.signedInUserID(),
2035
+ Window: new RecordingShardWindow(),
2036
+ Transport: transport,
2037
+ // Closes over this recording's session id and transport — never the runtime's current ones.
2038
+ Queue: new RecordingShardQueue(async (shard) => {
2039
+ if (!transport) {
2040
+ return 'Retry';
2041
+ }
2042
+ // The outcome goes to the queue unchanged: it logs a different reason for each, and stops
2043
+ // retrying a `Refused` shard at once (#6012).
2044
+ const outcome = await transport.UploadShard(sessionId, shard);
2045
+ if (outcome === 'Acked') {
2046
+ await this.deleteStoredShard(sessionId, shard.Index);
2047
+ if (shardsGivenUp) {
2048
+ shardsGivenUp = false;
2049
+ this.recoverStoredShardsAfterOutage(sessionId);
2050
+ }
2051
+ }
2052
+ return outcome;
2053
+ }, () => {
2054
+ shardsGivenUp = true;
2055
+ }),
2056
+ };
2057
+ this.shardFlushTimer = setInterval(() => { void this.flushRecordingShards(); }, RealtimeSessionRuntime.recordingShardFlushMs);
2058
+ }
2059
+ /**
2060
+ * Uploads succeed again after failing: re-sends, from the local store, the shards the queue gave up
2061
+ * on during the outage. The queue's attempts run out after about 10 s of a dead network (or an
2062
+ * expired token), and without this those shards would wait for a later page load — which a widget
2063
+ * guest never makes — leaving a permanent gap. Runs in the background, outside any page-lifecycle
2064
+ * handler, so the IndexedDB read is allowed. Never rejects.
2065
+ */
2066
+ recoverStoredShardsAfterOutage(sessionId) {
2067
+ console.log(`[RealtimeSession] Recording uploads for session ${sessionId} recovered; re-sending stored shards.`);
2068
+ this.rearmPendingShardDrain();
2069
+ void this.DrainPendingRecordingShards();
1750
2070
  }
1751
- /** Stops the periodic crash-recovery shard flush. */
1752
- stopSegmentFlushing() {
1753
- if (this.segmentTimer) {
1754
- clearInterval(this.segmentTimer);
1755
- this.segmentTimer = null;
2071
+ /** Stops the periodic shard flush. */
2072
+ stopShardFlushing() {
2073
+ if (this.shardFlushTimer) {
2074
+ clearInterval(this.shardFlushTimer);
2075
+ this.shardFlushTimer = null;
1756
2076
  }
1757
2077
  }
1758
2078
  /**
@@ -1811,128 +2131,345 @@ export class RealtimeSessionRuntime {
1811
2131
  }
1812
2132
  }
1813
2133
  /**
1814
- * Uploads the chunks captured since the last flush as one crash-recovery shard (durability only;
1815
- * the canonical file is still the full upload at teardown). Best-effort — never disturbs the call:
1816
- * a failed shard is retained by the queue and resent under its own index on later flushes, up to a
1817
- * bounded number of attempts (#5197).
2134
+ * One 5 s tick: cuts the packets encoded since the last tick into shards, stores each one, then
2135
+ * uploads them. Best-effort — never disturbs the call and never rejects. Overlapping ticks are
2136
+ * safe: `Append` and `SealTail` are synchronous, so indexes are assigned in order, and the queue
2137
+ * bounds how many flushes upload at once. A failed shard is retained by the queue and resent under
2138
+ * its own index on later ticks (#5197), and stays in the store until acknowledged.
2139
+ *
2140
+ * Every tick seals whatever it took, even under a full window (the window's 250-packet and byte caps
2141
+ * still split a larger batch). Capture starts slightly after this timer, so a tick almost never
2142
+ * holds exactly 250 packets; sealing only full windows would leave nearly 5 s of encoded audio in
2143
+ * memory at every moment, unstored and unsent, which a killed tab loses.
1818
2144
  */
1819
- async flushRecordingSegment() {
2145
+ async flushRecordingShards() {
1820
2146
  const recorder = this.recorder;
1821
- const agentSessionId = this.agentSessionId;
1822
- const queue = this.shardQueue;
1823
- if (!recorder || !agentSessionId || !queue) {
2147
+ const pipeline = this.recordingShards;
2148
+ if (!recorder || !pipeline) {
2149
+ return;
2150
+ }
2151
+ let sealed;
2152
+ try {
2153
+ sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
2154
+ const tick = pipeline.Window.SealTail();
2155
+ if (tick) {
2156
+ sealed.push(tick);
2157
+ }
2158
+ }
2159
+ catch (error) {
2160
+ console.warn(`[RealtimeSession] Failed to take encoded recording packets for session ${pipeline.SessionID}; this tick's audio is lost:`, error);
1824
2161
  return;
1825
2162
  }
1826
- // Read once: a shard's sample rate is fixed at capture, and the recorder may be stopped by the
1827
- // time a retry of it runs.
1828
- const sampleRate = recorder.SampleRate;
1829
- await queue.Flush(() => recorder.SnapshotNewSegmentBase64(), (segmentIndex, audioBase64) => this.uploadRecordingShard(agentSessionId, sampleRate, segmentIndex, audioBase64));
2163
+ await this.storeShards(pipeline, sealed);
2164
+ await pipeline.Queue.Flush(sealed);
2165
+ }
2166
+ /** Writes shards to the local store, in order, before their upload starts. Never rejects. */
2167
+ async storeShards(pipeline, shards) {
2168
+ const store = this.ensureShardStore();
2169
+ for (const shard of shards) {
2170
+ try {
2171
+ await store.Put({ SessionID: pipeline.SessionID, UserID: pipeline.UserID, Index: shard.Index, Bytes: shard.Bytes, SavedAt: Date.now() });
2172
+ }
2173
+ catch (error) {
2174
+ // The store contract is never-throw; an override that does must still not stop the upload.
2175
+ console.warn(`[RealtimeSession] Failed to store recording shard ${pipeline.SessionID}/${shard.Index}; uploading it anyway:`, error);
2176
+ }
2177
+ }
2178
+ }
2179
+ /** Removes an acknowledged shard from the local store. Never rejects. */
2180
+ deleteStoredShard(sessionId, index) {
2181
+ return DeleteStoredRecordingShard(this.ensureShardStore(), sessionId, index);
1830
2182
  }
1831
2183
  /**
1832
- * Sends one shard and reports whether the server stored it. The resolver reports a failed
1833
- * ownership load or storage write as `false`, not as an error — that is how #5197 lost a shard
1834
- * without a single client-side log line — so the typed result must be checked, not just the absence
1835
- * of a throw.
2184
+ * Tells the server, right now, how much of the recording exists — for a host's `pagehide` and
2185
+ * `visibilitychange → hidden` handlers, where the page may be gone a moment later. Synchronous:
2186
+ * it seals the pending partial shard and fires the `keepalive` terminal with it, then re-sends
2187
+ * with `keepalive` whatever the server has not acknowledged yet (see
2188
+ * {@link dispatchRecordingTerminal}), and returns without waiting. Never stops the recorder: after
2189
+ * a `hidden` the call may well continue, and later audio simply starts the next shard index (the
2190
+ * tail is sealed, so no index is ever re-sent with different bytes). Also works while End is still
2191
+ * stopping the recorder, which is exactly when a host that does not await End navigates away.
2192
+ * No-op when nothing is recording, there is no session id or no transport, or nothing was
2193
+ * captured. Never throws.
2194
+ *
2195
+ * Deliberately opens NO IndexedDB transaction inside the handler: a `pagehide` handler gets a few
2196
+ * milliseconds, and a store write started there could not complete during unload anyway. The
2197
+ * shards it sealed are written to the store from a `setTimeout(0)` task after the handler returns:
2198
+ * on a page that is merely hidden (the call goes on) that keeps the durable copy until the server
2199
+ * acknowledges it; on a page that is unloading the task just never runs.
1836
2200
  */
1837
- async uploadRecordingShard(agentSessionId, sampleRate, segmentIndex, audioBase64) {
1838
- const mutation = `
1839
- mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
1840
- UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
1841
- }
1842
- `;
1843
- // Shards are HEADER-LESS raw little-endian PCM16 (mime audio/L16 with the capture sample rate),
1844
- // NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
1845
- // seekable WAV is the consolidated end-of-call upload below.
1846
- const shardMime = `audio/L16;rate=${sampleRate}`;
1847
- const result = (await this.gql().ExecuteGQL(mutation, {
1848
- agentSessionId,
1849
- segmentIndex,
1850
- audioBase64,
1851
- mimeType: shardMime,
1852
- }));
1853
- return result?.UploadRealtimeRecordingSegment === true;
1854
- }
1855
- /**
1856
- * Stops the active recorder and encodes the captured audio. Split from the upload so teardown can
1857
- * close the server session in between (#5195). Best-effort: returns `null` when nothing was
1858
- * recorded or the stop failed (warned) — it must NEVER block teardown.
1859
- */
1860
- async stopRecording() {
1861
- this.stopSegmentFlushing();
1862
- // Close the queue with the recording: its retained shards (base64 audio) are released now, not at
1863
- // the next session start, and a flush still in flight stops sending, so no resend lands after the
1864
- // end-of-call upload below has deleted the shards.
1865
- this.shardQueue?.Close();
1866
- this.shardQueue = null;
2201
+ SendRecordingTerminal(options) {
1867
2202
  const recorder = this.recorder;
1868
- this.recorder = null;
2203
+ const pipeline = this.recordingShards;
2204
+ if (!recorder || !pipeline || !pipeline.Transport) {
2205
+ return;
2206
+ }
2207
+ try {
2208
+ const sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
2209
+ const tail = pipeline.Window.SealTail();
2210
+ if (pipeline.Window.LastSealedIndex < 0) {
2211
+ return;
2212
+ }
2213
+ const final = options?.Final ?? true;
2214
+ if (sealed.length === 0 && !tail && pipeline.FinalTerminal?.LastSealedIndex === pipeline.Window.LastSealedIndex) {
2215
+ return;
2216
+ }
2217
+ void this.dispatchRecordingTerminal(pipeline, sealed, tail, true, final);
2218
+ }
2219
+ catch (error) {
2220
+ console.warn(`[RealtimeSession] Failed to send the recording terminal for session ${pipeline.SessionID}:`, error);
2221
+ }
2222
+ }
2223
+ /**
2224
+ * Fires the terminal request, then re-sends with `keepalive` every shard the server has not
2225
+ * acknowledged, then stores the shards sealed for this terminal. Shared by End and the page-lifecycle
2226
+ * handlers. Everything that needs the network is dispatched synchronously, before any storage work,
2227
+ * because the page may close at any moment from here on: an ordinary `fetch` is aborted on unload,
2228
+ * and a `keepalive` one is not.
2229
+ *
2230
+ * - The terminal goes first so it always gets the keepalive budget. The server waits for the shards
2231
+ * the terminal says exist, and assembles as soon as the last missing one arrives.
2232
+ * - The re-sent shards are the queue's unacknowledged ones (on the wire, waiting in a batch, or
2233
+ * retained after a failure) plus `sealedNow`, oldest first, within
2234
+ * {@link recordingKeepaliveBudgetBytes} together with the tail. A `sealedNow` shard that does not
2235
+ * fit, or whose keepalive upload fails, goes through the ordinary queue instead (it may still
2236
+ * land if the page stays). The queue keeps its own shards either way; a duplicate is acked as one.
2237
+ * - `sealedNow` and the tail are stored after the dispatch, from a `setTimeout(0)` task when called
2238
+ * from a page-lifecycle handler (`deferStore`). A shard's stored copy is deleted on its ack, after
2239
+ * the store write, so a fast ack can never leave an orphan row.
2240
+ *
2241
+ * Resolves true when the terminal was acknowledged. Never rejects.
2242
+ */
2243
+ dispatchRecordingTerminal(pipeline, sealedNow, tail, deferStore, final) {
2244
+ const sessionId = pipeline.SessionID;
2245
+ const terminalAcked = this.sendRecordingTerminal(pipeline, tail, final);
2246
+ const toStore = tail ? [...sealedNow, tail] : sealedNow;
2247
+ let stored = Promise.resolve();
2248
+ if (toStore.length > 0) {
2249
+ stored = deferStore ? this.storeShardsLater(pipeline, toStore) : this.storeShards(pipeline, toStore);
2250
+ }
2251
+ this.resendUnacknowledgedWithKeepalive(pipeline, sealedNow, stored);
2252
+ const ackedPromise = terminalAcked.then(async (acked) => {
2253
+ await stored;
2254
+ if (!tail) {
2255
+ return acked;
2256
+ }
2257
+ if (acked) {
2258
+ await this.deleteStoredShard(sessionId, tail.Index);
2259
+ }
2260
+ else {
2261
+ // Retried as an ordinary shard; a Flush on a closed queue (the recording ended) is a logged no-op.
2262
+ await pipeline.Queue.Flush([tail]);
2263
+ }
2264
+ return acked;
2265
+ });
2266
+ if (final) {
2267
+ pipeline.FinalTerminal = { LastSealedIndex: pipeline.Window.LastSealedIndex, Acked: terminalAcked };
2268
+ }
2269
+ return ackedPromise;
2270
+ }
2271
+ /** Counts `bytes` against the page's keepalive budget until `request` settles. */
2272
+ trackKeepalive(bytes, request) {
2273
+ this.keepaliveBytesInFlight += bytes;
2274
+ return request.finally(() => {
2275
+ this.keepaliveBytesInFlight -= bytes;
2276
+ });
2277
+ }
2278
+ /**
2279
+ * The `keepalive` re-send part of {@link dispatchRecordingTerminal}, budgeted against everything this
2280
+ * page still has in flight (the terminal just sent included). A shard whose `keepalive` copy is already
2281
+ * on the wire is not sent again. Never throws.
2282
+ */
2283
+ resendUnacknowledgedWithKeepalive(pipeline, sealedNow, stored) {
2284
+ const transport = pipeline.Transport;
2285
+ const sessionId = pipeline.SessionID;
2286
+ const fresh = new Set(sealedNow.map((shard) => shard.Index));
2287
+ const toQueue = (shard) => stored.then(() => pipeline.Queue.Flush([shard]));
2288
+ const byIndex = new Map();
2289
+ for (const shard of [...pipeline.Queue.Unacknowledged(), ...sealedNow]) {
2290
+ byIndex.set(shard.Index, shard);
2291
+ }
2292
+ for (const shard of [...byIndex.values()].sort((a, b) => a.Index - b.Index)) {
2293
+ const key = `${sessionId}/${shard.Index}`;
2294
+ if (this.keepaliveShardsInFlight.has(key)) {
2295
+ continue;
2296
+ }
2297
+ const budget = RealtimeSessionRuntime.recordingKeepaliveBudgetBytes - this.keepaliveBytesInFlight;
2298
+ if (!transport || shard.Bytes.length > budget) {
2299
+ if (fresh.has(shard.Index)) {
2300
+ void toQueue(shard);
2301
+ }
2302
+ continue;
2303
+ }
2304
+ this.keepaliveShardsInFlight.add(key);
2305
+ void this.trackKeepalive(shard.Bytes.length, transport.UploadShard(sessionId, shard, { Keepalive: true }).then((outcome) => outcome === 'Acked'))
2306
+ .finally(() => this.keepaliveShardsInFlight.delete(key))
2307
+ .catch((error) => {
2308
+ console.warn(`[RealtimeSession] Keepalive upload of recording shard ${sessionId}/${shard.Index} failed: ${describeRecordingFailure(error)}`, error);
2309
+ return false;
2310
+ })
2311
+ .then(async (acked) => {
2312
+ await stored;
2313
+ if (acked) {
2314
+ await this.deleteStoredShard(sessionId, shard.Index);
2315
+ }
2316
+ else if (fresh.has(shard.Index)) {
2317
+ await pipeline.Queue.Flush([shard]);
2318
+ }
2319
+ });
2320
+ }
2321
+ }
2322
+ /**
2323
+ * Stores shards from a `setTimeout(0)` task rather than now — the path for shards sealed inside a
2324
+ * page-lifecycle handler, which must not open an IndexedDB transaction itself. Resolves once they
2325
+ * are stored (never, if the page unloads first). Never rejects.
2326
+ */
2327
+ storeShardsLater(pipeline, shards) {
2328
+ return new Promise((resolve) => {
2329
+ setTimeout(() => {
2330
+ void this.storeShards(pipeline, shards).then(resolve);
2331
+ }, 0);
2332
+ });
2333
+ }
2334
+ /**
2335
+ * Sends the terminal request for a pipeline whose tail is already sealed. Resolves true on a server
2336
+ * ack and logs the outcome with the session id. Never rejects.
2337
+ */
2338
+ async sendRecordingTerminal(pipeline, tail, final) {
2339
+ const lastShardIndex = pipeline.Window.LastSealedIndex;
2340
+ const capturedSamples = pipeline.Window.SealedSamples;
2341
+ const context = `session ${pipeline.SessionID} (last shard ${lastShardIndex}, ${capturedSamples} samples)`;
2342
+ if (!pipeline.Transport) {
2343
+ console.warn(`[RealtimeSession] Recording terminal NOT sent for ${context}: no recording transport; stored shards are drained on a later load.`);
2344
+ return false;
2345
+ }
2346
+ try {
2347
+ const acked = await this.trackKeepalive(tail?.Bytes.length ?? 0, pipeline.Transport.SendTerminal(pipeline.SessionID, {
2348
+ Tail: tail?.Bytes ?? null,
2349
+ LastShardIndex: lastShardIndex,
2350
+ CapturedSamples: capturedSamples,
2351
+ Final: final,
2352
+ }));
2353
+ if (!acked) {
2354
+ console.warn(`[RealtimeSession] Recording terminal for ${context} was not acknowledged; the server janitor assembles what arrived.`);
2355
+ }
2356
+ return acked;
2357
+ }
2358
+ catch (error) {
2359
+ console.warn(`[RealtimeSession] Recording terminal for ${context} failed: ${describeRecordingFailure(error)}`, error);
2360
+ return false;
2361
+ }
2362
+ }
2363
+ /**
2364
+ * Ends the recording at teardown and STARTS its terminal request: stops the recorder (flushing the
2365
+ * encoder), cuts the remaining packets into shards with the tail trimmed to the recorder's captured
2366
+ * sample count, and dispatches the `keepalive` terminal plus `keepalive` re-sends of everything not
2367
+ * yet acknowledged, before storing anything ({@link dispatchRecordingTerminal}). Teardown awaits the
2368
+ * returned terminal only after `CloseAgentSession` ({@link awaitRecordingTerminal}), so the small,
2369
+ * critical close never waits behind the recording (#5195).
2370
+ *
2371
+ * The recorder and pipeline stay visible to {@link SendRecordingTerminal} until the recorder has
2372
+ * stopped: a `pagehide` in that window (a host that navigates away without awaiting End) still
2373
+ * sends what was captured, and End then continues at the next index.
2374
+ *
2375
+ * Returns null when nothing was recording or nothing was captured (no terminal is sent then).
2376
+ * Best-effort: never throws, never blocks teardown on the network.
2377
+ */
2378
+ async finishRecording() {
2379
+ this.stopShardFlushing();
2380
+ const recorder = this.recorder;
2381
+ const pipeline = this.recordingShards;
1869
2382
  this.currentTurnStartMs = null;
1870
2383
  this.turnAudioStartCaptured = false;
1871
- if (!recorder) {
2384
+ this.userSpeechStartMs = null;
2385
+ this.userSpeechStartSuperseded = false;
2386
+ this.userSpeechStartByItemId.clear();
2387
+ if (!recorder || !pipeline) {
2388
+ this.recorder = null;
2389
+ this.recordingShards = null;
1872
2390
  return null;
1873
2391
  }
1874
2392
  try {
1875
- // Capture the recorder MIME (now 'audio/wav') BEFORE Stop() — the getter reads '' once stopped.
1876
- const mimeType = recorder.MimeType;
1877
- const audioBase64 = await recorder.StopAndEncode();
1878
- // Read the real waveform peaks computed during capture (survives the stop via the snapshot).
1879
- const peaks = recorder.GetPeaks();
1880
- if (!audioBase64) {
1881
- console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — empty recording.');
2393
+ const capturedSamples = await this.stopRecorder(recorder, pipeline.SessionID);
2394
+ this.recorder = null;
2395
+ this.recordingShards = null;
2396
+ const sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
2397
+ // Without a count from the recorder the tail keeps every packet it holds (SealTail's default).
2398
+ const tail = pipeline.Window.SealTail(capturedSamples === null ? undefined : Math.max(0, capturedSamples - pipeline.Window.SealedSamples));
2399
+ if (pipeline.Window.LastSealedIndex < 0) {
2400
+ pipeline.Queue.Close();
1882
2401
  return null;
1883
2402
  }
1884
- return { MimeType: mimeType, AudioBase64: audioBase64, Peaks: peaks };
2403
+ if (sealed.length === 0 && !tail && pipeline.FinalTerminal?.LastSealedIndex === pipeline.Window.LastSealedIndex) {
2404
+ return { Pipeline: pipeline, Acked: pipeline.FinalTerminal.Acked };
2405
+ }
2406
+ return { Pipeline: pipeline, Acked: this.dispatchRecordingTerminal(pipeline, sealed, tail, false, true) };
1885
2407
  }
1886
2408
  catch (error) {
1887
- console.warn('[RealtimeSession] Failed to stop call recording:', error);
2409
+ console.warn(`[RealtimeSession] Failed to finish the call recording for session ${pipeline.SessionID}; stored shards are drained at the next session start or load:`, error);
2410
+ this.recorder = null;
2411
+ this.recordingShards = null;
2412
+ pipeline.Queue.Close();
2413
+ this.rearmPendingShardDrain();
1888
2414
  return null;
1889
2415
  }
1890
2416
  }
1891
2417
  /**
1892
- * Uploads a recording returned by {@link stopRecording}. Never throws. No-op when there is
1893
- * nothing to upload; warns when there is no session id to attach the file to.
2418
+ * Stops the recorder, bounded by {@link RECORDER_STOP_TIMEOUT_MS}. Resolves the captured sample
2419
+ * count, or null when the recorder threw or did not finish in time (logged) — the caller then ends
2420
+ * the recording at the packets already encoded. Never rejects.
1894
2421
  */
1895
- async uploadStoppedRecording(agentSessionId, recording) {
1896
- if (!recording) {
1897
- return;
1898
- }
1899
- if (!agentSessionId) {
1900
- console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — no session id.');
1901
- return;
1902
- }
2422
+ async stopRecorder(recorder, sessionId) {
2423
+ let timer;
2424
+ const timedOut = new Promise((resolve) => {
2425
+ timer = setTimeout(() => resolve('timeout'), RECORDER_STOP_TIMEOUT_MS);
2426
+ });
1903
2427
  try {
1904
- await this.uploadRecording(agentSessionId, recording.AudioBase64, recording.MimeType, recording.Peaks);
2428
+ const outcome = await Promise.race([recorder.Stop(), timedOut]);
2429
+ if (outcome === 'timeout') {
2430
+ console.warn(`[RealtimeSession] Recorder did not stop within ${RECORDER_STOP_TIMEOUT_MS}ms for session ${sessionId}; ending the recording at the packets already encoded.`);
2431
+ return null;
2432
+ }
2433
+ return outcome;
1905
2434
  }
1906
2435
  catch (error) {
1907
- console.warn(`[RealtimeSession] Failed to upload call recording for session ${agentSessionId}:`, error);
2436
+ console.warn(`[RealtimeSession] Recorder failed to stop for session ${sessionId}; ending the recording at the packets already encoded:`, error);
2437
+ return null;
2438
+ }
2439
+ finally {
2440
+ clearTimeout(timer);
1908
2441
  }
1909
2442
  }
1910
2443
  /**
1911
- * Runs the `UploadRealtimeRecording` mutation. A `Success:false` payload is logged; a transport
1912
- * rejection propagates to `uploadStoppedRecording`, which catches and logs it. Sends the
1913
- * capture-time waveform `peaks` (max-abs per bucket, normalized 0..1) so the server can persist a
1914
- * `peaks.json` sidecar for fast waveform rendering without re-decoding the audio.
2444
+ * Waits for the terminal {@link finishRecording} started, bounded by
2445
+ * {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}, logs the outcome, then closes the recording's queue
2446
+ * (anything it had not delivered stays in the local store for the next-load drain). Never throws.
1915
2447
  */
1916
- async uploadRecording(agentSessionId, audioBase64, mimeType, peaks) {
1917
- const mutation = `
1918
- mutation UploadRealtimeRecording($agentSessionId: String!, $audioBase64: String!, $mimeType: String!, $consent: Boolean, $peaks: [Float!]) {
1919
- UploadRealtimeRecording(agentSessionId: $agentSessionId, audioBase64: $audioBase64, mimeType: $mimeType, consent: $consent, peaks: $peaks) {
1920
- Success
1921
- FileID
1922
- ErrorMessage
1923
- }
1924
- }
1925
- `;
1926
- const result = await this.gql().ExecuteGQL(mutation, {
1927
- agentSessionId,
1928
- audioBase64,
1929
- mimeType,
1930
- consent: true,
1931
- peaks
2448
+ async awaitRecordingTerminal(pending) {
2449
+ const sessionId = pending.Pipeline.SessionID;
2450
+ let timer;
2451
+ const timedOut = new Promise((resolve) => {
2452
+ timer = setTimeout(() => resolve('timeout'), RECORDING_TERMINAL_ACK_TIMEOUT_MS);
1932
2453
  });
1933
- const payload = result?.UploadRealtimeRecording;
1934
- if (!payload?.Success) {
1935
- console.warn(`[RealtimeSession] ❌ recording upload reported failure: ${payload?.ErrorMessage ?? 'unknown error'} (full result: ${JSON.stringify(result)})`);
2454
+ try {
2455
+ const outcome = await Promise.race([pending.Acked.then((ok) => (ok ? 'ack' : 'failed')), timedOut]);
2456
+ if (outcome === 'ack') {
2457
+ console.log(`[RealtimeSession] Recording terminal acknowledged for session ${sessionId}.`);
2458
+ }
2459
+ else if (outcome === 'timeout') {
2460
+ console.warn(`[RealtimeSession] Recording terminal for session ${sessionId} not acknowledged within ${RECORDING_TERMINAL_ACK_TIMEOUT_MS}ms; ending anyway (the keepalive request may still land).`);
2461
+ }
2462
+ // 'failed' was already logged with its reason by sendRecordingTerminal.
2463
+ }
2464
+ finally {
2465
+ clearTimeout(timer);
2466
+ pending.Pipeline.Queue.Close();
2467
+ // The recording may have left shards in the store (an unacknowledged tail, windows the queue
2468
+ // still held or had on the wire at Close), so re-arm the drain for the next session start on
2469
+ // this page. Re-armed here, after the wait, so it never races the recording's own terminal;
2470
+ // and unconditionally, because whether a fire-and-forget upload landed is not known here and
2471
+ // a drain of an empty store costs one read.
2472
+ this.rearmPendingShardDrain();
1936
2473
  }
1937
2474
  }
1938
2475
  /**
@@ -2939,6 +3476,27 @@ export class RealtimeSessionRuntime {
2939
3476
  console.error('[RealtimeSession] Provider error event:', JSON.stringify(error), error);
2940
3477
  if (error.Fatal && this.client === client) {
2941
3478
  this.lastSessionError = error;
3479
+ // While the start is still running, its own failure path reports the error and ends the call
3480
+ // (runMintedSession throws on this recorded Fatal after Connect, even if Connect resolves):
3481
+ // ending it here would bump startGeneration, so the start would read as ended by the host and
3482
+ // never set LastStartError.
3483
+ if (!this.callWentLive) {
3484
+ return;
3485
+ }
3486
+ // An explicit teardown is already ending the call and the client stays current until its
3487
+ // finally: a driver reporting its own close as Fatal from inside Disconnect() is part of that
3488
+ // clean End, and must not flip it to 'error' or start a second teardown.
3489
+ if (this.teardownInFlight) {
3490
+ return;
3491
+ }
3492
+ // The driver is dead on the wire; nothing resumes after it says Fatal. End the call the way a
3493
+ // failed start ends: the row closes as Error, the microphone is released, the host is told
3494
+ // through SessionEnded$ (MJ#4629). A host's own End afterwards finds nothing to do.
3495
+ // Captured now: teardown nulls agentSessionId, and the failure it reports must name the session.
3496
+ const failedSessionId = this.agentSessionId;
3497
+ void this.endOnFailure().catch((teardownError) => {
3498
+ console.error(`[RealtimeSession] Teardown after a fatal provider error failed for session ${failedSessionId}:`, teardownError);
3499
+ });
2942
3500
  }
2943
3501
  });
2944
3502
  // Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
@@ -2956,6 +3514,12 @@ export class RealtimeSessionRuntime {
2956
3514
  client.OnInterruption(() => {
2957
3515
  this.cancelPendingNarration();
2958
3516
  });
3517
+ // The provider VAD saw the user start speaking. Final-only user-transcript drivers (OpenAI,
3518
+ // HuggingFace, xAI) emit no user interim, so this is their only real user-turn start. It goes
3519
+ // to its own slot (or item-keyed map entry when `itemId` is present), not the shared turn slot:
3520
+ // the user transcript arrives asynchronously, so the assistant's reply may already be streaming
3521
+ // and must keep its own start.
3522
+ client.OnUserSpeechStart((itemId) => this.markUserSpeechStart(itemId));
2959
3523
  client.OnRemoteVideo((video) => this.onAgentVideo(client, video));
2960
3524
  }
2961
3525
  /**
@@ -3058,6 +3622,11 @@ export class RealtimeSessionRuntime {
3058
3622
  }
3059
3623
  return;
3060
3624
  }
3625
+ if (transcript.Role === 'User' && transcript.Text.trim().length === 0) {
3626
+ // An empty final user transcript (a VAD false positive, an empty transcription) never reaches
3627
+ // relayTranscript, so release its onset here: it must not become the next real turn's start.
3628
+ this.releaseUserSpeechStart(transcript.ItemId);
3629
+ }
3061
3630
  if (transcript.Role === 'Assistant') {
3062
3631
  this.hasActiveInterimUserCaption = false;
3063
3632
  this.pendingUserCaption = '';
@@ -3100,7 +3669,7 @@ export class RealtimeSessionRuntime {
3100
3669
  if (this.firstUserTranscript === null) {
3101
3670
  this.firstUserTranscript = transcript.Text;
3102
3671
  }
3103
- await this.relayTranscript('user', transcript.Text);
3672
+ await this.relayTranscript('user', transcript.Text, false, 'speech', transcript.ItemId);
3104
3673
  }
3105
3674
  else if (transcript.ReplacesPrevious) {
3106
3675
  // STREAMING user transcription: providers like Grok and OpenAI Live emit the growing utterance as repeated
@@ -3111,10 +3680,10 @@ export class RealtimeSessionRuntime {
3111
3680
  return;
3112
3681
  }
3113
3682
  this.replaceLastCaption('User', transcript.Text);
3114
- await this.relayTranscript('user', transcript.Text, true);
3683
+ await this.relayTranscript('user', transcript.Text, true, 'speech', transcript.ItemId);
3115
3684
  }
3116
3685
  else {
3117
- await this.onUserTranscript(transcript.Text);
3686
+ await this.onUserTranscript(transcript.Text, 'speech', transcript.ItemId);
3118
3687
  }
3119
3688
  }
3120
3689
  /**
@@ -3132,11 +3701,10 @@ export class RealtimeSessionRuntime {
3132
3701
  * - NORMAL turns only — NARRATION interims are ephemeral and never persisted, so they must not
3133
3702
  * claim the next real turn's start slot.
3134
3703
  *
3135
- * Works for any role whose driver surfaces interim deltas (all drivers for the assistant; the
3136
- * relevant case here — the post-tool-gap assistant answer — and user-interim drivers like
3137
- * Gemini/AssemblyAI). For final-only user turns (OpenAI/xAI/ElevenLabs) no interim arrives, so
3138
- * {@link relayTranscript} falls back to the seeded/prior start — the gap case that drifts is the
3139
- * assistant answer, which always has interims.
3704
+ * Stamped on a turn's first interim transcript (all drivers for the assistant — the post-tool-gap
3705
+ * assistant answer — and user-interim drivers like Gemini/AssemblyAI). Final-only user turns
3706
+ * (OpenAI-protocol drivers) take their start from {@link markUserSpeechStart}'s own slot
3707
+ * instead; one with no speech-start signal either (ElevenLabs) relays `null` as-is.
3140
3708
  */
3141
3709
  markTurnAudioStart(kind) {
3142
3710
  if (this.turnAudioStartCaptured || kind === 'narration') {
@@ -3149,15 +3717,75 @@ export class RealtimeSessionRuntime {
3149
3717
  this.currentTurnStartMs = offset;
3150
3718
  this.turnAudioStartCaptured = true;
3151
3719
  }
3720
+ /**
3721
+ * Stamps the user speech onset when the client signals that the user began to speak. When the
3722
+ * driver supplies a conversation `itemId`, records the FIRST onset for that item in
3723
+ * {@link userSpeechStartByItemId} so a mid-sentence pause that re-fires with the same id keeps the
3724
+ * original start. When no `itemId` is supplied, stamps the single-slot {@link userSpeechStartMs}
3725
+ * fallback (keeping the first onset unless {@link userSpeechStartSuperseded} is set).
3726
+ */
3727
+ markUserSpeechStart(itemId) {
3728
+ if (itemId !== undefined && itemId.length > 0) {
3729
+ if (this.userSpeechStartByItemId.has(itemId)) {
3730
+ return;
3731
+ }
3732
+ const offset = this.nowTurnOffsetMs();
3733
+ if (offset === null) {
3734
+ return;
3735
+ }
3736
+ this.userSpeechStartByItemId.set(itemId, offset);
3737
+ return;
3738
+ }
3739
+ if (this.userSpeechStartMs !== null && !this.userSpeechStartSuperseded) {
3740
+ return;
3741
+ }
3742
+ const offset = this.nowTurnOffsetMs();
3743
+ if (offset === null) {
3744
+ return;
3745
+ }
3746
+ this.userSpeechStartMs = offset;
3747
+ this.userSpeechStartSuperseded = false;
3748
+ }
3749
+ /**
3750
+ * Ends the held user onset: a final user transcript (empty or not) closed the utterance it began.
3751
+ * When `itemId` is supplied, deletes only that item's entry in {@link userSpeechStartByItemId};
3752
+ * otherwise clears the unkeyed single slot.
3753
+ */
3754
+ releaseUserSpeechStart(itemId) {
3755
+ if (itemId !== undefined && itemId.length > 0) {
3756
+ this.userSpeechStartByItemId.delete(itemId);
3757
+ return;
3758
+ }
3759
+ this.userSpeechStartMs = null;
3760
+ this.userSpeechStartSuperseded = false;
3761
+ }
3762
+ /**
3763
+ * Consumes and deletes the onset recorded for `itemId` in {@link userSpeechStartByItemId}. When a
3764
+ * matching onset is found, also deletes any remaining map entry with an earlier offset so a VAD
3765
+ * false positive whose transcription never arrived cannot accumulate in the map.
3766
+ */
3767
+ takeKeyedUserSpeechStart(itemId) {
3768
+ const onset = this.userSpeechStartByItemId.get(itemId) ?? null;
3769
+ this.userSpeechStartByItemId.delete(itemId);
3770
+ if (onset !== null) {
3771
+ for (const [otherId, otherOnset] of this.userSpeechStartByItemId) {
3772
+ if (otherOnset < onset) {
3773
+ this.userSpeechStartByItemId.delete(otherId);
3774
+ }
3775
+ }
3776
+ }
3777
+ return onset;
3778
+ }
3152
3779
  /**
3153
3780
  * The current per-turn offset in ms — the RECORDER's clock when one runs (an offset into a
3154
3781
  * seekable file), else the SESSION clock (#3832: orderable and displayable, not seekable),
3155
3782
  * else `null` before any call is live. One function so the two stamp sites cannot disagree
3156
- * about which clock a session is on.
3783
+ * about which clock a session is on. Always whole milliseconds: the offsets travel as GraphQL
3784
+ * `Int`, which refuses a fraction, and a 48 kHz sample clock rarely lands on a whole millisecond.
3157
3785
  */
3158
3786
  nowTurnOffsetMs() {
3159
3787
  if (this.recorder) {
3160
- return this.recorder.NowOffsetMs();
3788
+ return Math.round(this.recorder.NowOffsetMs());
3161
3789
  }
3162
3790
  if (this.sessionClockStartMs !== null) {
3163
3791
  return Math.max(0, Math.round(performance.now() - this.sessionClockStartMs));
@@ -3181,8 +3809,14 @@ export class RealtimeSessionRuntime {
3181
3809
  }
3182
3810
  this.appendCaption({ Role: role, Text: text });
3183
3811
  }
3184
- /** Finalizes the user turn: push a caption + relay the final transcript. */
3185
- async onUserTranscript(transcript) {
3812
+ /**
3813
+ * Finalizes the user turn: push a caption + relay the final transcript.
3814
+ *
3815
+ * @param source `'typed'` for a {@link SendText} turn: it has no speech onset, so it neither takes
3816
+ * nor clears the held one (a spoken transcript may still be pending for it).
3817
+ * @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
3818
+ */
3819
+ async onUserTranscript(transcript, source = 'speech', itemId) {
3186
3820
  if (transcript.trim().length === 0) {
3187
3821
  return;
3188
3822
  }
@@ -3191,7 +3825,7 @@ export class RealtimeSessionRuntime {
3191
3825
  this.firstUserTranscript = transcript;
3192
3826
  }
3193
3827
  this.appendCaption({ Role: 'User', Text: transcript });
3194
- await this.relayTranscript('user', transcript);
3828
+ await this.relayTranscript('user', transcript, false, source, itemId);
3195
3829
  }
3196
3830
  // ── Tool calling ───────────────────────────────────────────────────────────
3197
3831
  /**
@@ -3519,37 +4153,79 @@ export class RealtimeSessionRuntime {
3519
4153
  * Relays a final transcript turn to MJ via `RelayRealtimeTranscript`.
3520
4154
  *
3521
4155
  * When the session is being recorded, per-turn timing rides along: `utteranceEndMs` is the
3522
- * recording-relative offset at finalization, and `utteranceStartMs` is the offset captured by
3523
- * {@link markTurnAudioStart} when THIS turn's audio actually began (its first interim) — NOT
3524
- * inherited from the previous turn's end. That distinction is the timing fix: when a tool-call
3525
- * / silence gap sits between turns, the post-gap turn's audio starts much later, so inheriting
3526
- * the prior turn's end stamped the cue ~gap-length too early. Both are omitted (left `null`)
3527
- * when the session isn't being recorded.
4156
+ * recording-relative offset at finalization, and `utteranceStartMs` is where THIS turn's audio
4157
+ * actually began — NOT inherited from the previous turn's end. For a user turn whose onset the
4158
+ * client signalled, that is {@link userSpeechStartMs}; otherwise it is the offset captured by
4159
+ * {@link markTurnAudioStart} on the turn's first interim. That distinction is the timing fix:
4160
+ * when a tool-call / silence gap sits between turns, the post-gap turn's audio starts much
4161
+ * later, so inheriting the prior turn's end stamped the cue ~gap-length too early. Both are
4162
+ * omitted (left `null`) when the session isn't being recorded. `utteranceStartMs` is also
4163
+ * `null` when no start was measured for this turn; it is never replaced with a placeholder 0.
3528
4164
  *
3529
4165
  * A correction (`replacesPrevious`) doesn't open a new turn, so it carries no start and doesn't
3530
4166
  * reset the per-turn start guard. After a normal finalization the guard is cleared so the NEXT
3531
- * turn re-stamps its start from where ITS audio begins.
4167
+ * turn re-stamps its start from where ITS audio begins — except for a user turn that used its
4168
+ * own speech-start slot, which leaves the shared slot alone (it may hold the in-flight
4169
+ * assistant turn's start). Every final user transcript, corrections included, clears the
4170
+ * speech-start slot.
3532
4171
  *
3533
4172
  * @param replacesPrevious CORRECTION semantics: the server updates the session's most
3534
4173
  * recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
3535
4174
  * post-barge-in `agent_response_correction`).
4175
+ * @param source `'typed'` for a user turn sent with {@link SendText}: it does not read or clear
4176
+ * {@link userSpeechStartMs} or {@link userSpeechStartByItemId}, which belong to the user's speech.
4177
+ * @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
3536
4178
  */
3537
- async relayTranscript(role, text, replacesPrevious = false) {
4179
+ async relayTranscript(role, text, replacesPrevious = false, source = 'speech', itemId) {
3538
4180
  if (!this.agentSessionId) {
3539
4181
  return;
3540
4182
  }
4183
+ // Every final user transcript closes the signalled onset — a correction too, since it ends the
4184
+ // same utterance — so a later onset (an xAI mid-sentence resume) can't leak into the next turn.
4185
+ // When the transcript carries a provider itemId, consume its keyed onset from the map without
4186
+ // touching the unkeyed single slot or falling back to currentTurnStartMs.
4187
+ const hasItemId = role === 'user' && source === 'speech' && itemId !== undefined && itemId.length > 0;
4188
+ let userSpeechStartMs = null;
4189
+ if (hasItemId) {
4190
+ userSpeechStartMs = this.takeKeyedUserSpeechStart(itemId);
4191
+ }
4192
+ else if (role === 'user' && source === 'speech') {
4193
+ userSpeechStartMs = this.userSpeechStartMs;
4194
+ this.releaseUserSpeechStart();
4195
+ }
3541
4196
  // Per-turn timing against whichever clock the session is on (#3832): the recorder's when one
3542
- // runs, else the session clock. `utteranceStartMs` is where this turn's audio actually began
3543
- // (captured by markTurnAudioStart on the first interim); the `?? 0` fallback covers a turn
3544
- // whose interim was missed / a final-only first turn.
4197
+ // runs, else the session clock. `utteranceStartMs` is where this turn's audio actually began.
4198
+ // It is null when no start was measured for this turn (a final-only driver with no speech-start
4199
+ // signal), so the server stores NULL rather than a placeholder 0.
3545
4200
  const utteranceEndMs = this.nowTurnOffsetMs();
3546
- const utteranceStartMs = utteranceEndMs !== null && !replacesPrevious ? (this.currentTurnStartMs ?? 0) : null;
3547
- if (utteranceEndMs !== null && !replacesPrevious) {
4201
+ const opensTurn = utteranceEndMs !== null && !replacesPrevious;
4202
+ let utteranceStartMs = null;
4203
+ if (opensTurn && (hasItemId || userSpeechStartMs !== null)) {
4204
+ // A user turn with a signalled onset (or a keyed user turn whose itemId was not in the map,
4205
+ // which relays null). Leave the shared slot alone: the user transcript arrives asynchronously,
4206
+ // so it may already hold the in-flight assistant reply's start.
4207
+ utteranceStartMs = userSpeechStartMs;
4208
+ if (!this.turnAudioStartCaptured) {
4209
+ // No turn has captured a start, so the slot holds only the recording-start seed (0). This
4210
+ // user turn was the first turn; drop the seed so a later unmeasured turn relays null, not 0.
4211
+ this.currentTurnStartMs = null;
4212
+ }
4213
+ }
4214
+ else if (opensTurn) {
4215
+ utteranceStartMs = this.currentTurnStartMs;
3548
4216
  // This turn is finalized — arm the NEXT turn to re-stamp its start from its own first
3549
4217
  // interim (handles a tool-call gap before the next turn). Stop inheriting this end as the
3550
- // next start. `null` means "not yet captured"; relay falls back to `?? 0` if no interim fires.
4218
+ // next start. `null` means "not yet captured" and is relayed as-is if nothing stamps it.
3551
4219
  this.currentTurnStartMs = null;
3552
4220
  this.turnAudioStartCaptured = false;
4221
+ if (role === 'assistant' && this.userSpeechStartMs !== null && (utteranceStartMs === null || this.userSpeechStartMs < utteranceStartMs)) {
4222
+ // The floor moved on past an onset that began BEFORE this assistant turn: a late user
4223
+ // transcript may still claim it, but the next onset replaces it (see userSpeechStartSuperseded).
4224
+ // An onset that began DURING this turn is a barge-in; it belongs to the user speech still
4225
+ // pending, so it stays the first onset. With no measured assistant start we cannot tell, and
4226
+ // treat the onset as earlier.
4227
+ this.userSpeechStartSuperseded = true;
4228
+ }
3553
4229
  }
3554
4230
  try {
3555
4231
  const mutation = `
@@ -3943,10 +4619,18 @@ export class RealtimeSessionRuntime {
3943
4619
  }
3944
4620
  // ── Teardown ───────────────────────────────────────────────────────────────
3945
4621
  /**
3946
- * Tears down all client resources and (optionally) closes the server session.
3947
- * @param closeServerSession when true, calls `CloseAgentSession` on the server.
4622
+ * Tears down all client resources and closes the server session when one was minted.
4623
+ *
4624
+ * The server row is closed whichever way the call ended: a start that failed after the mint used to
4625
+ * leave its row `Active` until the janitor's staleness sweep, 15 minutes later, so every host that
4626
+ * counts live sessions saw a phantom call (MJ#6013). The janitor remains the fallback for a client
4627
+ * that dies before it can say anything; it is never the plan.
4628
+ *
4629
+ * @param end why the call ended — `'explicit'` when the host or user ended it, `'error'` when the
4630
+ * runtime ended it because the start or the call failed. Reported on `SessionEnded$` and stamped on
4631
+ * the server row as its `CloseReason` (`Explicit` / `Error`).
3948
4632
  */
3949
- async teardown(closeServerSession) {
4633
+ async teardown(end) {
3950
4634
  // Invalidate any start still in flight BEFORE anything else, so it abandons itself at its next
3951
4635
  // await rather than opening a microphone behind a session that is being ended.
3952
4636
  this.startGeneration++;
@@ -3956,7 +4640,7 @@ export class RealtimeSessionRuntime {
3956
4640
  await this.teardownInFlight;
3957
4641
  return;
3958
4642
  }
3959
- this.teardownInFlight = this.runTeardown(closeServerSession).finally(() => {
4643
+ this.teardownInFlight = this.runTeardown(end).finally(() => {
3960
4644
  if (this._savingRecording$.value) {
3961
4645
  this._savingRecording$.next(false);
3962
4646
  }
@@ -3965,11 +4649,22 @@ export class RealtimeSessionRuntime {
3965
4649
  await this.teardownInFlight;
3966
4650
  }
3967
4651
  /** The body of {@link teardown}; never called concurrently with itself. */
3968
- async runTeardown(closeServerSession) {
4652
+ async runTeardown(end) {
3969
4653
  // Arm BEFORE any await so a tab closed in the first milliseconds of teardown is still guarded.
3970
4654
  if (this.recorder) {
3971
4655
  this._savingRecording$.next(true);
3972
4656
  }
4657
+ // Whatever a step below does, the slot is released: a teardown that threw halfway used to leave
4658
+ // `IsActive` set for good, so every later start was refused (MJ#4629).
4659
+ try {
4660
+ await this.releaseCallResources(end);
4661
+ }
4662
+ finally {
4663
+ this.releaseSessionSlot(end);
4664
+ }
4665
+ }
4666
+ /** The teardown's work: stop the call, hand back what it held, and close the server session. */
4667
+ async releaseCallResources(end) {
3973
4668
  // First: stop asserting liveness. A pulse racing the close would re-stamp LastActiveAt on a
3974
4669
  // session we are deliberately ending, leaving an Idle row the janitor then has to age out.
3975
4670
  this.stopLivenessPulse();
@@ -3980,6 +4675,14 @@ export class RealtimeSessionRuntime {
3980
4675
  // set (the captured per-save id covers the race anyway), then dispose the plugins.
3981
4676
  this.flushAllChannelSaves();
3982
4677
  this.disposeChannels();
4678
+ // End the recording and START its keepalive terminal request now: stop the recorder (which flushes
4679
+ // the encoder) before anything else that awaits. A tab closed right after End gets only the pagehide
4680
+ // terminal, which carries packets already encoded; releasing the mic and disconnecting the driver can
4681
+ // take long enough for the page to die first, and the audio still inside the encoder would be lost.
4682
+ // It also goes out before the usage flush and the close, so it is on the wire if the page dies during
4683
+ // either. Best-effort — finishRecording never throws. Null when nothing was recorded.
4684
+ const recordingTerminal = await this.finishRecording();
4685
+ this.recordingStartedAtIso = null;
3983
4686
  // The camera and screen share go first: the camera runs on the controller closed next.
3984
4687
  this.closeCaptures();
3985
4688
  // Defensive: stop the mic even when Connect never ran (the client also stops the
@@ -4013,11 +4716,6 @@ export class RealtimeSessionRuntime {
4013
4716
  this.client = null;
4014
4717
  }
4015
4718
  }
4016
- // Stop + encode the call recording now (the recorder must be detached before the session id is
4017
- // cleared); the upload itself happens after the close below. Best-effort — stopRecording
4018
- // swallows its own errors. Null when nothing was recorded.
4019
- const recording = await this.stopRecording();
4020
- this.recordingStartedAtIso = null;
4021
4719
  // Final usage flush WHILE the live session id is still set (the relay mutation also
4022
4720
  // accepts a Closed session, so ordering vs. CloseAgentSession is belt-and-braces).
4023
4721
  if (this.usageFlushTimer) {
@@ -4026,19 +4724,24 @@ export class RealtimeSessionRuntime {
4026
4724
  }
4027
4725
  await this.flushPendingUsage(this.agentSessionId);
4028
4726
  this.resetUsageRelay();
4029
- if (closeServerSession && this.agentSessionId) {
4030
- await this.closeServerSession(this.agentSessionId);
4031
- }
4032
- // Upload AFTER the close. The WAV is tens of MB and takes seconds; a page unload aborts the
4033
- // request, and nothing after an aborted await ever runs — so the small, critical close goes
4034
- // first (#5195). The server accepts the upload on a Closed session (loadOwnedSession does not
4035
- // check status), and uploadStoppedRecording never throws.
4036
- await this.uploadStoppedRecording(this.agentSessionId, recording);
4727
+ if (this.agentSessionId) {
4728
+ await this.closeServerSession(this.agentSessionId, end === 'error' ? 'Error' : 'Explicit');
4729
+ }
4730
+ // Wait for the terminal's ack only AFTER the close: nothing after an aborted await ever runs, so
4731
+ // the small, critical close must never queue behind the recording (#5195). Bounded by
4732
+ // RECORDING_TERMINAL_ACK_TIMEOUT_MS; SavingRecording$ stays up until it settles.
4733
+ if (recordingTerminal) {
4734
+ await this.awaitRecordingTerminal(recordingTerminal);
4735
+ }
4736
+ }
4737
+ /** Clears the slot and tells the host the call ended; runs even when {@link releaseCallResources} threw. */
4738
+ releaseSessionSlot(end) {
4037
4739
  // Capture the session id BEFORE we null it so the lifecycle emit carries it.
4038
4740
  // Skip emitting when there was no live session (defensive — teardown is safe
4039
4741
  // to call without an active session).
4040
4742
  const closedSessionId = this.agentSessionId;
4041
4743
  this.agentSessionId = null;
4744
+ this.callWentLive = false;
4042
4745
  this.sessionConversationId = null;
4043
4746
  this.narrationTemplate = null;
4044
4747
  this.clientToolHandlers.clear();
@@ -4051,14 +4754,9 @@ export class RealtimeSessionRuntime {
4051
4754
  if (this._connectionState$.value !== 'error') {
4052
4755
  this.setConnectionState('closed');
4053
4756
  }
4054
- // Surface generic session-ended for the conversations runtime bridge.
4055
- // `closeServerSession=true` means the user explicitly called EndRealtimeSession;
4056
- // `false` means teardown ran from a catch block (start path error path).
4757
+ // Surface generic session-ended for the conversations runtime bridge, with why it ended.
4057
4758
  if (closedSessionId) {
4058
- this._sessionEnded$.next({
4059
- sessionId: closedSessionId,
4060
- reason: closeServerSession ? 'explicit' : 'error',
4061
- });
4759
+ this._sessionEnded$.next({ sessionId: closedSessionId, reason: end });
4062
4760
  }
4063
4761
  }
4064
4762
  /**
@@ -4073,18 +4771,22 @@ export class RealtimeSessionRuntime {
4073
4771
  console.error('[RealtimeSession] Media host failed to release the microphone:', error);
4074
4772
  }
4075
4773
  }
4076
- /** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
4077
- async closeServerSession(agentSessionId) {
4774
+ /**
4775
+ * Calls the `CloseAgentSession` mutation (provisioned in P4b) with why the client ended the call, which
4776
+ * the server stamps as the row's `CloseReason`. Requires a server that accepts the `closeReason`
4777
+ * argument (MJ#6013); the runtime and the server ship together.
4778
+ */
4779
+ async closeServerSession(agentSessionId, closeReason) {
4078
4780
  try {
4079
4781
  const mutation = `
4080
- mutation CloseAgentSession($agentSessionId: String!) {
4081
- CloseAgentSession(agentSessionId: $agentSessionId)
4782
+ mutation CloseAgentSession($agentSessionId: String!, $closeReason: String) {
4783
+ CloseAgentSession(agentSessionId: $agentSessionId, closeReason: $closeReason)
4082
4784
  }
4083
4785
  `;
4084
- await this.gql().ExecuteGQL(mutation, { agentSessionId });
4786
+ await this.gql().ExecuteGQL(mutation, { agentSessionId, closeReason });
4085
4787
  }
4086
4788
  catch (error) {
4087
- console.error('[RealtimeSession] Failed to close server session:', error);
4789
+ console.error(`[RealtimeSession] Failed to close server session ${agentSessionId} (closeReason ${closeReason}):`, error);
4088
4790
  }
4089
4791
  }
4090
4792
  // ── Helpers ────────────────────────────────────────────────────────────────
@@ -4098,12 +4800,17 @@ export class RealtimeSessionRuntime {
4098
4800
  this.pendingUserCaption = '';
4099
4801
  this.hasActiveInterimUserCaption = false;
4100
4802
  this.SetMinimized(false);
4101
- this.stopSegmentFlushing();
4102
- this.shardQueue = null;
4803
+ this.stopShardFlushing();
4804
+ // Defensive: teardown already ended any previous recording; a stray pipeline must not keep sending.
4805
+ this.recordingShards?.Queue.Close();
4806
+ this.recordingShards = null;
4103
4807
  this.recorder = null;
4104
4808
  this.recordingStartedAtIso = null;
4105
4809
  this.currentTurnStartMs = null;
4106
4810
  this.turnAudioStartCaptured = false;
4811
+ this.userSpeechStartMs = null;
4812
+ this.userSpeechStartSuperseded = false;
4813
+ this.userSpeechStartByItemId.clear();
4107
4814
  this.usedChannelNames.clear();
4108
4815
  this.microphoneMuted = false;
4109
4816
  }