@memberjunction/realtime-runtime 6.2.0-edge.4 → 6.2.0-edge.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/hosts/IRealtimeMediaHost.d.ts +25 -30
- package/dist/hosts/IRealtimeMediaHost.d.ts.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -2
- package/dist/index.js.map +1 -1
- package/dist/session/RealtimeSessionRuntime.d.ts +372 -73
- package/dist/session/RealtimeSessionRuntime.d.ts.map +1 -1
- package/dist/session/RealtimeSessionRuntime.js +919 -212
- package/dist/session/RealtimeSessionRuntime.js.map +1 -1
- package/dist/session/RecordingShardDrain.d.ts +59 -0
- package/dist/session/RecordingShardDrain.d.ts.map +1 -0
- package/dist/session/RecordingShardDrain.js +135 -0
- package/dist/session/RecordingShardDrain.js.map +1 -0
- package/dist/session/RecordingShardQueue.d.ts +64 -38
- package/dist/session/RecordingShardQueue.d.ts.map +1 -1
- package/dist/session/RecordingShardQueue.js +106 -84
- package/dist/session/RecordingShardQueue.js.map +1 -1
- package/dist/session/RecordingShardStore.d.ts +57 -0
- package/dist/session/RecordingShardStore.d.ts.map +1 -0
- package/dist/session/RecordingShardStore.js +184 -0
- package/dist/session/RecordingShardStore.js.map +1 -0
- package/dist/session/RecordingShardWindow.d.ts +42 -0
- package/dist/session/RecordingShardWindow.d.ts.map +1 -0
- package/dist/session/RecordingShardWindow.js +88 -0
- package/dist/session/RecordingShardWindow.js.map +1 -0
- package/dist/session/RecordingTransport.d.ts +67 -0
- package/dist/session/RecordingTransport.d.ts.map +1 -0
- package/dist/session/RecordingTransport.js +61 -0
- package/dist/session/RecordingTransport.js.map +1 -0
- package/package.json +10 -9
|
@@ -5,8 +5,8 @@ import { AddRealtimeUsageRecord, AIEngineBase, HasRealtimeUsage } from '@memberj
|
|
|
5
5
|
import { GraphQLRealtimeSessionClient } from '@memberjunction/graphql-dataprovider';
|
|
6
6
|
import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
|
|
7
7
|
import { DEFAULT_REALTIME_AUDIO_TRACKS, ParseRealtimeAvatarStatus, ParseRealtimeClientTransport } from '@memberjunction/ai';
|
|
8
|
-
import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
|
|
9
|
-
import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
|
|
8
|
+
import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, RECORDING_TERMINAL_MAX_BYTES, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
|
|
9
|
+
import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadElevenRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
|
|
10
10
|
import { DefaultRealtimeSessionLauncher, HasClientCredential } from './session-launcher.js';
|
|
11
11
|
import { RequestsAgentVideo, ResolveAvatarNotice } from './avatar-notice.js';
|
|
12
12
|
import { ClientSessionDeadline } from './client-session-deadline.js';
|
|
@@ -23,6 +23,26 @@ import { AppClientToolRegistry, DEFAULT_APP_TOOL_OWNER } from './app-client-tool
|
|
|
23
23
|
import { UserSettingsExposurePreferences } from './channel-exposure-preferences.js';
|
|
24
24
|
import { BuildChannelCandidate, FindPreparedChannel, MergeToolMetadata, PreparedChannelsShowAgentVideo, ReconcileChannelsWithPolicy, ResolveLocalChannelScope, ToolsByChannelKey } from './channel-session-scope.js';
|
|
25
25
|
import { RecordingShardQueue } from './RecordingShardQueue.js';
|
|
26
|
+
import { RecordingShardWindow } from './RecordingShardWindow.js';
|
|
27
|
+
import { CreateRecordingShardStore } from './RecordingShardStore.js';
|
|
28
|
+
import { CreateRecordingTransportForProvider, DeleteStoredRecordingShard, DrainStoredRecordingShardsOnce } from './RecordingShardDrain.js';
|
|
29
|
+
/**
|
|
30
|
+
* How long teardown waits for the server to acknowledge the recording's terminal request, measured
|
|
31
|
+
* from after `CloseAgentSession`. The terminal is `keepalive`, so it completes even if the page goes
|
|
32
|
+
* away; the wait only keeps `SavingRecording$` (and so a host's "don't close yet" guard) up while the
|
|
33
|
+
* server has not confirmed. Bounded so a dead server can never hold the End of a call hostage —
|
|
34
|
+
* unacknowledged shards stay in the local store and the server janitor assembles what arrived.
|
|
35
|
+
*/
|
|
36
|
+
export const RECORDING_TERMINAL_ACK_TIMEOUT_MS = 3000;
|
|
37
|
+
/**
|
|
38
|
+
* How long teardown waits for the recorder to stop and flush its encoder. `Stop()` runs BEFORE
|
|
39
|
+
* `CloseAgentSession`, so a wedged encoder must not be able to hold the End of a call (and the
|
|
40
|
+
* server session) open; past this the recording ends at the packets already encoded.
|
|
41
|
+
*/
|
|
42
|
+
export const RECORDER_STOP_TIMEOUT_MS = 2000;
|
|
43
|
+
function describeRecordingFailure(failure) {
|
|
44
|
+
return failure instanceof Error ? failure.message : String(failure);
|
|
45
|
+
}
|
|
26
46
|
/**
|
|
27
47
|
* `MJ: User Settings` key for the per-user "record this voice call" consent toggle. Stored as
|
|
28
48
|
* the literal string `'true'`/`'false'` (read with `=== 'true'`), cross-device via
|
|
@@ -52,6 +72,7 @@ LoadGeminiEnterpriseRealtimeClient();
|
|
|
52
72
|
LoadElevenLabsRealtimeClient();
|
|
53
73
|
LoadAssemblyAIRealtimeClient();
|
|
54
74
|
LoadxAIRealtimeClient();
|
|
75
|
+
LoadElevenRealtimeClient();
|
|
55
76
|
LoadHuggingFaceRealtimeClient();
|
|
56
77
|
/**
|
|
57
78
|
* A registry row's surface placement. A row whose `UIConfig` is not valid JSON places its surface as a row without one
|
|
@@ -149,6 +170,14 @@ function microphoneStartError(reason, message) {
|
|
|
149
170
|
error.name = MICROPHONE_ERROR_NAMES[reason];
|
|
150
171
|
return error;
|
|
151
172
|
}
|
|
173
|
+
/** Thrown by a start while a call is live: the runtime holds one driver, and never replaces a live one. */
|
|
174
|
+
export class RealtimeSessionAlreadyActiveError extends Error {
|
|
175
|
+
constructor(ActiveAgentSessionId) {
|
|
176
|
+
super(`A realtime session is already active${ActiveAgentSessionId ? ` (${ActiveAgentSessionId})` : ''}; end it before starting another.`);
|
|
177
|
+
this.ActiveAgentSessionId = ActiveAgentSessionId;
|
|
178
|
+
this.name = 'RealtimeSessionAlreadyActiveError';
|
|
179
|
+
}
|
|
180
|
+
}
|
|
152
181
|
/**
|
|
153
182
|
* Drives a **client-direct** real-time voice session: the browser mints an ephemeral
|
|
154
183
|
* token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
|
|
@@ -174,9 +203,9 @@ function microphoneStartError(reason, message) {
|
|
|
174
203
|
export class RealtimeSessionRuntime {
|
|
175
204
|
/**
|
|
176
205
|
* @param mediaHost the platform's media capabilities. The runtime never touches `navigator`,
|
|
177
|
-
* `Blob` or `FileReader` itself — microphone acquisition and audio
|
|
178
|
-
* are the host's, because both are platform-specific
|
|
179
|
-
*
|
|
206
|
+
* `Blob` or `FileReader` itself — microphone acquisition and audio capture/encoding
|
|
207
|
+
* are the host's, because both are platform-specific (permission UX, which
|
|
208
|
+
* encoder the platform offers).
|
|
180
209
|
*/
|
|
181
210
|
constructor(mediaHost) {
|
|
182
211
|
this.mediaHost = mediaHost;
|
|
@@ -254,10 +283,11 @@ export class RealtimeSessionRuntime {
|
|
|
254
283
|
this.Minimized$ = this._minimized$.asObservable();
|
|
255
284
|
/**
|
|
256
285
|
* True while the end-of-call recording is being saved: from the first statement of a teardown
|
|
257
|
-
* that holds a recorder until that teardown settles
|
|
258
|
-
*
|
|
259
|
-
*
|
|
260
|
-
*
|
|
286
|
+
* that holds a recorder until that teardown settles — which includes waiting (at most
|
|
287
|
+
* {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}) for the server to acknowledge the recording's terminal
|
|
288
|
+
* request. False at construction and for unrecorded sessions. Exists for #5195; the runtime is
|
|
289
|
+
* platform-free, so hosts own the reaction (a "saving" indicator, a disabled navigation button,
|
|
290
|
+
* etc.).
|
|
261
291
|
*/
|
|
262
292
|
this.SavingRecording$ = this._savingRecording$.asObservable();
|
|
263
293
|
/**
|
|
@@ -341,8 +371,8 @@ export class RealtimeSessionRuntime {
|
|
|
341
371
|
* Fired EXACTLY ONCE per session as teardown begins, with the prior
|
|
342
372
|
* `agentSessionId` (so subscribers can correlate against `SessionStarted$`'s
|
|
343
373
|
* sessionId) and the client-distinguishable reason — `'explicit'` when the
|
|
344
|
-
* user called `EndRealtimeSession`, `'error'` when
|
|
345
|
-
*
|
|
374
|
+
* user called `EndRealtimeSession`, `'error'` when the runtime ended the call over a failure: a
|
|
375
|
+
* failed start, or a fatal error from the provider mid-call. Server-side close paths (janitor, shutdown) do NOT propagate here —
|
|
346
376
|
* they happen out-of-process and have no client push channel today.
|
|
347
377
|
*/
|
|
348
378
|
this.SessionEnded$ = this._sessionEnded$.asObservable();
|
|
@@ -423,23 +453,40 @@ export class RealtimeSessionRuntime {
|
|
|
423
453
|
// ── Browser-side call recording ────────────────────────────────────────────
|
|
424
454
|
/**
|
|
425
455
|
* The active session's audio recorder (mic + agent mix), or `null` when the user didn't
|
|
426
|
-
* consent or the browser can't record. Created after the client connects;
|
|
427
|
-
* at teardown.
|
|
456
|
+
* consent or the browser can't record. Created after the client connects; its packets are
|
|
457
|
+
* shipped as 5 s shards during the call, and it is stopped at teardown.
|
|
428
458
|
*/
|
|
429
459
|
this.recorder = null;
|
|
430
460
|
/** ISO timestamp of when recording started — sent to the server on session start. */
|
|
431
461
|
this.recordingStartedAtIso = null;
|
|
432
|
-
/** Interval that
|
|
433
|
-
this.
|
|
462
|
+
/** Interval that cuts and uploads a shard every {@link recordingShardFlushMs} during a recording. */
|
|
463
|
+
this.shardFlushTimer = null;
|
|
464
|
+
/**
|
|
465
|
+
* The current recording's shard pipeline (window → store → queue → transport), or null when not
|
|
466
|
+
* recording. A fresh pipeline per recording keeps a flush still in flight from a previous session
|
|
467
|
+
* off this session's indexes. Its queue is closed when the recording ends; anything it had not
|
|
468
|
+
* delivered stays in the local store for the next-load drain.
|
|
469
|
+
*/
|
|
470
|
+
this.recordingShards = null;
|
|
471
|
+
/**
|
|
472
|
+
* Bytes of this page's recording `keepalive` requests still in flight. The browser's keepalive budget
|
|
473
|
+
* is per page, and closing a tab fires `visibilitychange` and `pagehide` back to back, so each dispatch
|
|
474
|
+
* budgets against what the previous one has not finished sending.
|
|
475
|
+
*/
|
|
476
|
+
this.keepaliveBytesInFlight = 0;
|
|
477
|
+
/** `sessionId/index` of shards whose `keepalive` copy is still in flight; never sent twice at once. */
|
|
478
|
+
this.keepaliveShardsInFlight = new Set();
|
|
479
|
+
/** The durable local shard store, created on first use and shared by every recording of this runtime. */
|
|
480
|
+
this.shardStore = null;
|
|
434
481
|
/**
|
|
435
|
-
*
|
|
436
|
-
*
|
|
437
|
-
* still in flight from a previous session off this session's indexes. Null when not recording.
|
|
438
|
-
* A recording that ends discards its queue, and any retained shard with it: the consolidated
|
|
439
|
-
* end-of-call upload supersedes them, and if that upload is lost too, server-side recovery
|
|
440
|
-
* (#5196) fills the gap with silence.
|
|
482
|
+
* True once a drain ran with a transport; the drain then does nothing until a recording ends and
|
|
483
|
+
* re-arms it ({@link rearmPendingShardDrain}).
|
|
441
484
|
*/
|
|
442
|
-
this.
|
|
485
|
+
this.pendingShardsDrained = false;
|
|
486
|
+
/** Bumped by every re-arm, so a drain already in flight cannot mark itself done over a re-arm. */
|
|
487
|
+
this.drainGeneration = 0;
|
|
488
|
+
/** The drain in flight, so overlapping callers share it instead of uploading the same shards twice. */
|
|
489
|
+
this.drainInFlight = null;
|
|
443
490
|
// ── Server-side liveness ───────────────────────────────────────────────────
|
|
444
491
|
/**
|
|
445
492
|
* Interval that tells the server this session is still in use, or `null` when no session is
|
|
@@ -450,12 +497,40 @@ export class RealtimeSessionRuntime {
|
|
|
450
497
|
* Recording-relative ms offset at which the IN-FLIGHT (not-yet-finalized) turn's audio
|
|
451
498
|
* actually BEGAN — captured the moment that turn's audio/text starts flowing (its first
|
|
452
499
|
* interim transcript), NOT inherited from the previous turn's end. `null` before the first
|
|
453
|
-
* turn / between turns (until the next turn's audio starts)
|
|
454
|
-
*
|
|
455
|
-
*
|
|
456
|
-
* the
|
|
500
|
+
* turn / between turns (until the next turn's audio starts), and it stays `null` for a turn
|
|
501
|
+
* whose start was never measured. Sent as `utteranceStartMs` on the turn's final transcript
|
|
502
|
+
* AS-IS — `null` is relayed as `null`, never replaced with 0 — so per-turn timing lines up
|
|
503
|
+
* with the recording even when a tool-call / silence gap sits between turns (the
|
|
504
|
+
* inherit-previous-end model mis-stamped the post-gap turn at the pre-gap offset). See
|
|
505
|
+
* {@link markTurnAudioStart}. A user speech onset signalled by the client does NOT land here;
|
|
506
|
+
* it has its own slot, {@link userSpeechStartMs}.
|
|
457
507
|
*/
|
|
458
508
|
this.currentTurnStartMs = null;
|
|
509
|
+
/**
|
|
510
|
+
* Recording-relative ms offset of the first user speech onset the client signalled
|
|
511
|
+
* (`OnUserSpeechStart`) since the last final user transcript, or `null` when none was signalled.
|
|
512
|
+
* It has its own slot, apart from {@link currentTurnStartMs}, so a user onset never claims the
|
|
513
|
+
* assistant turn's start: OpenAI user transcripts arrive asynchronously, often after the
|
|
514
|
+
* assistant's interims (or its final) for the reply have already flowed. See
|
|
515
|
+
* {@link markUserSpeechStart}.
|
|
516
|
+
*/
|
|
517
|
+
this.userSpeechStartMs = null;
|
|
518
|
+
/**
|
|
519
|
+
* `true` once an assistant turn has finalized after {@link userSpeechStartMs} was stamped BEFORE
|
|
520
|
+
* that turn began. The held onset may still be claimed by a late user transcript (OpenAI
|
|
521
|
+
* transcribes asynchronously), but the NEXT onset replaces it: an onset no transcript ever claimed
|
|
522
|
+
* (a VAD false positive, a failed transcription) must not become the start of the next real user
|
|
523
|
+
* turn. A barge-in onset (stamped during the assistant turn) is not superseded by that turn's
|
|
524
|
+
* final. An EMPTY user transcript releases the onset outright (see onClientTranscript).
|
|
525
|
+
*/
|
|
526
|
+
this.userSpeechStartSuperseded = false;
|
|
527
|
+
/**
|
|
528
|
+
* Recording-relative ms offsets of user speech onsets keyed by the provider's conversation item id
|
|
529
|
+
* (`input_audio_buffer.speech_started.item_id`), paired with the matching final user transcript
|
|
530
|
+
* (`RealtimeClientTranscript.ItemId`) so a late user transcript never claims a later turn's onset
|
|
531
|
+
* (MJ#5902). Unkeyed drivers continue to use {@link userSpeechStartMs}.
|
|
532
|
+
*/
|
|
533
|
+
this.userSpeechStartByItemId = new Map();
|
|
459
534
|
/**
|
|
460
535
|
* Wall-anchor of the SESSION clock (#3832): `performance.now()` at the moment the call went
|
|
461
536
|
* live, or `null` before any call has. Read only through {@link nowTurnOffsetMs}.
|
|
@@ -540,6 +615,15 @@ export class RealtimeSessionRuntime {
|
|
|
540
615
|
* two `SessionEnded$` emissions for one session.
|
|
541
616
|
*/
|
|
542
617
|
this.teardownInFlight = null;
|
|
618
|
+
/**
|
|
619
|
+
* True from the moment a start finishes (connected, liveness pulse running) until the slot is
|
|
620
|
+
* released. A driver's Fatal ends the call only while this is set: during a start, drivers emit
|
|
621
|
+
* Fatal from their socket handlers while Connect is still pending, and the start's own failure path
|
|
622
|
+
* ({@link failSessionStart}) is what reports that error, whether Connect then rejects or resolves
|
|
623
|
+
* (runMintedSession checks {@link lastSessionError} after Connect). Not {@link isSessionLive}: drivers move the
|
|
624
|
+
* state to 'listening' from inside Connect, and a live call that is resuming reads 'connecting'.
|
|
625
|
+
*/
|
|
626
|
+
this.callWentLive = false;
|
|
543
627
|
/**
|
|
544
628
|
* The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
|
|
545
629
|
* back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
|
|
@@ -662,6 +746,16 @@ export class RealtimeSessionRuntime {
|
|
|
662
746
|
* Owner-keyed — see {@link AppClientToolRegistry}.
|
|
663
747
|
*/
|
|
664
748
|
this.appToolRegistry = new AppClientToolRegistry();
|
|
749
|
+
// Re-upload shards a previous, interrupted call left in the local store. Deferred a tick so a
|
|
750
|
+
// host can assign `Provider` right after construction; when it still has no transport the drain
|
|
751
|
+
// defers itself and session start tries again. Unref'd so it never holds a Node process open.
|
|
752
|
+
const drainTimer = setTimeout(() => {
|
|
753
|
+
void this.DrainPendingRecordingShards();
|
|
754
|
+
}, 0);
|
|
755
|
+
// Node returns a Timeout object with unref(); browsers return a number.
|
|
756
|
+
if (typeof drainTimer === 'object' && drainTimer !== null && 'unref' in drainTimer && typeof drainTimer.unref === 'function') {
|
|
757
|
+
drainTimer.unref();
|
|
758
|
+
}
|
|
665
759
|
}
|
|
666
760
|
/** Synchronous read of {@link SavingRecording$}. */
|
|
667
761
|
get IsSavingRecording() {
|
|
@@ -730,8 +824,20 @@ export class RealtimeSessionRuntime {
|
|
|
730
824
|
UpdateAppContext(snapshot) {
|
|
731
825
|
this._appContext$.next(snapshot);
|
|
732
826
|
}
|
|
733
|
-
/** How often
|
|
734
|
-
static { this.
|
|
827
|
+
/** How often packets are cut into shards: the 5 s shard window, so each tick seals about one window. */
|
|
828
|
+
static { this.recordingShardFlushMs = 5000; }
|
|
829
|
+
/**
|
|
830
|
+
* Bytes of `keepalive` request bodies one terminal may use, the tail included. Browsers share about
|
|
831
|
+
* 64 KiB of in-flight keepalive bodies per page; the brief reserves 48 KiB of it for the recording,
|
|
832
|
+
* leaving room for a host's own close beacon (Caliber sends one from `pagehide` too).
|
|
833
|
+
*/
|
|
834
|
+
static { this.recordingKeepaliveBudgetBytes = RECORDING_TERMINAL_MAX_BYTES; }
|
|
835
|
+
/**
|
|
836
|
+
* Most runs one drain call makes (each run is itself bounded): it runs again when it was re-armed while
|
|
837
|
+
* running, or when a run deleted shards the server refused for good, which frees the page for the
|
|
838
|
+
* shards behind them.
|
|
839
|
+
*/
|
|
840
|
+
static { this.maxDrainRunsPerCall = 3; }
|
|
735
841
|
/**
|
|
736
842
|
* How often the client asserts liveness. Comfortably under `SessionJanitor`'s
|
|
737
843
|
* `closeThresholdMinutes` (15 by default) so several pulses must be missed in a row before a
|
|
@@ -873,6 +979,25 @@ export class RealtimeSessionRuntime {
|
|
|
873
979
|
IsActiveFor(conversationId) {
|
|
874
980
|
return this.IsActive && conversationId != null && UUIDsEqual(this.sessionConversationId, conversationId);
|
|
875
981
|
}
|
|
982
|
+
/**
|
|
983
|
+
* Takes the runtime's one driver slot for a new start; the two starts call it after awaiting any
|
|
984
|
+
* teardown in flight, and only when no End arrived during that wait (a changed {@link startGeneration}).
|
|
985
|
+
*
|
|
986
|
+
* A teardown still in flight is awaited rather than read as "busy": ending a call and starting the
|
|
987
|
+
* next one back to back is normal host behaviour (a retry, a re-mounted surface), and the flag it
|
|
988
|
+
* used to read flips only at the END of teardown, so the start was dropped with nothing to show
|
|
989
|
+
* for it — a bare `undefined`, which hosts had to guess at (MJ#4629). A call that is genuinely
|
|
990
|
+
* live is refused with an error the host can show; it is never silently ignored, and never
|
|
991
|
+
* overwritten, since a driver no slot points at can never be stopped again.
|
|
992
|
+
*
|
|
993
|
+
* Deliberately synchronous: the start's state must be set in the tick the host called it in, or an
|
|
994
|
+
* End issued straight after the Start (a user tapping back during the mint) finds nothing to abandon.
|
|
995
|
+
*/
|
|
996
|
+
claimSessionSlot() {
|
|
997
|
+
if (this.IsActive) {
|
|
998
|
+
throw new RealtimeSessionAlreadyActiveError(this.agentSessionId);
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
876
1001
|
/**
|
|
877
1002
|
* Start a client-direct voice session fronting `targetAgentId`.
|
|
878
1003
|
*
|
|
@@ -903,7 +1028,7 @@ export class RealtimeSessionRuntime {
|
|
|
903
1028
|
* pickers, and never synthesize overrides beyond what the user explicitly chose.
|
|
904
1029
|
* Omit/`null` for the server's defaults (today's behavior).
|
|
905
1030
|
* @param recordingConsent Optional EXPLICIT "record this call" consent for THIS session. When
|
|
906
|
-
* `true`, the browser records a mic + agent-audio mix and uploads it
|
|
1031
|
+
* `true`, the browser records a mic + agent-audio mix and uploads it as 5 s Opus shards during the call. When
|
|
907
1032
|
* omitted/`null`, the per-user persisted preference (`mj.realtimeVoice.recordingConsent.v1`
|
|
908
1033
|
* via {@link UserInfoEngine}) is read as the default. `false` never records.
|
|
909
1034
|
* @param mediaCollectionId Optional per-session media-kit override (`MJ: Collections.ID`). When set,
|
|
@@ -919,13 +1044,22 @@ export class RealtimeSessionRuntime {
|
|
|
919
1044
|
* ({@link RealtimeSessionStartOptions.ConversationBranchID}; omitted is Main, where its transcript is written).
|
|
920
1045
|
*/
|
|
921
1046
|
async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext, options) {
|
|
922
|
-
|
|
923
|
-
|
|
1047
|
+
// Captured BEFORE any wait: a teardown from here on bumps it, and the checks below (and in
|
|
1048
|
+
// runMintedSession) abandon the start. Re-capturing after the wait would absorb an End issued during it.
|
|
1049
|
+
const generation = this.startGeneration;
|
|
1050
|
+
// Synchronous unless a teardown is in flight: a start must take the slot in the same tick it was
|
|
1051
|
+
// called in, so a host's End issued right after it finds a start to abandon (see claimSessionSlot).
|
|
1052
|
+
if (this.teardownInFlight) {
|
|
1053
|
+
await this.teardownInFlight;
|
|
1054
|
+
// An End issued during the wait found IsActive still true (the last call was closing), so it only
|
|
1055
|
+
// joined that teardown and bumped the generation. Nothing is acquired yet, so nothing to release.
|
|
1056
|
+
if (this.startGeneration !== generation) {
|
|
1057
|
+
return;
|
|
1058
|
+
}
|
|
924
1059
|
}
|
|
1060
|
+
this.claimSessionSlot();
|
|
925
1061
|
const consent = this.beginSessionStart({ agentName, recordingConsent, applicationId, appContext });
|
|
926
1062
|
this.sessionConversationId = conversationId ?? null;
|
|
927
|
-
// A teardown while the channels are prepared or the mint runs bumps this; see the checks below.
|
|
928
|
-
const generation = this.startGeneration;
|
|
929
1063
|
// Captured BEFORE the channels are prepared so the mint carries exactly the snapshot the prologue
|
|
930
1064
|
// resolved, whatever a channel plugin may push in the meantime.
|
|
931
1065
|
const effectiveAppContext = this._appContext$.value;
|
|
@@ -951,14 +1085,14 @@ export class RealtimeSessionRuntime {
|
|
|
951
1085
|
// The host ended the start while the mint ran. That teardown had no session id to close, so
|
|
952
1086
|
// close the minted row here, and touch no shared state: a newer start may already own it.
|
|
953
1087
|
if (this.startGeneration !== generation) {
|
|
954
|
-
await this.closeServerSession(session.AgentSessionId);
|
|
1088
|
+
await this.closeServerSession(session.AgentSessionId, 'Explicit');
|
|
955
1089
|
return;
|
|
956
1090
|
}
|
|
957
1091
|
if (!this.hostCanUseProvider(session.Provider)) {
|
|
958
1092
|
await this.abortUnusableSession(session);
|
|
959
1093
|
return;
|
|
960
1094
|
}
|
|
961
|
-
await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true);
|
|
1095
|
+
await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true, generation);
|
|
962
1096
|
}
|
|
963
1097
|
/**
|
|
964
1098
|
* Declares which provider keys this host can actually carry audio for.
|
|
@@ -994,8 +1128,7 @@ export class RealtimeSessionRuntime {
|
|
|
994
1128
|
// published them on ActiveChannels$; skipping teardown would leave them undisposed, their tool
|
|
995
1129
|
// handlers registered, and their subscriptions live until the next start replaced them.
|
|
996
1130
|
this.agentSessionId = session.AgentSessionId ?? this.agentSessionId;
|
|
997
|
-
this.
|
|
998
|
-
await this.teardown(true);
|
|
1131
|
+
await this.endOnFailure();
|
|
999
1132
|
}
|
|
1000
1133
|
/**
|
|
1001
1134
|
* Run a session the HOST has already minted itself — the second half of
|
|
@@ -1019,22 +1152,28 @@ export class RealtimeSessionRuntime {
|
|
|
1019
1152
|
* Every field defaults exactly as its {@link StartRealtimeSession} counterpart does.
|
|
1020
1153
|
*/
|
|
1021
1154
|
async StartRealtimeSessionFromResult(result, options) {
|
|
1022
|
-
|
|
1023
|
-
|
|
1155
|
+
// Same wait, same generation check, same reasons as StartRealtimeSession.
|
|
1156
|
+
const generation = this.startGeneration;
|
|
1157
|
+
if (this.teardownInFlight) {
|
|
1158
|
+
await this.teardownInFlight;
|
|
1159
|
+
if (this.startGeneration !== generation) {
|
|
1160
|
+
return;
|
|
1161
|
+
}
|
|
1024
1162
|
}
|
|
1163
|
+
this.claimSessionSlot();
|
|
1025
1164
|
const effectiveOptions = options ?? {};
|
|
1026
1165
|
if (!this.hostCanUseProvider(result.Provider)) {
|
|
1027
1166
|
await this.abortUnusableSession(result);
|
|
1028
1167
|
return;
|
|
1029
1168
|
}
|
|
1030
1169
|
const consent = this.beginSessionStart(effectiveOptions);
|
|
1031
|
-
await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false);
|
|
1170
|
+
await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false, generation);
|
|
1032
1171
|
}
|
|
1033
1172
|
/**
|
|
1034
1173
|
* Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
|
|
1035
1174
|
* publish the agent name, reset per-session state, and flip the session live (which is ALSO what
|
|
1036
|
-
* makes
|
|
1037
|
-
* runs before minting, not after). Returns the resolved recording consent, which the mint half
|
|
1175
|
+
* makes {@link claimSessionSlot} refuse a second start with {@link RealtimeSessionAlreadyActiveError}
|
|
1176
|
+
* while the mint is still in flight — hence it runs before minting, not after). Returns the resolved recording consent, which the mint half
|
|
1038
1177
|
* reports to the server and the run half uses to decide whether to record.
|
|
1039
1178
|
*/
|
|
1040
1179
|
beginSessionStart(options) {
|
|
@@ -1060,6 +1199,9 @@ export class RealtimeSessionRuntime {
|
|
|
1060
1199
|
this.resetState();
|
|
1061
1200
|
this._active$.next(true);
|
|
1062
1201
|
this.setConnectionState('connecting');
|
|
1202
|
+
// A drain the constructor deferred (no provider/transport yet) gets its second chance here, and
|
|
1203
|
+
// runs whether or not THIS session records: the pending shards belong to an earlier call.
|
|
1204
|
+
void this.DrainPendingRecordingShards();
|
|
1063
1205
|
// Resolve recording consent for this session: explicit value wins, else the per-user
|
|
1064
1206
|
// persisted preference. Computed before mint so it can be reported to the server.
|
|
1065
1207
|
const consent = options.recordingConsent ?? this.readPersistedRecordingConsent();
|
|
@@ -1071,10 +1213,10 @@ export class RealtimeSessionRuntime {
|
|
|
1071
1213
|
* the provider connection, and go live. `inputConversationId` is the conversation the START asked
|
|
1072
1214
|
* for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
|
|
1073
1215
|
* whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
|
|
1216
|
+
* `generation` is the {@link startGeneration} the entry point captured before its first await; every
|
|
1217
|
+
* await below is a window in which the host can end the session, and so was every await before this.
|
|
1074
1218
|
*/
|
|
1075
|
-
async runMintedSession(session, inputConversationId, consent, cameraCheck) {
|
|
1076
|
-
// Captured up front: every await below is a window in which the host can end the session.
|
|
1077
|
-
const generation = this.startGeneration;
|
|
1219
|
+
async runMintedSession(session, inputConversationId, consent, cameraCheck, generation) {
|
|
1078
1220
|
try {
|
|
1079
1221
|
// Mount the channels the resolved policy puts in the session. Deliberately BEFORE the session
|
|
1080
1222
|
// id is adopted: `ActiveChannels$` consumers (the sessions adapter) treat an emission that
|
|
@@ -1123,6 +1265,13 @@ export class RealtimeSessionRuntime {
|
|
|
1123
1265
|
if (connectError) {
|
|
1124
1266
|
throw connectError;
|
|
1125
1267
|
}
|
|
1268
|
+
// A Fatal the driver sent during this start (the fatal handler leaves it to the start, see
|
|
1269
|
+
// callWentLive) fails the start even when Connect resolved: the real drivers keep awaiting mic
|
|
1270
|
+
// capture after their connect-failure channel settles, then report 'listening' and resolve on a
|
|
1271
|
+
// socket that already closed. lastSessionError is reset per start and set only by the current client.
|
|
1272
|
+
if (this.lastSessionError?.Fatal) {
|
|
1273
|
+
throw new Error(this.lastSessionError.Message);
|
|
1274
|
+
}
|
|
1126
1275
|
// Tracks are negotiated now, so a capture can tell whether the model takes video, and the call whether it shows
|
|
1127
1276
|
// the avatar its agent asked for. The driver holds the microphone now, so the user may switch it.
|
|
1128
1277
|
this.openCaptures(client, cameraCheck);
|
|
@@ -1174,6 +1323,7 @@ export class RealtimeSessionRuntime {
|
|
|
1174
1323
|
// Same place, same reason: the session is connected and its id is known, which is exactly
|
|
1175
1324
|
// the window in which the server needs to be told it is alive.
|
|
1176
1325
|
this.startLivenessPulse();
|
|
1326
|
+
this.callWentLive = true;
|
|
1177
1327
|
}
|
|
1178
1328
|
catch (error) {
|
|
1179
1329
|
await this.failSessionStart(error);
|
|
@@ -1235,7 +1385,7 @@ export class RealtimeSessionRuntime {
|
|
|
1235
1385
|
// session, and clearing the id would leave that teardown no session to report.
|
|
1236
1386
|
if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId && !this.teardownInFlight) {
|
|
1237
1387
|
this.agentSessionId = null;
|
|
1238
|
-
await this.closeServerSession(session.AgentSessionId);
|
|
1388
|
+
await this.closeServerSession(session.AgentSessionId, 'Explicit');
|
|
1239
1389
|
}
|
|
1240
1390
|
}
|
|
1241
1391
|
/**
|
|
@@ -1546,8 +1696,20 @@ export class RealtimeSessionRuntime {
|
|
|
1546
1696
|
async failSessionStart(error) {
|
|
1547
1697
|
console.error('[RealtimeSession] Failed to start session:', error);
|
|
1548
1698
|
this.lastStartError = error instanceof Error ? error : new Error(String(error));
|
|
1699
|
+
await this.endOnFailure();
|
|
1700
|
+
}
|
|
1701
|
+
/**
|
|
1702
|
+
* Ends the call over a failure: a failed start, an unusable provider, or a fatal provider error.
|
|
1703
|
+
*
|
|
1704
|
+
* The teardown is claimed BEFORE `'error'` is published. `ConnectionState$` emits synchronously, and a
|
|
1705
|
+
* host may end the call the moment it sees `'error'` (MobileApp's voice screen does); claimed first, that
|
|
1706
|
+
* host's End joins this teardown instead of starting its own as `'explicit'`, so the row still closes as
|
|
1707
|
+
* `Error` and `SessionEnded$` still says `'error'`. Teardown keeps a terminal `'error'` state.
|
|
1708
|
+
*/
|
|
1709
|
+
endOnFailure() {
|
|
1710
|
+
const ending = this.teardown('error');
|
|
1549
1711
|
this.setConnectionState('error');
|
|
1550
|
-
|
|
1712
|
+
return ending;
|
|
1551
1713
|
}
|
|
1552
1714
|
/**
|
|
1553
1715
|
* End the active session: stop the mic, tear down the provider connection, and close
|
|
@@ -1557,7 +1719,7 @@ export class RealtimeSessionRuntime {
|
|
|
1557
1719
|
if (!this.IsActive && !this.agentSessionId) {
|
|
1558
1720
|
return;
|
|
1559
1721
|
}
|
|
1560
|
-
await this.teardown(
|
|
1722
|
+
await this.teardown('explicit');
|
|
1561
1723
|
}
|
|
1562
1724
|
/**
|
|
1563
1725
|
* Inject a typed message into the live session as a user turn.
|
|
@@ -1583,7 +1745,7 @@ export class RealtimeSessionRuntime {
|
|
|
1583
1745
|
}
|
|
1584
1746
|
client.SendText(trimmed);
|
|
1585
1747
|
// Relay as a user turn — same path spoken input uses (caption + persisted transcript).
|
|
1586
|
-
void this.onUserTranscript(trimmed);
|
|
1748
|
+
void this.onUserTranscript(trimmed, 'typed');
|
|
1587
1749
|
}
|
|
1588
1750
|
/**
|
|
1589
1751
|
* Mute / unmute the local microphone track. Returns the new muted state. While the call has no microphone track (a
|
|
@@ -1648,7 +1810,17 @@ export class RealtimeSessionRuntime {
|
|
|
1648
1810
|
if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
|
|
1649
1811
|
return false;
|
|
1650
1812
|
}
|
|
1651
|
-
|
|
1813
|
+
// The opening is part of the conversation, so it goes out as a spoken TURN: captioned, relayed
|
|
1814
|
+
// and persisted like any turn the model takes on its own. It used to ride the delegation
|
|
1815
|
+
// narration channel, whose replies are disposable by contract, so the greeting the participant
|
|
1816
|
+
// heard was never in the transcript (MJ#4591). A driver from before the capability existed
|
|
1817
|
+
// falls back to that channel rather than leaving the session silent.
|
|
1818
|
+
if (this.client.RequestSpokenTurn) {
|
|
1819
|
+
this.client.RequestSpokenTurn(trimmed);
|
|
1820
|
+
}
|
|
1821
|
+
else {
|
|
1822
|
+
this.client.RequestSpokenUpdate(trimmed);
|
|
1823
|
+
}
|
|
1652
1824
|
return true;
|
|
1653
1825
|
}
|
|
1654
1826
|
/**
|
|
@@ -1728,6 +1900,9 @@ export class RealtimeSessionRuntime {
|
|
|
1728
1900
|
// turn one's cue wherever the clocks happen to differ.
|
|
1729
1901
|
this.currentTurnStartMs = recorder.IsRecording ? 0 : null;
|
|
1730
1902
|
this.turnAudioStartCaptured = false;
|
|
1903
|
+
this.userSpeechStartMs = null;
|
|
1904
|
+
this.userSpeechStartSuperseded = false;
|
|
1905
|
+
this.userSpeechStartByItemId.clear();
|
|
1731
1906
|
if (this.recorder) {
|
|
1732
1907
|
// WebRTC drivers (OpenAI): the agent's track usually lands AFTER Connect() resolves, so
|
|
1733
1908
|
// `remoteStream` above is null here and this handler attaches it later. PCM-playback
|
|
@@ -1735,7 +1910,7 @@ export class RealtimeSessionRuntime {
|
|
|
1735
1910
|
// `remoteStream` is already set and the handler fires immediately with the same stream;
|
|
1736
1911
|
// AttachRemoteStream is idempotent, so it is mixed only once.
|
|
1737
1912
|
client.OnRemoteMediaStream?.((stream) => this.recorder?.AttachRemoteStream(stream));
|
|
1738
|
-
this.
|
|
1913
|
+
this.startShardFlushing();
|
|
1739
1914
|
}
|
|
1740
1915
|
}
|
|
1741
1916
|
catch (error) {
|
|
@@ -1743,16 +1918,161 @@ export class RealtimeSessionRuntime {
|
|
|
1743
1918
|
this.recorder = null;
|
|
1744
1919
|
}
|
|
1745
1920
|
}
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1921
|
+
// ── Recording shards (MJ#5294) ───────────────────────────────────────────────
|
|
1922
|
+
//
|
|
1923
|
+
// The recorder hands over Opus packets; the runtime cuts them into 5 s `MJOPUS1` shards, writes
|
|
1924
|
+
// each one to the local store BEFORE its upload starts, uploads it as raw bytes, and deletes it from
|
|
1925
|
+
// the store only on the server's ack. The call ends with a `keepalive` terminal request carrying the
|
|
1926
|
+
// final partial shard and the counts the server needs to know it has everything.
|
|
1927
|
+
/**
|
|
1928
|
+
* Builds the transport for shard uploads and terminal requests. The default posts to MJAPI's REST
|
|
1929
|
+
* recording routes, derived from the session's GraphQL provider configuration, with credentials
|
|
1930
|
+
* read per request so a refreshed token is used. Returns null — shards then wait in the local
|
|
1931
|
+
* store — when the provider has no URL (a test double, a provider that is not GraphQL, or one not
|
|
1932
|
+
* configured yet). Override to supply another transport.
|
|
1933
|
+
*/
|
|
1934
|
+
createRecordingTransport() {
|
|
1935
|
+
return CreateRecordingTransportForProvider(this.Provider);
|
|
1936
|
+
}
|
|
1937
|
+
/**
|
|
1938
|
+
* Builds the durable local shard store. The default is IndexedDB where the host has it and a no-op
|
|
1939
|
+
* store elsewhere (Node, some React Native runtimes). Called once per runtime. Override to supply
|
|
1940
|
+
* another store.
|
|
1941
|
+
*/
|
|
1942
|
+
createRecordingShardStore() {
|
|
1943
|
+
return CreateRecordingShardStore(globalThis.indexedDB);
|
|
1944
|
+
}
|
|
1945
|
+
/** The provider's signed-in user, or null (no provider yet, or an anonymous page). */
|
|
1946
|
+
signedInUserID() {
|
|
1947
|
+
return this.Provider?.CurrentUser?.ID ?? null;
|
|
1948
|
+
}
|
|
1949
|
+
/** The runtime's shard store, created on first use. */
|
|
1950
|
+
ensureShardStore() {
|
|
1951
|
+
this.shardStore ??= this.createRecordingShardStore();
|
|
1952
|
+
return this.shardStore;
|
|
1953
|
+
}
|
|
1954
|
+
/**
|
|
1955
|
+
* Re-uploads shards an earlier, interrupted call left in the local store (crash, closed tab, dead
|
|
1956
|
+
* network), each under its OWN session id and index, oldest first, at most
|
|
1957
|
+
* {@link MAX_DRAINED_SHARDS_PER_LOAD} per runtime. A shard is deleted only on the server's ack.
|
|
1958
|
+
*
|
|
1959
|
+
* Runs once per runtime instance, re-armed whenever a recording ends (which may leave shards behind
|
|
1960
|
+
* for the next session start on the same page). When there is no transport yet (no provider configuration, e.g.
|
|
1961
|
+
* before login) it returns 0 WITHOUT marking itself done, so a later call — session start — drains
|
|
1962
|
+
* instead of the shards being skipped for this page. Concurrent calls share one run. Never rejects.
|
|
1963
|
+
*
|
|
1964
|
+
* @returns the number of shards the server acknowledged in this call.
|
|
1965
|
+
*/
|
|
1966
|
+
DrainPendingRecordingShards() {
|
|
1967
|
+
if (this.pendingShardsDrained) {
|
|
1968
|
+
return Promise.resolve(0);
|
|
1969
|
+
}
|
|
1970
|
+
this.drainInFlight ??= this.drainPendingShards().finally(() => {
|
|
1971
|
+
this.drainInFlight = null;
|
|
1972
|
+
});
|
|
1973
|
+
return this.drainInFlight;
|
|
1974
|
+
}
|
|
1975
|
+
/** Makes the next {@link DrainPendingRecordingShards} call run again, even one already in flight. */
|
|
1976
|
+
rearmPendingShardDrain() {
|
|
1977
|
+
this.pendingShardsDrained = false;
|
|
1978
|
+
this.drainGeneration++;
|
|
1979
|
+
}
|
|
1980
|
+
/**
|
|
1981
|
+
* The body of {@link DrainPendingRecordingShards}, up to {@link maxDrainRunsPerCall} runs. It runs
|
|
1982
|
+
* again after a re-arm while a run is in flight (a recovery after an outage gave up on more shards),
|
|
1983
|
+
* because the running pass listed the store before those shards were there; and after a run deleted
|
|
1984
|
+
* refused shards, because those filled the page that the shards behind them now get.
|
|
1985
|
+
*/
|
|
1986
|
+
async drainPendingShards() {
|
|
1987
|
+
try {
|
|
1988
|
+
const transport = this.createRecordingTransport();
|
|
1989
|
+
if (!transport) {
|
|
1990
|
+
return 0;
|
|
1991
|
+
}
|
|
1992
|
+
let delivered = 0;
|
|
1993
|
+
for (let run = 1; run <= RealtimeSessionRuntime.maxDrainRunsPerCall; run++) {
|
|
1994
|
+
// A re-arm while this run is in flight must win over this run marking itself done.
|
|
1995
|
+
const generation = this.drainGeneration;
|
|
1996
|
+
const outcome = await this.drainStoredShardsOnce(transport);
|
|
1997
|
+
delivered += outcome.Delivered;
|
|
1998
|
+
// Done only when every listed shard was delivered or refused for good: a drain during an outage
|
|
1999
|
+
// must not stop the next session start or recovery from trying again.
|
|
2000
|
+
this.pendingShardsDrained = generation === this.drainGeneration && outcome.Delivered + outcome.Refused === outcome.Listed;
|
|
2001
|
+
if (generation === this.drainGeneration && outcome.Refused === 0) {
|
|
2002
|
+
break;
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
return delivered;
|
|
2006
|
+
}
|
|
2007
|
+
catch (error) {
|
|
2008
|
+
console.warn(`[RealtimeSession] Draining stored recording shards failed: ${describeRecordingFailure(error)}; will retry at the next session start.`, error);
|
|
2009
|
+
return 0;
|
|
2010
|
+
}
|
|
2011
|
+
}
|
|
2012
|
+
/** One drain pass over the signed-in user's stored shards (see {@link DrainStoredRecordingShardsOnce}). */
|
|
2013
|
+
drainStoredShardsOnce(transport) {
|
|
2014
|
+
return DrainStoredRecordingShardsOnce(this.ensureShardStore(), transport, this.signedInUserID());
|
|
2015
|
+
}
|
|
2016
|
+
/** Builds this recording's shard pipeline and starts the 5 s flush. */
|
|
2017
|
+
startShardFlushing() {
|
|
2018
|
+
const sessionId = this.agentSessionId;
|
|
2019
|
+
if (!sessionId) {
|
|
2020
|
+
// Unreachable in practice (the session id is adopted before recording starts), but without
|
|
2021
|
+
// one there is nowhere to send shards, so say so rather than recording into the void.
|
|
2022
|
+
console.warn('[RealtimeSession] Recording started without an agent session id; shards will not be saved.');
|
|
2023
|
+
return;
|
|
2024
|
+
}
|
|
2025
|
+
const transport = this.createRecordingTransport();
|
|
2026
|
+
if (!transport) {
|
|
2027
|
+
console.warn(`[RealtimeSession] No recording transport for session ${sessionId} (provider has no REST configuration); shards are kept in the local store only.`);
|
|
2028
|
+
}
|
|
2029
|
+
// True once the queue gave up on a shard and until an upload succeeds again: that first success
|
|
2030
|
+
// re-sends what the queue dropped meanwhile (see recoverStoredShardsAfterOutage).
|
|
2031
|
+
let shardsGivenUp = false;
|
|
2032
|
+
this.recordingShards = {
|
|
2033
|
+
SessionID: sessionId,
|
|
2034
|
+
UserID: this.signedInUserID(),
|
|
2035
|
+
Window: new RecordingShardWindow(),
|
|
2036
|
+
Transport: transport,
|
|
2037
|
+
// Closes over this recording's session id and transport — never the runtime's current ones.
|
|
2038
|
+
Queue: new RecordingShardQueue(async (shard) => {
|
|
2039
|
+
if (!transport) {
|
|
2040
|
+
return 'Retry';
|
|
2041
|
+
}
|
|
2042
|
+
// The outcome goes to the queue unchanged: it logs a different reason for each, and stops
|
|
2043
|
+
// retrying a `Refused` shard at once (#6012).
|
|
2044
|
+
const outcome = await transport.UploadShard(sessionId, shard);
|
|
2045
|
+
if (outcome === 'Acked') {
|
|
2046
|
+
await this.deleteStoredShard(sessionId, shard.Index);
|
|
2047
|
+
if (shardsGivenUp) {
|
|
2048
|
+
shardsGivenUp = false;
|
|
2049
|
+
this.recoverStoredShardsAfterOutage(sessionId);
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
return outcome;
|
|
2053
|
+
}, () => {
|
|
2054
|
+
shardsGivenUp = true;
|
|
2055
|
+
}),
|
|
2056
|
+
};
|
|
2057
|
+
this.shardFlushTimer = setInterval(() => { void this.flushRecordingShards(); }, RealtimeSessionRuntime.recordingShardFlushMs);
|
|
2058
|
+
}
|
|
2059
|
+
/**
|
|
2060
|
+
* Uploads succeed again after failing: re-sends, from the local store, the shards the queue gave up
|
|
2061
|
+
* on during the outage. The queue's attempts run out after about 10 s of a dead network (or an
|
|
2062
|
+
* expired token), and without this those shards would wait for a later page load — which a widget
|
|
2063
|
+
* guest never makes — leaving a permanent gap. Runs in the background, outside any page-lifecycle
|
|
2064
|
+
* handler, so the IndexedDB read is allowed. Never rejects.
|
|
2065
|
+
*/
|
|
2066
|
+
recoverStoredShardsAfterOutage(sessionId) {
|
|
2067
|
+
console.log(`[RealtimeSession] Recording uploads for session ${sessionId} recovered; re-sending stored shards.`);
|
|
2068
|
+
this.rearmPendingShardDrain();
|
|
2069
|
+
void this.DrainPendingRecordingShards();
|
|
1750
2070
|
}
|
|
1751
|
-
/** Stops the periodic
|
|
1752
|
-
|
|
1753
|
-
if (this.
|
|
1754
|
-
clearInterval(this.
|
|
1755
|
-
this.
|
|
2071
|
+
/** Stops the periodic shard flush. */
|
|
2072
|
+
stopShardFlushing() {
|
|
2073
|
+
if (this.shardFlushTimer) {
|
|
2074
|
+
clearInterval(this.shardFlushTimer);
|
|
2075
|
+
this.shardFlushTimer = null;
|
|
1756
2076
|
}
|
|
1757
2077
|
}
|
|
1758
2078
|
/**
|
|
@@ -1811,128 +2131,345 @@ export class RealtimeSessionRuntime {
|
|
|
1811
2131
|
}
|
|
1812
2132
|
}
|
|
1813
2133
|
/**
|
|
1814
|
-
*
|
|
1815
|
-
*
|
|
1816
|
-
*
|
|
1817
|
-
*
|
|
2134
|
+
* One 5 s tick: cuts the packets encoded since the last tick into shards, stores each one, then
|
|
2135
|
+
* uploads them. Best-effort — never disturbs the call and never rejects. Overlapping ticks are
|
|
2136
|
+
* safe: `Append` and `SealTail` are synchronous, so indexes are assigned in order, and the queue
|
|
2137
|
+
* bounds how many flushes upload at once. A failed shard is retained by the queue and resent under
|
|
2138
|
+
* its own index on later ticks (#5197), and stays in the store until acknowledged.
|
|
2139
|
+
*
|
|
2140
|
+
* Every tick seals whatever it took, even under a full window (the window's 250-packet and byte caps
|
|
2141
|
+
* still split a larger batch). Capture starts slightly after this timer, so a tick almost never
|
|
2142
|
+
* holds exactly 250 packets; sealing only full windows would leave nearly 5 s of encoded audio in
|
|
2143
|
+
* memory at every moment, unstored and unsent, which a killed tab loses.
|
|
1818
2144
|
*/
|
|
1819
|
-
async
|
|
2145
|
+
async flushRecordingShards() {
|
|
1820
2146
|
const recorder = this.recorder;
|
|
1821
|
-
const
|
|
1822
|
-
|
|
1823
|
-
|
|
2147
|
+
const pipeline = this.recordingShards;
|
|
2148
|
+
if (!recorder || !pipeline) {
|
|
2149
|
+
return;
|
|
2150
|
+
}
|
|
2151
|
+
let sealed;
|
|
2152
|
+
try {
|
|
2153
|
+
sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
|
|
2154
|
+
const tick = pipeline.Window.SealTail();
|
|
2155
|
+
if (tick) {
|
|
2156
|
+
sealed.push(tick);
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
2159
|
+
catch (error) {
|
|
2160
|
+
console.warn(`[RealtimeSession] Failed to take encoded recording packets for session ${pipeline.SessionID}; this tick's audio is lost:`, error);
|
|
1824
2161
|
return;
|
|
1825
2162
|
}
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
2163
|
+
await this.storeShards(pipeline, sealed);
|
|
2164
|
+
await pipeline.Queue.Flush(sealed);
|
|
2165
|
+
}
|
|
2166
|
+
/** Writes shards to the local store, in order, before their upload starts. Never rejects. */
|
|
2167
|
+
async storeShards(pipeline, shards) {
|
|
2168
|
+
const store = this.ensureShardStore();
|
|
2169
|
+
for (const shard of shards) {
|
|
2170
|
+
try {
|
|
2171
|
+
await store.Put({ SessionID: pipeline.SessionID, UserID: pipeline.UserID, Index: shard.Index, Bytes: shard.Bytes, SavedAt: Date.now() });
|
|
2172
|
+
}
|
|
2173
|
+
catch (error) {
|
|
2174
|
+
// The store contract is never-throw; an override that does must still not stop the upload.
|
|
2175
|
+
console.warn(`[RealtimeSession] Failed to store recording shard ${pipeline.SessionID}/${shard.Index}; uploading it anyway:`, error);
|
|
2176
|
+
}
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
/** Removes an acknowledged shard from the local store. Never rejects. */
|
|
2180
|
+
deleteStoredShard(sessionId, index) {
|
|
2181
|
+
return DeleteStoredRecordingShard(this.ensureShardStore(), sessionId, index);
|
|
1830
2182
|
}
|
|
1831
2183
|
/**
|
|
1832
|
-
*
|
|
1833
|
-
*
|
|
1834
|
-
*
|
|
1835
|
-
*
|
|
2184
|
+
* Tells the server, right now, how much of the recording exists — for a host's `pagehide` and
|
|
2185
|
+
* `visibilitychange → hidden` handlers, where the page may be gone a moment later. Synchronous:
|
|
2186
|
+
* it seals the pending partial shard and fires the `keepalive` terminal with it, then re-sends
|
|
2187
|
+
* with `keepalive` whatever the server has not acknowledged yet (see
|
|
2188
|
+
* {@link dispatchRecordingTerminal}), and returns without waiting. Never stops the recorder: after
|
|
2189
|
+
* a `hidden` the call may well continue, and later audio simply starts the next shard index (the
|
|
2190
|
+
* tail is sealed, so no index is ever re-sent with different bytes). Also works while End is still
|
|
2191
|
+
* stopping the recorder, which is exactly when a host that does not await End navigates away.
|
|
2192
|
+
* No-op when nothing is recording, there is no session id or no transport, or nothing was
|
|
2193
|
+
* captured. Never throws.
|
|
2194
|
+
*
|
|
2195
|
+
* Deliberately opens NO IndexedDB transaction inside the handler: a `pagehide` handler gets a few
|
|
2196
|
+
* milliseconds, and a store write started there could not complete during unload anyway. The
|
|
2197
|
+
* shards it sealed are written to the store from a `setTimeout(0)` task after the handler returns:
|
|
2198
|
+
* on a page that is merely hidden (the call goes on) that keeps the durable copy until the server
|
|
2199
|
+
* acknowledges it; on a page that is unloading the task just never runs.
|
|
1836
2200
|
*/
|
|
1837
|
-
|
|
1838
|
-
const mutation = `
|
|
1839
|
-
mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
|
|
1840
|
-
UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
|
|
1841
|
-
}
|
|
1842
|
-
`;
|
|
1843
|
-
// Shards are HEADER-LESS raw little-endian PCM16 (mime audio/L16 with the capture sample rate),
|
|
1844
|
-
// NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
|
|
1845
|
-
// seekable WAV is the consolidated end-of-call upload below.
|
|
1846
|
-
const shardMime = `audio/L16;rate=${sampleRate}`;
|
|
1847
|
-
const result = (await this.gql().ExecuteGQL(mutation, {
|
|
1848
|
-
agentSessionId,
|
|
1849
|
-
segmentIndex,
|
|
1850
|
-
audioBase64,
|
|
1851
|
-
mimeType: shardMime,
|
|
1852
|
-
}));
|
|
1853
|
-
return result?.UploadRealtimeRecordingSegment === true;
|
|
1854
|
-
}
|
|
1855
|
-
/**
|
|
1856
|
-
* Stops the active recorder and encodes the captured audio. Split from the upload so teardown can
|
|
1857
|
-
* close the server session in between (#5195). Best-effort: returns `null` when nothing was
|
|
1858
|
-
* recorded or the stop failed (warned) — it must NEVER block teardown.
|
|
1859
|
-
*/
|
|
1860
|
-
async stopRecording() {
|
|
1861
|
-
this.stopSegmentFlushing();
|
|
1862
|
-
// Close the queue with the recording: its retained shards (base64 audio) are released now, not at
|
|
1863
|
-
// the next session start, and a flush still in flight stops sending, so no resend lands after the
|
|
1864
|
-
// end-of-call upload below has deleted the shards.
|
|
1865
|
-
this.shardQueue?.Close();
|
|
1866
|
-
this.shardQueue = null;
|
|
2201
|
+
SendRecordingTerminal(options) {
|
|
1867
2202
|
const recorder = this.recorder;
|
|
1868
|
-
|
|
2203
|
+
const pipeline = this.recordingShards;
|
|
2204
|
+
if (!recorder || !pipeline || !pipeline.Transport) {
|
|
2205
|
+
return;
|
|
2206
|
+
}
|
|
2207
|
+
try {
|
|
2208
|
+
const sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
|
|
2209
|
+
const tail = pipeline.Window.SealTail();
|
|
2210
|
+
if (pipeline.Window.LastSealedIndex < 0) {
|
|
2211
|
+
return;
|
|
2212
|
+
}
|
|
2213
|
+
const final = options?.Final ?? true;
|
|
2214
|
+
if (sealed.length === 0 && !tail && pipeline.FinalTerminal?.LastSealedIndex === pipeline.Window.LastSealedIndex) {
|
|
2215
|
+
return;
|
|
2216
|
+
}
|
|
2217
|
+
void this.dispatchRecordingTerminal(pipeline, sealed, tail, true, final);
|
|
2218
|
+
}
|
|
2219
|
+
catch (error) {
|
|
2220
|
+
console.warn(`[RealtimeSession] Failed to send the recording terminal for session ${pipeline.SessionID}:`, error);
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
/**
|
|
2224
|
+
* Fires the terminal request, then re-sends with `keepalive` every shard the server has not
|
|
2225
|
+
* acknowledged, then stores the shards sealed for this terminal. Shared by End and the page-lifecycle
|
|
2226
|
+
* handlers. Everything that needs the network is dispatched synchronously, before any storage work,
|
|
2227
|
+
* because the page may close at any moment from here on: an ordinary `fetch` is aborted on unload,
|
|
2228
|
+
* and a `keepalive` one is not.
|
|
2229
|
+
*
|
|
2230
|
+
* - The terminal goes first so it always gets the keepalive budget. The server waits for the shards
|
|
2231
|
+
* the terminal says exist, and assembles as soon as the last missing one arrives.
|
|
2232
|
+
* - The re-sent shards are the queue's unacknowledged ones (on the wire, waiting in a batch, or
|
|
2233
|
+
* retained after a failure) plus `sealedNow`, oldest first, within
|
|
2234
|
+
* {@link recordingKeepaliveBudgetBytes} together with the tail. A `sealedNow` shard that does not
|
|
2235
|
+
* fit, or whose keepalive upload fails, goes through the ordinary queue instead (it may still
|
|
2236
|
+
* land if the page stays). The queue keeps its own shards either way; a duplicate is acked as one.
|
|
2237
|
+
* - `sealedNow` and the tail are stored after the dispatch, from a `setTimeout(0)` task when called
|
|
2238
|
+
* from a page-lifecycle handler (`deferStore`). A shard's stored copy is deleted on its ack, after
|
|
2239
|
+
* the store write, so a fast ack can never leave an orphan row.
|
|
2240
|
+
*
|
|
2241
|
+
* Resolves true when the terminal was acknowledged. Never rejects.
|
|
2242
|
+
*/
|
|
2243
|
+
dispatchRecordingTerminal(pipeline, sealedNow, tail, deferStore, final) {
|
|
2244
|
+
const sessionId = pipeline.SessionID;
|
|
2245
|
+
const terminalAcked = this.sendRecordingTerminal(pipeline, tail, final);
|
|
2246
|
+
const toStore = tail ? [...sealedNow, tail] : sealedNow;
|
|
2247
|
+
let stored = Promise.resolve();
|
|
2248
|
+
if (toStore.length > 0) {
|
|
2249
|
+
stored = deferStore ? this.storeShardsLater(pipeline, toStore) : this.storeShards(pipeline, toStore);
|
|
2250
|
+
}
|
|
2251
|
+
this.resendUnacknowledgedWithKeepalive(pipeline, sealedNow, stored);
|
|
2252
|
+
const ackedPromise = terminalAcked.then(async (acked) => {
|
|
2253
|
+
await stored;
|
|
2254
|
+
if (!tail) {
|
|
2255
|
+
return acked;
|
|
2256
|
+
}
|
|
2257
|
+
if (acked) {
|
|
2258
|
+
await this.deleteStoredShard(sessionId, tail.Index);
|
|
2259
|
+
}
|
|
2260
|
+
else {
|
|
2261
|
+
// Retried as an ordinary shard; a Flush on a closed queue (the recording ended) is a logged no-op.
|
|
2262
|
+
await pipeline.Queue.Flush([tail]);
|
|
2263
|
+
}
|
|
2264
|
+
return acked;
|
|
2265
|
+
});
|
|
2266
|
+
if (final) {
|
|
2267
|
+
pipeline.FinalTerminal = { LastSealedIndex: pipeline.Window.LastSealedIndex, Acked: terminalAcked };
|
|
2268
|
+
}
|
|
2269
|
+
return ackedPromise;
|
|
2270
|
+
}
|
|
2271
|
+
/** Counts `bytes` against the page's keepalive budget until `request` settles. */
|
|
2272
|
+
trackKeepalive(bytes, request) {
|
|
2273
|
+
this.keepaliveBytesInFlight += bytes;
|
|
2274
|
+
return request.finally(() => {
|
|
2275
|
+
this.keepaliveBytesInFlight -= bytes;
|
|
2276
|
+
});
|
|
2277
|
+
}
|
|
2278
|
+
/**
|
|
2279
|
+
* The `keepalive` re-send part of {@link dispatchRecordingTerminal}, budgeted against everything this
|
|
2280
|
+
* page still has in flight (the terminal just sent included). A shard whose `keepalive` copy is already
|
|
2281
|
+
* on the wire is not sent again. Never throws.
|
|
2282
|
+
*/
|
|
2283
|
+
resendUnacknowledgedWithKeepalive(pipeline, sealedNow, stored) {
|
|
2284
|
+
const transport = pipeline.Transport;
|
|
2285
|
+
const sessionId = pipeline.SessionID;
|
|
2286
|
+
const fresh = new Set(sealedNow.map((shard) => shard.Index));
|
|
2287
|
+
const toQueue = (shard) => stored.then(() => pipeline.Queue.Flush([shard]));
|
|
2288
|
+
const byIndex = new Map();
|
|
2289
|
+
for (const shard of [...pipeline.Queue.Unacknowledged(), ...sealedNow]) {
|
|
2290
|
+
byIndex.set(shard.Index, shard);
|
|
2291
|
+
}
|
|
2292
|
+
for (const shard of [...byIndex.values()].sort((a, b) => a.Index - b.Index)) {
|
|
2293
|
+
const key = `${sessionId}/${shard.Index}`;
|
|
2294
|
+
if (this.keepaliveShardsInFlight.has(key)) {
|
|
2295
|
+
continue;
|
|
2296
|
+
}
|
|
2297
|
+
const budget = RealtimeSessionRuntime.recordingKeepaliveBudgetBytes - this.keepaliveBytesInFlight;
|
|
2298
|
+
if (!transport || shard.Bytes.length > budget) {
|
|
2299
|
+
if (fresh.has(shard.Index)) {
|
|
2300
|
+
void toQueue(shard);
|
|
2301
|
+
}
|
|
2302
|
+
continue;
|
|
2303
|
+
}
|
|
2304
|
+
this.keepaliveShardsInFlight.add(key);
|
|
2305
|
+
void this.trackKeepalive(shard.Bytes.length, transport.UploadShard(sessionId, shard, { Keepalive: true }).then((outcome) => outcome === 'Acked'))
|
|
2306
|
+
.finally(() => this.keepaliveShardsInFlight.delete(key))
|
|
2307
|
+
.catch((error) => {
|
|
2308
|
+
console.warn(`[RealtimeSession] Keepalive upload of recording shard ${sessionId}/${shard.Index} failed: ${describeRecordingFailure(error)}`, error);
|
|
2309
|
+
return false;
|
|
2310
|
+
})
|
|
2311
|
+
.then(async (acked) => {
|
|
2312
|
+
await stored;
|
|
2313
|
+
if (acked) {
|
|
2314
|
+
await this.deleteStoredShard(sessionId, shard.Index);
|
|
2315
|
+
}
|
|
2316
|
+
else if (fresh.has(shard.Index)) {
|
|
2317
|
+
await pipeline.Queue.Flush([shard]);
|
|
2318
|
+
}
|
|
2319
|
+
});
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
/**
|
|
2323
|
+
* Stores shards from a `setTimeout(0)` task rather than now — the path for shards sealed inside a
|
|
2324
|
+
* page-lifecycle handler, which must not open an IndexedDB transaction itself. Resolves once they
|
|
2325
|
+
* are stored (never, if the page unloads first). Never rejects.
|
|
2326
|
+
*/
|
|
2327
|
+
storeShardsLater(pipeline, shards) {
|
|
2328
|
+
return new Promise((resolve) => {
|
|
2329
|
+
setTimeout(() => {
|
|
2330
|
+
void this.storeShards(pipeline, shards).then(resolve);
|
|
2331
|
+
}, 0);
|
|
2332
|
+
});
|
|
2333
|
+
}
|
|
2334
|
+
/**
|
|
2335
|
+
* Sends the terminal request for a pipeline whose tail is already sealed. Resolves true on a server
|
|
2336
|
+
* ack and logs the outcome with the session id. Never rejects.
|
|
2337
|
+
*/
|
|
2338
|
+
async sendRecordingTerminal(pipeline, tail, final) {
|
|
2339
|
+
const lastShardIndex = pipeline.Window.LastSealedIndex;
|
|
2340
|
+
const capturedSamples = pipeline.Window.SealedSamples;
|
|
2341
|
+
const context = `session ${pipeline.SessionID} (last shard ${lastShardIndex}, ${capturedSamples} samples)`;
|
|
2342
|
+
if (!pipeline.Transport) {
|
|
2343
|
+
console.warn(`[RealtimeSession] Recording terminal NOT sent for ${context}: no recording transport; stored shards are drained on a later load.`);
|
|
2344
|
+
return false;
|
|
2345
|
+
}
|
|
2346
|
+
try {
|
|
2347
|
+
const acked = await this.trackKeepalive(tail?.Bytes.length ?? 0, pipeline.Transport.SendTerminal(pipeline.SessionID, {
|
|
2348
|
+
Tail: tail?.Bytes ?? null,
|
|
2349
|
+
LastShardIndex: lastShardIndex,
|
|
2350
|
+
CapturedSamples: capturedSamples,
|
|
2351
|
+
Final: final,
|
|
2352
|
+
}));
|
|
2353
|
+
if (!acked) {
|
|
2354
|
+
console.warn(`[RealtimeSession] Recording terminal for ${context} was not acknowledged; the server janitor assembles what arrived.`);
|
|
2355
|
+
}
|
|
2356
|
+
return acked;
|
|
2357
|
+
}
|
|
2358
|
+
catch (error) {
|
|
2359
|
+
console.warn(`[RealtimeSession] Recording terminal for ${context} failed: ${describeRecordingFailure(error)}`, error);
|
|
2360
|
+
return false;
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2363
|
+
/**
|
|
2364
|
+
* Ends the recording at teardown and STARTS its terminal request: stops the recorder (flushing the
|
|
2365
|
+
* encoder), cuts the remaining packets into shards with the tail trimmed to the recorder's captured
|
|
2366
|
+
* sample count, and dispatches the `keepalive` terminal plus `keepalive` re-sends of everything not
|
|
2367
|
+
* yet acknowledged, before storing anything ({@link dispatchRecordingTerminal}). Teardown awaits the
|
|
2368
|
+
* returned terminal only after `CloseAgentSession` ({@link awaitRecordingTerminal}), so the small,
|
|
2369
|
+
* critical close never waits behind the recording (#5195).
|
|
2370
|
+
*
|
|
2371
|
+
* The recorder and pipeline stay visible to {@link SendRecordingTerminal} until the recorder has
|
|
2372
|
+
* stopped: a `pagehide` in that window (a host that navigates away without awaiting End) still
|
|
2373
|
+
* sends what was captured, and End then continues at the next index.
|
|
2374
|
+
*
|
|
2375
|
+
* Returns null when nothing was recording or nothing was captured (no terminal is sent then).
|
|
2376
|
+
* Best-effort: never throws, never blocks teardown on the network.
|
|
2377
|
+
*/
|
|
2378
|
+
async finishRecording() {
|
|
2379
|
+
this.stopShardFlushing();
|
|
2380
|
+
const recorder = this.recorder;
|
|
2381
|
+
const pipeline = this.recordingShards;
|
|
1869
2382
|
this.currentTurnStartMs = null;
|
|
1870
2383
|
this.turnAudioStartCaptured = false;
|
|
1871
|
-
|
|
2384
|
+
this.userSpeechStartMs = null;
|
|
2385
|
+
this.userSpeechStartSuperseded = false;
|
|
2386
|
+
this.userSpeechStartByItemId.clear();
|
|
2387
|
+
if (!recorder || !pipeline) {
|
|
2388
|
+
this.recorder = null;
|
|
2389
|
+
this.recordingShards = null;
|
|
1872
2390
|
return null;
|
|
1873
2391
|
}
|
|
1874
2392
|
try {
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
2393
|
+
const capturedSamples = await this.stopRecorder(recorder, pipeline.SessionID);
|
|
2394
|
+
this.recorder = null;
|
|
2395
|
+
this.recordingShards = null;
|
|
2396
|
+
const sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
|
|
2397
|
+
// Without a count from the recorder the tail keeps every packet it holds (SealTail's default).
|
|
2398
|
+
const tail = pipeline.Window.SealTail(capturedSamples === null ? undefined : Math.max(0, capturedSamples - pipeline.Window.SealedSamples));
|
|
2399
|
+
if (pipeline.Window.LastSealedIndex < 0) {
|
|
2400
|
+
pipeline.Queue.Close();
|
|
1882
2401
|
return null;
|
|
1883
2402
|
}
|
|
1884
|
-
|
|
2403
|
+
if (sealed.length === 0 && !tail && pipeline.FinalTerminal?.LastSealedIndex === pipeline.Window.LastSealedIndex) {
|
|
2404
|
+
return { Pipeline: pipeline, Acked: pipeline.FinalTerminal.Acked };
|
|
2405
|
+
}
|
|
2406
|
+
return { Pipeline: pipeline, Acked: this.dispatchRecordingTerminal(pipeline, sealed, tail, false, true) };
|
|
1885
2407
|
}
|
|
1886
2408
|
catch (error) {
|
|
1887
|
-
console.warn(
|
|
2409
|
+
console.warn(`[RealtimeSession] Failed to finish the call recording for session ${pipeline.SessionID}; stored shards are drained at the next session start or load:`, error);
|
|
2410
|
+
this.recorder = null;
|
|
2411
|
+
this.recordingShards = null;
|
|
2412
|
+
pipeline.Queue.Close();
|
|
2413
|
+
this.rearmPendingShardDrain();
|
|
1888
2414
|
return null;
|
|
1889
2415
|
}
|
|
1890
2416
|
}
|
|
1891
2417
|
/**
|
|
1892
|
-
*
|
|
1893
|
-
*
|
|
2418
|
+
* Stops the recorder, bounded by {@link RECORDER_STOP_TIMEOUT_MS}. Resolves the captured sample
|
|
2419
|
+
* count, or null when the recorder threw or did not finish in time (logged) — the caller then ends
|
|
2420
|
+
* the recording at the packets already encoded. Never rejects.
|
|
1894
2421
|
*/
|
|
1895
|
-
async
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — no session id.');
|
|
1901
|
-
return;
|
|
1902
|
-
}
|
|
2422
|
+
async stopRecorder(recorder, sessionId) {
|
|
2423
|
+
let timer;
|
|
2424
|
+
const timedOut = new Promise((resolve) => {
|
|
2425
|
+
timer = setTimeout(() => resolve('timeout'), RECORDER_STOP_TIMEOUT_MS);
|
|
2426
|
+
});
|
|
1903
2427
|
try {
|
|
1904
|
-
await
|
|
2428
|
+
const outcome = await Promise.race([recorder.Stop(), timedOut]);
|
|
2429
|
+
if (outcome === 'timeout') {
|
|
2430
|
+
console.warn(`[RealtimeSession] Recorder did not stop within ${RECORDER_STOP_TIMEOUT_MS}ms for session ${sessionId}; ending the recording at the packets already encoded.`);
|
|
2431
|
+
return null;
|
|
2432
|
+
}
|
|
2433
|
+
return outcome;
|
|
1905
2434
|
}
|
|
1906
2435
|
catch (error) {
|
|
1907
|
-
console.warn(`[RealtimeSession]
|
|
2436
|
+
console.warn(`[RealtimeSession] Recorder failed to stop for session ${sessionId}; ending the recording at the packets already encoded:`, error);
|
|
2437
|
+
return null;
|
|
2438
|
+
}
|
|
2439
|
+
finally {
|
|
2440
|
+
clearTimeout(timer);
|
|
1908
2441
|
}
|
|
1909
2442
|
}
|
|
1910
2443
|
/**
|
|
1911
|
-
*
|
|
1912
|
-
*
|
|
1913
|
-
*
|
|
1914
|
-
* `peaks.json` sidecar for fast waveform rendering without re-decoding the audio.
|
|
2444
|
+
* Waits for the terminal {@link finishRecording} started, bounded by
|
|
2445
|
+
* {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}, logs the outcome, then closes the recording's queue
|
|
2446
|
+
* (anything it had not delivered stays in the local store for the next-load drain). Never throws.
|
|
1915
2447
|
*/
|
|
1916
|
-
async
|
|
1917
|
-
const
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
FileID
|
|
1922
|
-
ErrorMessage
|
|
1923
|
-
}
|
|
1924
|
-
}
|
|
1925
|
-
`;
|
|
1926
|
-
const result = await this.gql().ExecuteGQL(mutation, {
|
|
1927
|
-
agentSessionId,
|
|
1928
|
-
audioBase64,
|
|
1929
|
-
mimeType,
|
|
1930
|
-
consent: true,
|
|
1931
|
-
peaks
|
|
2448
|
+
async awaitRecordingTerminal(pending) {
|
|
2449
|
+
const sessionId = pending.Pipeline.SessionID;
|
|
2450
|
+
let timer;
|
|
2451
|
+
const timedOut = new Promise((resolve) => {
|
|
2452
|
+
timer = setTimeout(() => resolve('timeout'), RECORDING_TERMINAL_ACK_TIMEOUT_MS);
|
|
1932
2453
|
});
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
2454
|
+
try {
|
|
2455
|
+
const outcome = await Promise.race([pending.Acked.then((ok) => (ok ? 'ack' : 'failed')), timedOut]);
|
|
2456
|
+
if (outcome === 'ack') {
|
|
2457
|
+
console.log(`[RealtimeSession] Recording terminal acknowledged for session ${sessionId}.`);
|
|
2458
|
+
}
|
|
2459
|
+
else if (outcome === 'timeout') {
|
|
2460
|
+
console.warn(`[RealtimeSession] Recording terminal for session ${sessionId} not acknowledged within ${RECORDING_TERMINAL_ACK_TIMEOUT_MS}ms; ending anyway (the keepalive request may still land).`);
|
|
2461
|
+
}
|
|
2462
|
+
// 'failed' was already logged with its reason by sendRecordingTerminal.
|
|
2463
|
+
}
|
|
2464
|
+
finally {
|
|
2465
|
+
clearTimeout(timer);
|
|
2466
|
+
pending.Pipeline.Queue.Close();
|
|
2467
|
+
// The recording may have left shards in the store (an unacknowledged tail, windows the queue
|
|
2468
|
+
// still held or had on the wire at Close), so re-arm the drain for the next session start on
|
|
2469
|
+
// this page. Re-armed here, after the wait, so it never races the recording's own terminal;
|
|
2470
|
+
// and unconditionally, because whether a fire-and-forget upload landed is not known here and
|
|
2471
|
+
// a drain of an empty store costs one read.
|
|
2472
|
+
this.rearmPendingShardDrain();
|
|
1936
2473
|
}
|
|
1937
2474
|
}
|
|
1938
2475
|
/**
|
|
@@ -2939,6 +3476,27 @@ export class RealtimeSessionRuntime {
|
|
|
2939
3476
|
console.error('[RealtimeSession] Provider error event:', JSON.stringify(error), error);
|
|
2940
3477
|
if (error.Fatal && this.client === client) {
|
|
2941
3478
|
this.lastSessionError = error;
|
|
3479
|
+
// While the start is still running, its own failure path reports the error and ends the call
|
|
3480
|
+
// (runMintedSession throws on this recorded Fatal after Connect, even if Connect resolves):
|
|
3481
|
+
// ending it here would bump startGeneration, so the start would read as ended by the host and
|
|
3482
|
+
// never set LastStartError.
|
|
3483
|
+
if (!this.callWentLive) {
|
|
3484
|
+
return;
|
|
3485
|
+
}
|
|
3486
|
+
// An explicit teardown is already ending the call and the client stays current until its
|
|
3487
|
+
// finally: a driver reporting its own close as Fatal from inside Disconnect() is part of that
|
|
3488
|
+
// clean End, and must not flip it to 'error' or start a second teardown.
|
|
3489
|
+
if (this.teardownInFlight) {
|
|
3490
|
+
return;
|
|
3491
|
+
}
|
|
3492
|
+
// The driver is dead on the wire; nothing resumes after it says Fatal. End the call the way a
|
|
3493
|
+
// failed start ends: the row closes as Error, the microphone is released, the host is told
|
|
3494
|
+
// through SessionEnded$ (MJ#4629). A host's own End afterwards finds nothing to do.
|
|
3495
|
+
// Captured now: teardown nulls agentSessionId, and the failure it reports must name the session.
|
|
3496
|
+
const failedSessionId = this.agentSessionId;
|
|
3497
|
+
void this.endOnFailure().catch((teardownError) => {
|
|
3498
|
+
console.error(`[RealtimeSession] Teardown after a fatal provider error failed for session ${failedSessionId}:`, teardownError);
|
|
3499
|
+
});
|
|
2942
3500
|
}
|
|
2943
3501
|
});
|
|
2944
3502
|
// Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
|
|
@@ -2956,6 +3514,12 @@ export class RealtimeSessionRuntime {
|
|
|
2956
3514
|
client.OnInterruption(() => {
|
|
2957
3515
|
this.cancelPendingNarration();
|
|
2958
3516
|
});
|
|
3517
|
+
// The provider VAD saw the user start speaking. Final-only user-transcript drivers (OpenAI,
|
|
3518
|
+
// HuggingFace, xAI) emit no user interim, so this is their only real user-turn start. It goes
|
|
3519
|
+
// to its own slot (or item-keyed map entry when `itemId` is present), not the shared turn slot:
|
|
3520
|
+
// the user transcript arrives asynchronously, so the assistant's reply may already be streaming
|
|
3521
|
+
// and must keep its own start.
|
|
3522
|
+
client.OnUserSpeechStart((itemId) => this.markUserSpeechStart(itemId));
|
|
2959
3523
|
client.OnRemoteVideo((video) => this.onAgentVideo(client, video));
|
|
2960
3524
|
}
|
|
2961
3525
|
/**
|
|
@@ -3058,6 +3622,11 @@ export class RealtimeSessionRuntime {
|
|
|
3058
3622
|
}
|
|
3059
3623
|
return;
|
|
3060
3624
|
}
|
|
3625
|
+
if (transcript.Role === 'User' && transcript.Text.trim().length === 0) {
|
|
3626
|
+
// An empty final user transcript (a VAD false positive, an empty transcription) never reaches
|
|
3627
|
+
// relayTranscript, so release its onset here: it must not become the next real turn's start.
|
|
3628
|
+
this.releaseUserSpeechStart(transcript.ItemId);
|
|
3629
|
+
}
|
|
3061
3630
|
if (transcript.Role === 'Assistant') {
|
|
3062
3631
|
this.hasActiveInterimUserCaption = false;
|
|
3063
3632
|
this.pendingUserCaption = '';
|
|
@@ -3100,7 +3669,7 @@ export class RealtimeSessionRuntime {
|
|
|
3100
3669
|
if (this.firstUserTranscript === null) {
|
|
3101
3670
|
this.firstUserTranscript = transcript.Text;
|
|
3102
3671
|
}
|
|
3103
|
-
await this.relayTranscript('user', transcript.Text);
|
|
3672
|
+
await this.relayTranscript('user', transcript.Text, false, 'speech', transcript.ItemId);
|
|
3104
3673
|
}
|
|
3105
3674
|
else if (transcript.ReplacesPrevious) {
|
|
3106
3675
|
// STREAMING user transcription: providers like Grok and OpenAI Live emit the growing utterance as repeated
|
|
@@ -3111,10 +3680,10 @@ export class RealtimeSessionRuntime {
|
|
|
3111
3680
|
return;
|
|
3112
3681
|
}
|
|
3113
3682
|
this.replaceLastCaption('User', transcript.Text);
|
|
3114
|
-
await this.relayTranscript('user', transcript.Text, true);
|
|
3683
|
+
await this.relayTranscript('user', transcript.Text, true, 'speech', transcript.ItemId);
|
|
3115
3684
|
}
|
|
3116
3685
|
else {
|
|
3117
|
-
await this.onUserTranscript(transcript.Text);
|
|
3686
|
+
await this.onUserTranscript(transcript.Text, 'speech', transcript.ItemId);
|
|
3118
3687
|
}
|
|
3119
3688
|
}
|
|
3120
3689
|
/**
|
|
@@ -3132,11 +3701,10 @@ export class RealtimeSessionRuntime {
|
|
|
3132
3701
|
* - NORMAL turns only — NARRATION interims are ephemeral and never persisted, so they must not
|
|
3133
3702
|
* claim the next real turn's start slot.
|
|
3134
3703
|
*
|
|
3135
|
-
*
|
|
3136
|
-
*
|
|
3137
|
-
*
|
|
3138
|
-
*
|
|
3139
|
-
* assistant answer, which always has interims.
|
|
3704
|
+
* Stamped on a turn's first interim transcript (all drivers for the assistant — the post-tool-gap
|
|
3705
|
+
* assistant answer — and user-interim drivers like Gemini/AssemblyAI). Final-only user turns
|
|
3706
|
+
* (OpenAI-protocol drivers) take their start from {@link markUserSpeechStart}'s own slot
|
|
3707
|
+
* instead; one with no speech-start signal either (ElevenLabs) relays `null` as-is.
|
|
3140
3708
|
*/
|
|
3141
3709
|
markTurnAudioStart(kind) {
|
|
3142
3710
|
if (this.turnAudioStartCaptured || kind === 'narration') {
|
|
@@ -3149,15 +3717,75 @@ export class RealtimeSessionRuntime {
|
|
|
3149
3717
|
this.currentTurnStartMs = offset;
|
|
3150
3718
|
this.turnAudioStartCaptured = true;
|
|
3151
3719
|
}
|
|
3720
|
+
/**
|
|
3721
|
+
* Stamps the user speech onset when the client signals that the user began to speak. When the
|
|
3722
|
+
* driver supplies a conversation `itemId`, records the FIRST onset for that item in
|
|
3723
|
+
* {@link userSpeechStartByItemId} so a mid-sentence pause that re-fires with the same id keeps the
|
|
3724
|
+
* original start. When no `itemId` is supplied, stamps the single-slot {@link userSpeechStartMs}
|
|
3725
|
+
* fallback (keeping the first onset unless {@link userSpeechStartSuperseded} is set).
|
|
3726
|
+
*/
|
|
3727
|
+
markUserSpeechStart(itemId) {
|
|
3728
|
+
if (itemId !== undefined && itemId.length > 0) {
|
|
3729
|
+
if (this.userSpeechStartByItemId.has(itemId)) {
|
|
3730
|
+
return;
|
|
3731
|
+
}
|
|
3732
|
+
const offset = this.nowTurnOffsetMs();
|
|
3733
|
+
if (offset === null) {
|
|
3734
|
+
return;
|
|
3735
|
+
}
|
|
3736
|
+
this.userSpeechStartByItemId.set(itemId, offset);
|
|
3737
|
+
return;
|
|
3738
|
+
}
|
|
3739
|
+
if (this.userSpeechStartMs !== null && !this.userSpeechStartSuperseded) {
|
|
3740
|
+
return;
|
|
3741
|
+
}
|
|
3742
|
+
const offset = this.nowTurnOffsetMs();
|
|
3743
|
+
if (offset === null) {
|
|
3744
|
+
return;
|
|
3745
|
+
}
|
|
3746
|
+
this.userSpeechStartMs = offset;
|
|
3747
|
+
this.userSpeechStartSuperseded = false;
|
|
3748
|
+
}
|
|
3749
|
+
/**
|
|
3750
|
+
* Ends the held user onset: a final user transcript (empty or not) closed the utterance it began.
|
|
3751
|
+
* When `itemId` is supplied, deletes only that item's entry in {@link userSpeechStartByItemId};
|
|
3752
|
+
* otherwise clears the unkeyed single slot.
|
|
3753
|
+
*/
|
|
3754
|
+
releaseUserSpeechStart(itemId) {
|
|
3755
|
+
if (itemId !== undefined && itemId.length > 0) {
|
|
3756
|
+
this.userSpeechStartByItemId.delete(itemId);
|
|
3757
|
+
return;
|
|
3758
|
+
}
|
|
3759
|
+
this.userSpeechStartMs = null;
|
|
3760
|
+
this.userSpeechStartSuperseded = false;
|
|
3761
|
+
}
|
|
3762
|
+
/**
|
|
3763
|
+
* Consumes and deletes the onset recorded for `itemId` in {@link userSpeechStartByItemId}. When a
|
|
3764
|
+
* matching onset is found, also deletes any remaining map entry with an earlier offset so a VAD
|
|
3765
|
+
* false positive whose transcription never arrived cannot accumulate in the map.
|
|
3766
|
+
*/
|
|
3767
|
+
takeKeyedUserSpeechStart(itemId) {
|
|
3768
|
+
const onset = this.userSpeechStartByItemId.get(itemId) ?? null;
|
|
3769
|
+
this.userSpeechStartByItemId.delete(itemId);
|
|
3770
|
+
if (onset !== null) {
|
|
3771
|
+
for (const [otherId, otherOnset] of this.userSpeechStartByItemId) {
|
|
3772
|
+
if (otherOnset < onset) {
|
|
3773
|
+
this.userSpeechStartByItemId.delete(otherId);
|
|
3774
|
+
}
|
|
3775
|
+
}
|
|
3776
|
+
}
|
|
3777
|
+
return onset;
|
|
3778
|
+
}
|
|
3152
3779
|
/**
|
|
3153
3780
|
* The current per-turn offset in ms — the RECORDER's clock when one runs (an offset into a
|
|
3154
3781
|
* seekable file), else the SESSION clock (#3832: orderable and displayable, not seekable),
|
|
3155
3782
|
* else `null` before any call is live. One function so the two stamp sites cannot disagree
|
|
3156
|
-
* about which clock a session is on.
|
|
3783
|
+
* about which clock a session is on. Always whole milliseconds: the offsets travel as GraphQL
|
|
3784
|
+
* `Int`, which refuses a fraction, and a 48 kHz sample clock rarely lands on a whole millisecond.
|
|
3157
3785
|
*/
|
|
3158
3786
|
nowTurnOffsetMs() {
|
|
3159
3787
|
if (this.recorder) {
|
|
3160
|
-
return this.recorder.NowOffsetMs();
|
|
3788
|
+
return Math.round(this.recorder.NowOffsetMs());
|
|
3161
3789
|
}
|
|
3162
3790
|
if (this.sessionClockStartMs !== null) {
|
|
3163
3791
|
return Math.max(0, Math.round(performance.now() - this.sessionClockStartMs));
|
|
@@ -3181,8 +3809,14 @@ export class RealtimeSessionRuntime {
|
|
|
3181
3809
|
}
|
|
3182
3810
|
this.appendCaption({ Role: role, Text: text });
|
|
3183
3811
|
}
|
|
3184
|
-
/**
|
|
3185
|
-
|
|
3812
|
+
/**
|
|
3813
|
+
* Finalizes the user turn: push a caption + relay the final transcript.
|
|
3814
|
+
*
|
|
3815
|
+
* @param source `'typed'` for a {@link SendText} turn: it has no speech onset, so it neither takes
|
|
3816
|
+
* nor clears the held one (a spoken transcript may still be pending for it).
|
|
3817
|
+
* @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
|
|
3818
|
+
*/
|
|
3819
|
+
async onUserTranscript(transcript, source = 'speech', itemId) {
|
|
3186
3820
|
if (transcript.trim().length === 0) {
|
|
3187
3821
|
return;
|
|
3188
3822
|
}
|
|
@@ -3191,7 +3825,7 @@ export class RealtimeSessionRuntime {
|
|
|
3191
3825
|
this.firstUserTranscript = transcript;
|
|
3192
3826
|
}
|
|
3193
3827
|
this.appendCaption({ Role: 'User', Text: transcript });
|
|
3194
|
-
await this.relayTranscript('user', transcript);
|
|
3828
|
+
await this.relayTranscript('user', transcript, false, source, itemId);
|
|
3195
3829
|
}
|
|
3196
3830
|
// ── Tool calling ───────────────────────────────────────────────────────────
|
|
3197
3831
|
/**
|
|
@@ -3519,37 +4153,79 @@ export class RealtimeSessionRuntime {
|
|
|
3519
4153
|
* Relays a final transcript turn to MJ via `RelayRealtimeTranscript`.
|
|
3520
4154
|
*
|
|
3521
4155
|
* When the session is being recorded, per-turn timing rides along: `utteranceEndMs` is the
|
|
3522
|
-
* recording-relative offset at finalization, and `utteranceStartMs` is
|
|
3523
|
-
*
|
|
3524
|
-
*
|
|
3525
|
-
*
|
|
3526
|
-
*
|
|
3527
|
-
*
|
|
4156
|
+
* recording-relative offset at finalization, and `utteranceStartMs` is where THIS turn's audio
|
|
4157
|
+
* actually began — NOT inherited from the previous turn's end. For a user turn whose onset the
|
|
4158
|
+
* client signalled, that is {@link userSpeechStartMs}; otherwise it is the offset captured by
|
|
4159
|
+
* {@link markTurnAudioStart} on the turn's first interim. That distinction is the timing fix:
|
|
4160
|
+
* when a tool-call / silence gap sits between turns, the post-gap turn's audio starts much
|
|
4161
|
+
* later, so inheriting the prior turn's end stamped the cue ~gap-length too early. Both are
|
|
4162
|
+
* omitted (left `null`) when the session isn't being recorded. `utteranceStartMs` is also
|
|
4163
|
+
* `null` when no start was measured for this turn; it is never replaced with a placeholder 0.
|
|
3528
4164
|
*
|
|
3529
4165
|
* A correction (`replacesPrevious`) doesn't open a new turn, so it carries no start and doesn't
|
|
3530
4166
|
* reset the per-turn start guard. After a normal finalization the guard is cleared so the NEXT
|
|
3531
|
-
* turn re-stamps its start from where ITS audio begins
|
|
4167
|
+
* turn re-stamps its start from where ITS audio begins — except for a user turn that used its
|
|
4168
|
+
* own speech-start slot, which leaves the shared slot alone (it may hold the in-flight
|
|
4169
|
+
* assistant turn's start). Every final user transcript, corrections included, clears the
|
|
4170
|
+
* speech-start slot.
|
|
3532
4171
|
*
|
|
3533
4172
|
* @param replacesPrevious CORRECTION semantics: the server updates the session's most
|
|
3534
4173
|
* recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
|
|
3535
4174
|
* post-barge-in `agent_response_correction`).
|
|
4175
|
+
* @param source `'typed'` for a user turn sent with {@link SendText}: it does not read or clear
|
|
4176
|
+
* {@link userSpeechStartMs} or {@link userSpeechStartByItemId}, which belong to the user's speech.
|
|
4177
|
+
* @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
|
|
3536
4178
|
*/
|
|
3537
|
-
async relayTranscript(role, text, replacesPrevious = false) {
|
|
4179
|
+
async relayTranscript(role, text, replacesPrevious = false, source = 'speech', itemId) {
|
|
3538
4180
|
if (!this.agentSessionId) {
|
|
3539
4181
|
return;
|
|
3540
4182
|
}
|
|
4183
|
+
// Every final user transcript closes the signalled onset — a correction too, since it ends the
|
|
4184
|
+
// same utterance — so a later onset (an xAI mid-sentence resume) can't leak into the next turn.
|
|
4185
|
+
// When the transcript carries a provider itemId, consume its keyed onset from the map without
|
|
4186
|
+
// touching the unkeyed single slot or falling back to currentTurnStartMs.
|
|
4187
|
+
const hasItemId = role === 'user' && source === 'speech' && itemId !== undefined && itemId.length > 0;
|
|
4188
|
+
let userSpeechStartMs = null;
|
|
4189
|
+
if (hasItemId) {
|
|
4190
|
+
userSpeechStartMs = this.takeKeyedUserSpeechStart(itemId);
|
|
4191
|
+
}
|
|
4192
|
+
else if (role === 'user' && source === 'speech') {
|
|
4193
|
+
userSpeechStartMs = this.userSpeechStartMs;
|
|
4194
|
+
this.releaseUserSpeechStart();
|
|
4195
|
+
}
|
|
3541
4196
|
// Per-turn timing against whichever clock the session is on (#3832): the recorder's when one
|
|
3542
|
-
// runs, else the session clock. `utteranceStartMs` is where this turn's audio actually began
|
|
3543
|
-
//
|
|
3544
|
-
//
|
|
4197
|
+
// runs, else the session clock. `utteranceStartMs` is where this turn's audio actually began.
|
|
4198
|
+
// It is null when no start was measured for this turn (a final-only driver with no speech-start
|
|
4199
|
+
// signal), so the server stores NULL rather than a placeholder 0.
|
|
3545
4200
|
const utteranceEndMs = this.nowTurnOffsetMs();
|
|
3546
|
-
const
|
|
3547
|
-
|
|
4201
|
+
const opensTurn = utteranceEndMs !== null && !replacesPrevious;
|
|
4202
|
+
let utteranceStartMs = null;
|
|
4203
|
+
if (opensTurn && (hasItemId || userSpeechStartMs !== null)) {
|
|
4204
|
+
// A user turn with a signalled onset (or a keyed user turn whose itemId was not in the map,
|
|
4205
|
+
// which relays null). Leave the shared slot alone: the user transcript arrives asynchronously,
|
|
4206
|
+
// so it may already hold the in-flight assistant reply's start.
|
|
4207
|
+
utteranceStartMs = userSpeechStartMs;
|
|
4208
|
+
if (!this.turnAudioStartCaptured) {
|
|
4209
|
+
// No turn has captured a start, so the slot holds only the recording-start seed (0). This
|
|
4210
|
+
// user turn was the first turn; drop the seed so a later unmeasured turn relays null, not 0.
|
|
4211
|
+
this.currentTurnStartMs = null;
|
|
4212
|
+
}
|
|
4213
|
+
}
|
|
4214
|
+
else if (opensTurn) {
|
|
4215
|
+
utteranceStartMs = this.currentTurnStartMs;
|
|
3548
4216
|
// This turn is finalized — arm the NEXT turn to re-stamp its start from its own first
|
|
3549
4217
|
// interim (handles a tool-call gap before the next turn). Stop inheriting this end as the
|
|
3550
|
-
// next start. `null` means "not yet captured"
|
|
4218
|
+
// next start. `null` means "not yet captured" and is relayed as-is if nothing stamps it.
|
|
3551
4219
|
this.currentTurnStartMs = null;
|
|
3552
4220
|
this.turnAudioStartCaptured = false;
|
|
4221
|
+
if (role === 'assistant' && this.userSpeechStartMs !== null && (utteranceStartMs === null || this.userSpeechStartMs < utteranceStartMs)) {
|
|
4222
|
+
// The floor moved on past an onset that began BEFORE this assistant turn: a late user
|
|
4223
|
+
// transcript may still claim it, but the next onset replaces it (see userSpeechStartSuperseded).
|
|
4224
|
+
// An onset that began DURING this turn is a barge-in; it belongs to the user speech still
|
|
4225
|
+
// pending, so it stays the first onset. With no measured assistant start we cannot tell, and
|
|
4226
|
+
// treat the onset as earlier.
|
|
4227
|
+
this.userSpeechStartSuperseded = true;
|
|
4228
|
+
}
|
|
3553
4229
|
}
|
|
3554
4230
|
try {
|
|
3555
4231
|
const mutation = `
|
|
@@ -3943,10 +4619,18 @@ export class RealtimeSessionRuntime {
|
|
|
3943
4619
|
}
|
|
3944
4620
|
// ── Teardown ───────────────────────────────────────────────────────────────
|
|
3945
4621
|
/**
|
|
3946
|
-
* Tears down all client resources and
|
|
3947
|
-
*
|
|
4622
|
+
* Tears down all client resources and closes the server session when one was minted.
|
|
4623
|
+
*
|
|
4624
|
+
* The server row is closed whichever way the call ended: a start that failed after the mint used to
|
|
4625
|
+
* leave its row `Active` until the janitor's staleness sweep, 15 minutes later, so every host that
|
|
4626
|
+
* counts live sessions saw a phantom call (MJ#6013). The janitor remains the fallback for a client
|
|
4627
|
+
* that dies before it can say anything; it is never the plan.
|
|
4628
|
+
*
|
|
4629
|
+
* @param end why the call ended — `'explicit'` when the host or user ended it, `'error'` when the
|
|
4630
|
+
* runtime ended it because the start or the call failed. Reported on `SessionEnded$` and stamped on
|
|
4631
|
+
* the server row as its `CloseReason` (`Explicit` / `Error`).
|
|
3948
4632
|
*/
|
|
3949
|
-
async teardown(
|
|
4633
|
+
async teardown(end) {
|
|
3950
4634
|
// Invalidate any start still in flight BEFORE anything else, so it abandons itself at its next
|
|
3951
4635
|
// await rather than opening a microphone behind a session that is being ended.
|
|
3952
4636
|
this.startGeneration++;
|
|
@@ -3956,7 +4640,7 @@ export class RealtimeSessionRuntime {
|
|
|
3956
4640
|
await this.teardownInFlight;
|
|
3957
4641
|
return;
|
|
3958
4642
|
}
|
|
3959
|
-
this.teardownInFlight = this.runTeardown(
|
|
4643
|
+
this.teardownInFlight = this.runTeardown(end).finally(() => {
|
|
3960
4644
|
if (this._savingRecording$.value) {
|
|
3961
4645
|
this._savingRecording$.next(false);
|
|
3962
4646
|
}
|
|
@@ -3965,11 +4649,22 @@ export class RealtimeSessionRuntime {
|
|
|
3965
4649
|
await this.teardownInFlight;
|
|
3966
4650
|
}
|
|
3967
4651
|
/** The body of {@link teardown}; never called concurrently with itself. */
|
|
3968
|
-
async runTeardown(
|
|
4652
|
+
async runTeardown(end) {
|
|
3969
4653
|
// Arm BEFORE any await so a tab closed in the first milliseconds of teardown is still guarded.
|
|
3970
4654
|
if (this.recorder) {
|
|
3971
4655
|
this._savingRecording$.next(true);
|
|
3972
4656
|
}
|
|
4657
|
+
// Whatever a step below does, the slot is released: a teardown that threw halfway used to leave
|
|
4658
|
+
// `IsActive` set for good, so every later start was refused (MJ#4629).
|
|
4659
|
+
try {
|
|
4660
|
+
await this.releaseCallResources(end);
|
|
4661
|
+
}
|
|
4662
|
+
finally {
|
|
4663
|
+
this.releaseSessionSlot(end);
|
|
4664
|
+
}
|
|
4665
|
+
}
|
|
4666
|
+
/** The teardown's work: stop the call, hand back what it held, and close the server session. */
|
|
4667
|
+
async releaseCallResources(end) {
|
|
3973
4668
|
// First: stop asserting liveness. A pulse racing the close would re-stamp LastActiveAt on a
|
|
3974
4669
|
// session we are deliberately ending, leaving an Idle row the janitor then has to age out.
|
|
3975
4670
|
this.stopLivenessPulse();
|
|
@@ -3980,6 +4675,14 @@ export class RealtimeSessionRuntime {
|
|
|
3980
4675
|
// set (the captured per-save id covers the race anyway), then dispose the plugins.
|
|
3981
4676
|
this.flushAllChannelSaves();
|
|
3982
4677
|
this.disposeChannels();
|
|
4678
|
+
// End the recording and START its keepalive terminal request now: stop the recorder (which flushes
|
|
4679
|
+
// the encoder) before anything else that awaits. A tab closed right after End gets only the pagehide
|
|
4680
|
+
// terminal, which carries packets already encoded; releasing the mic and disconnecting the driver can
|
|
4681
|
+
// take long enough for the page to die first, and the audio still inside the encoder would be lost.
|
|
4682
|
+
// It also goes out before the usage flush and the close, so it is on the wire if the page dies during
|
|
4683
|
+
// either. Best-effort — finishRecording never throws. Null when nothing was recorded.
|
|
4684
|
+
const recordingTerminal = await this.finishRecording();
|
|
4685
|
+
this.recordingStartedAtIso = null;
|
|
3983
4686
|
// The camera and screen share go first: the camera runs on the controller closed next.
|
|
3984
4687
|
this.closeCaptures();
|
|
3985
4688
|
// Defensive: stop the mic even when Connect never ran (the client also stops the
|
|
@@ -4013,11 +4716,6 @@ export class RealtimeSessionRuntime {
|
|
|
4013
4716
|
this.client = null;
|
|
4014
4717
|
}
|
|
4015
4718
|
}
|
|
4016
|
-
// Stop + encode the call recording now (the recorder must be detached before the session id is
|
|
4017
|
-
// cleared); the upload itself happens after the close below. Best-effort — stopRecording
|
|
4018
|
-
// swallows its own errors. Null when nothing was recorded.
|
|
4019
|
-
const recording = await this.stopRecording();
|
|
4020
|
-
this.recordingStartedAtIso = null;
|
|
4021
4719
|
// Final usage flush WHILE the live session id is still set (the relay mutation also
|
|
4022
4720
|
// accepts a Closed session, so ordering vs. CloseAgentSession is belt-and-braces).
|
|
4023
4721
|
if (this.usageFlushTimer) {
|
|
@@ -4026,19 +4724,24 @@ export class RealtimeSessionRuntime {
|
|
|
4026
4724
|
}
|
|
4027
4725
|
await this.flushPendingUsage(this.agentSessionId);
|
|
4028
4726
|
this.resetUsageRelay();
|
|
4029
|
-
if (
|
|
4030
|
-
await this.closeServerSession(this.agentSessionId);
|
|
4031
|
-
}
|
|
4032
|
-
//
|
|
4033
|
-
//
|
|
4034
|
-
//
|
|
4035
|
-
|
|
4036
|
-
|
|
4727
|
+
if (this.agentSessionId) {
|
|
4728
|
+
await this.closeServerSession(this.agentSessionId, end === 'error' ? 'Error' : 'Explicit');
|
|
4729
|
+
}
|
|
4730
|
+
// Wait for the terminal's ack only AFTER the close: nothing after an aborted await ever runs, so
|
|
4731
|
+
// the small, critical close must never queue behind the recording (#5195). Bounded by
|
|
4732
|
+
// RECORDING_TERMINAL_ACK_TIMEOUT_MS; SavingRecording$ stays up until it settles.
|
|
4733
|
+
if (recordingTerminal) {
|
|
4734
|
+
await this.awaitRecordingTerminal(recordingTerminal);
|
|
4735
|
+
}
|
|
4736
|
+
}
|
|
4737
|
+
/** Clears the slot and tells the host the call ended; runs even when {@link releaseCallResources} threw. */
|
|
4738
|
+
releaseSessionSlot(end) {
|
|
4037
4739
|
// Capture the session id BEFORE we null it so the lifecycle emit carries it.
|
|
4038
4740
|
// Skip emitting when there was no live session (defensive — teardown is safe
|
|
4039
4741
|
// to call without an active session).
|
|
4040
4742
|
const closedSessionId = this.agentSessionId;
|
|
4041
4743
|
this.agentSessionId = null;
|
|
4744
|
+
this.callWentLive = false;
|
|
4042
4745
|
this.sessionConversationId = null;
|
|
4043
4746
|
this.narrationTemplate = null;
|
|
4044
4747
|
this.clientToolHandlers.clear();
|
|
@@ -4051,14 +4754,9 @@ export class RealtimeSessionRuntime {
|
|
|
4051
4754
|
if (this._connectionState$.value !== 'error') {
|
|
4052
4755
|
this.setConnectionState('closed');
|
|
4053
4756
|
}
|
|
4054
|
-
// Surface generic session-ended for the conversations runtime bridge.
|
|
4055
|
-
// `closeServerSession=true` means the user explicitly called EndRealtimeSession;
|
|
4056
|
-
// `false` means teardown ran from a catch block (start path error path).
|
|
4757
|
+
// Surface generic session-ended for the conversations runtime bridge, with why it ended.
|
|
4057
4758
|
if (closedSessionId) {
|
|
4058
|
-
this._sessionEnded$.next({
|
|
4059
|
-
sessionId: closedSessionId,
|
|
4060
|
-
reason: closeServerSession ? 'explicit' : 'error',
|
|
4061
|
-
});
|
|
4759
|
+
this._sessionEnded$.next({ sessionId: closedSessionId, reason: end });
|
|
4062
4760
|
}
|
|
4063
4761
|
}
|
|
4064
4762
|
/**
|
|
@@ -4073,18 +4771,22 @@ export class RealtimeSessionRuntime {
|
|
|
4073
4771
|
console.error('[RealtimeSession] Media host failed to release the microphone:', error);
|
|
4074
4772
|
}
|
|
4075
4773
|
}
|
|
4076
|
-
/**
|
|
4077
|
-
|
|
4774
|
+
/**
|
|
4775
|
+
* Calls the `CloseAgentSession` mutation (provisioned in P4b) with why the client ended the call, which
|
|
4776
|
+
* the server stamps as the row's `CloseReason`. Requires a server that accepts the `closeReason`
|
|
4777
|
+
* argument (MJ#6013); the runtime and the server ship together.
|
|
4778
|
+
*/
|
|
4779
|
+
async closeServerSession(agentSessionId, closeReason) {
|
|
4078
4780
|
try {
|
|
4079
4781
|
const mutation = `
|
|
4080
|
-
mutation CloseAgentSession($agentSessionId: String
|
|
4081
|
-
CloseAgentSession(agentSessionId: $agentSessionId)
|
|
4782
|
+
mutation CloseAgentSession($agentSessionId: String!, $closeReason: String) {
|
|
4783
|
+
CloseAgentSession(agentSessionId: $agentSessionId, closeReason: $closeReason)
|
|
4082
4784
|
}
|
|
4083
4785
|
`;
|
|
4084
|
-
await this.gql().ExecuteGQL(mutation, { agentSessionId });
|
|
4786
|
+
await this.gql().ExecuteGQL(mutation, { agentSessionId, closeReason });
|
|
4085
4787
|
}
|
|
4086
4788
|
catch (error) {
|
|
4087
|
-
console.error(
|
|
4789
|
+
console.error(`[RealtimeSession] Failed to close server session ${agentSessionId} (closeReason ${closeReason}):`, error);
|
|
4088
4790
|
}
|
|
4089
4791
|
}
|
|
4090
4792
|
// ── Helpers ────────────────────────────────────────────────────────────────
|
|
@@ -4098,12 +4800,17 @@ export class RealtimeSessionRuntime {
|
|
|
4098
4800
|
this.pendingUserCaption = '';
|
|
4099
4801
|
this.hasActiveInterimUserCaption = false;
|
|
4100
4802
|
this.SetMinimized(false);
|
|
4101
|
-
this.
|
|
4102
|
-
|
|
4803
|
+
this.stopShardFlushing();
|
|
4804
|
+
// Defensive: teardown already ended any previous recording; a stray pipeline must not keep sending.
|
|
4805
|
+
this.recordingShards?.Queue.Close();
|
|
4806
|
+
this.recordingShards = null;
|
|
4103
4807
|
this.recorder = null;
|
|
4104
4808
|
this.recordingStartedAtIso = null;
|
|
4105
4809
|
this.currentTurnStartMs = null;
|
|
4106
4810
|
this.turnAudioStartCaptured = false;
|
|
4811
|
+
this.userSpeechStartMs = null;
|
|
4812
|
+
this.userSpeechStartSuperseded = false;
|
|
4813
|
+
this.userSpeechStartByItemId.clear();
|
|
4107
4814
|
this.usedChannelNames.clear();
|
|
4108
4815
|
this.microphoneMuted = false;
|
|
4109
4816
|
}
|