@memberjunction/realtime-runtime 6.2.0-edge.4 → 6.2.0-edge.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/hosts/IRealtimeMediaHost.d.ts +25 -30
- package/dist/hosts/IRealtimeMediaHost.d.ts.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -2
- package/dist/index.js.map +1 -1
- package/dist/session/RealtimeSessionRuntime.d.ts +372 -73
- package/dist/session/RealtimeSessionRuntime.d.ts.map +1 -1
- package/dist/session/RealtimeSessionRuntime.js +919 -212
- package/dist/session/RealtimeSessionRuntime.js.map +1 -1
- package/dist/session/RecordingShardDrain.d.ts +59 -0
- package/dist/session/RecordingShardDrain.d.ts.map +1 -0
- package/dist/session/RecordingShardDrain.js +135 -0
- package/dist/session/RecordingShardDrain.js.map +1 -0
- package/dist/session/RecordingShardQueue.d.ts +64 -38
- package/dist/session/RecordingShardQueue.d.ts.map +1 -1
- package/dist/session/RecordingShardQueue.js +106 -84
- package/dist/session/RecordingShardQueue.js.map +1 -1
- package/dist/session/RecordingShardStore.d.ts +57 -0
- package/dist/session/RecordingShardStore.d.ts.map +1 -0
- package/dist/session/RecordingShardStore.js +184 -0
- package/dist/session/RecordingShardStore.js.map +1 -0
- package/dist/session/RecordingShardWindow.d.ts +42 -0
- package/dist/session/RecordingShardWindow.d.ts.map +1 -0
- package/dist/session/RecordingShardWindow.js +88 -0
- package/dist/session/RecordingShardWindow.js.map +1 -0
- package/dist/session/RecordingTransport.d.ts +67 -0
- package/dist/session/RecordingTransport.d.ts.map +1 -0
- package/dist/session/RecordingTransport.js +61 -0
- package/dist/session/RecordingTransport.js.map +1 -0
- package/package.json +10 -9
|
@@ -15,6 +15,31 @@ import type { RealtimeContextActionResult } from '../channels/channel-contract-t
|
|
|
15
15
|
import { type AppClientToolRegistration } from './app-client-tool-registry.js';
|
|
16
16
|
import { type IChannelExposurePreferences } from './channel-exposure-preferences.js';
|
|
17
17
|
import { type RealtimeSessionStartOptions } from './channel-session-scope.js';
|
|
18
|
+
import { type IRecordingShardStore } from './RecordingShardStore.js';
|
|
19
|
+
import type { IRecordingTransport } from './RecordingTransport.js';
|
|
20
|
+
/**
|
|
21
|
+
* How long teardown waits for the server to acknowledge the recording's terminal request, measured
|
|
22
|
+
* from after `CloseAgentSession`. The terminal is `keepalive`, so it completes even if the page goes
|
|
23
|
+
* away; the wait only keeps `SavingRecording$` (and so a host's "don't close yet" guard) up while the
|
|
24
|
+
* server has not confirmed. Bounded so a dead server can never hold the End of a call hostage —
|
|
25
|
+
* unacknowledged shards stay in the local store and the server janitor assembles what arrived.
|
|
26
|
+
*/
|
|
27
|
+
export declare const RECORDING_TERMINAL_ACK_TIMEOUT_MS = 3000;
|
|
28
|
+
/**
|
|
29
|
+
* How long teardown waits for the recorder to stop and flush its encoder. `Stop()` runs BEFORE
|
|
30
|
+
* `CloseAgentSession`, so a wedged encoder must not be able to hold the End of a call (and the
|
|
31
|
+
* server session) open; past this the recording ends at the packets already encoded.
|
|
32
|
+
*/
|
|
33
|
+
export declare const RECORDER_STOP_TIMEOUT_MS = 2000;
|
|
34
|
+
/** Options of {@link RealtimeSessionRuntime.SendRecordingTerminal}. */
|
|
35
|
+
export interface RecordingTerminalOptions {
|
|
36
|
+
/**
|
|
37
|
+
* False when the call may continue — the host's `visibilitychange → hidden` handler. The server then
|
|
38
|
+
* stamps the counts but assembles only after its grace period, so switching tabs does not rewrite the
|
|
39
|
+
* recording each time. Defaults to true (`pagehide`: the page is going away).
|
|
40
|
+
*/
|
|
41
|
+
Final?: boolean;
|
|
42
|
+
}
|
|
18
43
|
/**
|
|
19
44
|
* `MJ: User Settings` key for the per-user "record this voice call" consent toggle. Stored as
|
|
20
45
|
* the literal string `'true'`/`'false'` (read with `=== 'true'`), cross-device via
|
|
@@ -224,6 +249,15 @@ export interface RealtimeSessionRunOptions {
|
|
|
224
249
|
*/
|
|
225
250
|
readonly appContext?: AppContextSnapshot | null;
|
|
226
251
|
}
|
|
252
|
+
/** Why a call ended, as the runtime reports it on `SessionEnded$`. */
|
|
253
|
+
export type SessionEndReason = 'explicit' | 'error';
|
|
254
|
+
/** The `CloseReason` values a client may stamp on its own session row; the others belong to the server. */
|
|
255
|
+
export type ClientCloseReason = 'Explicit' | 'Error';
|
|
256
|
+
/** Thrown by a start while a call is live: the runtime holds one driver, and never replaces a live one. */
|
|
257
|
+
export declare class RealtimeSessionAlreadyActiveError extends Error {
|
|
258
|
+
readonly ActiveAgentSessionId: string | null;
|
|
259
|
+
constructor(ActiveAgentSessionId: string | null);
|
|
260
|
+
}
|
|
227
261
|
/**
|
|
228
262
|
* Drives a **client-direct** real-time voice session: the browser mints an ephemeral
|
|
229
263
|
* token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
|
|
@@ -250,9 +284,9 @@ export declare class RealtimeSessionRuntime {
|
|
|
250
284
|
protected readonly mediaHost: IRealtimeMediaHost;
|
|
251
285
|
/**
|
|
252
286
|
* @param mediaHost the platform's media capabilities. The runtime never touches `navigator`,
|
|
253
|
-
* `Blob` or `FileReader` itself — microphone acquisition and audio
|
|
254
|
-
* are the host's, because both are platform-specific
|
|
255
|
-
*
|
|
287
|
+
* `Blob` or `FileReader` itself — microphone acquisition and audio capture/encoding
|
|
288
|
+
* are the host's, because both are platform-specific (permission UX, which
|
|
289
|
+
* encoder the platform offers).
|
|
256
290
|
*/
|
|
257
291
|
constructor(mediaHost: IRealtimeMediaHost);
|
|
258
292
|
private _connectionState$;
|
|
@@ -322,10 +356,11 @@ export declare class RealtimeSessionRuntime {
|
|
|
322
356
|
readonly Minimized$: Observable<boolean>;
|
|
323
357
|
/**
|
|
324
358
|
* True while the end-of-call recording is being saved: from the first statement of a teardown
|
|
325
|
-
* that holds a recorder until that teardown settles
|
|
326
|
-
*
|
|
327
|
-
*
|
|
328
|
-
*
|
|
359
|
+
* that holds a recorder until that teardown settles — which includes waiting (at most
|
|
360
|
+
* {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}) for the server to acknowledge the recording's terminal
|
|
361
|
+
* request. False at construction and for unrecorded sessions. Exists for #5195; the runtime is
|
|
362
|
+
* platform-free, so hosts own the reaction (a "saving" indicator, a disabled navigation button,
|
|
363
|
+
* etc.).
|
|
329
364
|
*/
|
|
330
365
|
readonly SavingRecording$: Observable<boolean>;
|
|
331
366
|
/** Synchronous read of {@link SavingRecording$}. */
|
|
@@ -416,13 +451,13 @@ export declare class RealtimeSessionRuntime {
|
|
|
416
451
|
* Fired EXACTLY ONCE per session as teardown begins, with the prior
|
|
417
452
|
* `agentSessionId` (so subscribers can correlate against `SessionStarted$`'s
|
|
418
453
|
* sessionId) and the client-distinguishable reason — `'explicit'` when the
|
|
419
|
-
* user called `EndRealtimeSession`, `'error'` when
|
|
420
|
-
*
|
|
454
|
+
* user called `EndRealtimeSession`, `'error'` when the runtime ended the call over a failure: a
|
|
455
|
+
* failed start, or a fatal error from the provider mid-call. Server-side close paths (janitor, shutdown) do NOT propagate here —
|
|
421
456
|
* they happen out-of-process and have no client push channel today.
|
|
422
457
|
*/
|
|
423
458
|
readonly SessionEnded$: Observable<{
|
|
424
459
|
sessionId: string;
|
|
425
|
-
reason:
|
|
460
|
+
reason: SessionEndReason;
|
|
426
461
|
}>;
|
|
427
462
|
/**
|
|
428
463
|
* Fires with the channel PLUGIN every time the agent ACTS on that channel (a tool call
|
|
@@ -536,25 +571,54 @@ export declare class RealtimeSessionRuntime {
|
|
|
536
571
|
private narrationTemplate;
|
|
537
572
|
/**
|
|
538
573
|
* The active session's audio recorder (mic + agent mix), or `null` when the user didn't
|
|
539
|
-
* consent or the browser can't record. Created after the client connects;
|
|
540
|
-
* at teardown.
|
|
574
|
+
* consent or the browser can't record. Created after the client connects; its packets are
|
|
575
|
+
* shipped as 5 s shards during the call, and it is stopped at teardown.
|
|
541
576
|
*/
|
|
542
577
|
private recorder;
|
|
543
578
|
/** ISO timestamp of when recording started — sent to the server on session start. */
|
|
544
579
|
private recordingStartedAtIso;
|
|
545
|
-
/** Interval that
|
|
546
|
-
private
|
|
580
|
+
/** Interval that cuts and uploads a shard every {@link recordingShardFlushMs} during a recording. */
|
|
581
|
+
private shardFlushTimer;
|
|
547
582
|
/**
|
|
548
|
-
*
|
|
549
|
-
*
|
|
550
|
-
*
|
|
551
|
-
*
|
|
552
|
-
* end-of-call upload supersedes them, and if that upload is lost too, server-side recovery
|
|
553
|
-
* (#5196) fills the gap with silence.
|
|
583
|
+
* The current recording's shard pipeline (window → store → queue → transport), or null when not
|
|
584
|
+
* recording. A fresh pipeline per recording keeps a flush still in flight from a previous session
|
|
585
|
+
* off this session's indexes. Its queue is closed when the recording ends; anything it had not
|
|
586
|
+
* delivered stays in the local store for the next-load drain.
|
|
554
587
|
*/
|
|
555
|
-
private
|
|
556
|
-
/** How often
|
|
557
|
-
private static readonly
|
|
588
|
+
private recordingShards;
|
|
589
|
+
/** How often packets are cut into shards: the 5 s shard window, so each tick seals about one window. */
|
|
590
|
+
private static readonly recordingShardFlushMs;
|
|
591
|
+
/**
|
|
592
|
+
* Bytes of `keepalive` request bodies one terminal may use, the tail included. Browsers share about
|
|
593
|
+
* 64 KiB of in-flight keepalive bodies per page; the brief reserves 48 KiB of it for the recording,
|
|
594
|
+
* leaving room for a host's own close beacon (Caliber sends one from `pagehide` too).
|
|
595
|
+
*/
|
|
596
|
+
private static readonly recordingKeepaliveBudgetBytes;
|
|
597
|
+
/**
|
|
598
|
+
* Bytes of this page's recording `keepalive` requests still in flight. The browser's keepalive budget
|
|
599
|
+
* is per page, and closing a tab fires `visibilitychange` and `pagehide` back to back, so each dispatch
|
|
600
|
+
* budgets against what the previous one has not finished sending.
|
|
601
|
+
*/
|
|
602
|
+
private keepaliveBytesInFlight;
|
|
603
|
+
/** `sessionId/index` of shards whose `keepalive` copy is still in flight; never sent twice at once. */
|
|
604
|
+
private readonly keepaliveShardsInFlight;
|
|
605
|
+
/**
|
|
606
|
+
* Most runs one drain call makes (each run is itself bounded): it runs again when it was re-armed while
|
|
607
|
+
* running, or when a run deleted shards the server refused for good, which frees the page for the
|
|
608
|
+
* shards behind them.
|
|
609
|
+
*/
|
|
610
|
+
private static readonly maxDrainRunsPerCall;
|
|
611
|
+
/** The durable local shard store, created on first use and shared by every recording of this runtime. */
|
|
612
|
+
private shardStore;
|
|
613
|
+
/**
|
|
614
|
+
* True once a drain ran with a transport; the drain then does nothing until a recording ends and
|
|
615
|
+
* re-arms it ({@link rearmPendingShardDrain}).
|
|
616
|
+
*/
|
|
617
|
+
private pendingShardsDrained;
|
|
618
|
+
/** Bumped by every re-arm, so a drain already in flight cannot mark itself done over a re-arm. */
|
|
619
|
+
private drainGeneration;
|
|
620
|
+
/** The drain in flight, so overlapping callers share it instead of uploading the same shards twice. */
|
|
621
|
+
private drainInFlight;
|
|
558
622
|
/**
|
|
559
623
|
* Interval that tells the server this session is still in use, or `null` when no session is
|
|
560
624
|
* running. See {@link startLivenessPulse} for why the server cannot work this out itself.
|
|
@@ -571,12 +635,40 @@ export declare class RealtimeSessionRuntime {
|
|
|
571
635
|
* Recording-relative ms offset at which the IN-FLIGHT (not-yet-finalized) turn's audio
|
|
572
636
|
* actually BEGAN — captured the moment that turn's audio/text starts flowing (its first
|
|
573
637
|
* interim transcript), NOT inherited from the previous turn's end. `null` before the first
|
|
574
|
-
* turn / between turns (until the next turn's audio starts)
|
|
575
|
-
*
|
|
576
|
-
*
|
|
577
|
-
* the
|
|
638
|
+
* turn / between turns (until the next turn's audio starts), and it stays `null` for a turn
|
|
639
|
+
* whose start was never measured. Sent as `utteranceStartMs` on the turn's final transcript
|
|
640
|
+
* AS-IS — `null` is relayed as `null`, never replaced with 0 — so per-turn timing lines up
|
|
641
|
+
* with the recording even when a tool-call / silence gap sits between turns (the
|
|
642
|
+
* inherit-previous-end model mis-stamped the post-gap turn at the pre-gap offset). See
|
|
643
|
+
* {@link markTurnAudioStart}. A user speech onset signalled by the client does NOT land here;
|
|
644
|
+
* it has its own slot, {@link userSpeechStartMs}.
|
|
578
645
|
*/
|
|
579
646
|
private currentTurnStartMs;
|
|
647
|
+
/**
|
|
648
|
+
* Recording-relative ms offset of the first user speech onset the client signalled
|
|
649
|
+
* (`OnUserSpeechStart`) since the last final user transcript, or `null` when none was signalled.
|
|
650
|
+
* It has its own slot, apart from {@link currentTurnStartMs}, so a user onset never claims the
|
|
651
|
+
* assistant turn's start: OpenAI user transcripts arrive asynchronously, often after the
|
|
652
|
+
* assistant's interims (or its final) for the reply have already flowed. See
|
|
653
|
+
* {@link markUserSpeechStart}.
|
|
654
|
+
*/
|
|
655
|
+
private userSpeechStartMs;
|
|
656
|
+
/**
|
|
657
|
+
* `true` once an assistant turn has finalized after {@link userSpeechStartMs} was stamped BEFORE
|
|
658
|
+
* that turn began. The held onset may still be claimed by a late user transcript (OpenAI
|
|
659
|
+
* transcribes asynchronously), but the NEXT onset replaces it: an onset no transcript ever claimed
|
|
660
|
+
* (a VAD false positive, a failed transcription) must not become the start of the next real user
|
|
661
|
+
* turn. A barge-in onset (stamped during the assistant turn) is not superseded by that turn's
|
|
662
|
+
* final. An EMPTY user transcript releases the onset outright (see onClientTranscript).
|
|
663
|
+
*/
|
|
664
|
+
private userSpeechStartSuperseded;
|
|
665
|
+
/**
|
|
666
|
+
* Recording-relative ms offsets of user speech onsets keyed by the provider's conversation item id
|
|
667
|
+
* (`input_audio_buffer.speech_started.item_id`), paired with the matching final user transcript
|
|
668
|
+
* (`RealtimeClientTranscript.ItemId`) so a late user transcript never claims a later turn's onset
|
|
669
|
+
* (MJ#5902). Unkeyed drivers continue to use {@link userSpeechStartMs}.
|
|
670
|
+
*/
|
|
671
|
+
private readonly userSpeechStartByItemId;
|
|
580
672
|
/**
|
|
581
673
|
* Wall-anchor of the SESSION clock (#3832): `performance.now()` at the moment the call went
|
|
582
674
|
* live, or `null` before any call has. Read only through {@link nowTurnOffsetMs}.
|
|
@@ -673,6 +765,15 @@ export declare class RealtimeSessionRuntime {
|
|
|
673
765
|
* two `SessionEnded$` emissions for one session.
|
|
674
766
|
*/
|
|
675
767
|
private teardownInFlight;
|
|
768
|
+
/**
|
|
769
|
+
* True from the moment a start finishes (connected, liveness pulse running) until the slot is
|
|
770
|
+
* released. A driver's Fatal ends the call only while this is set: during a start, drivers emit
|
|
771
|
+
* Fatal from their socket handlers while Connect is still pending, and the start's own failure path
|
|
772
|
+
* ({@link failSessionStart}) is what reports that error, whether Connect then rejects or resolves
|
|
773
|
+
* (runMintedSession checks {@link lastSessionError} after Connect). Not {@link isSessionLive}: drivers move the
|
|
774
|
+
* state to 'listening' from inside Connect, and a live call that is resuming reads 'connecting'.
|
|
775
|
+
*/
|
|
776
|
+
private callWentLive;
|
|
676
777
|
/**
|
|
677
778
|
* The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
|
|
678
779
|
* back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
|
|
@@ -801,6 +902,21 @@ export declare class RealtimeSessionRuntime {
|
|
|
801
902
|
* regard to case.
|
|
802
903
|
*/
|
|
803
904
|
IsActiveFor(conversationId: string | null | undefined): boolean;
|
|
905
|
+
/**
|
|
906
|
+
* Takes the runtime's one driver slot for a new start; the two starts call it after awaiting any
|
|
907
|
+
* teardown in flight, and only when no End arrived during that wait (a changed {@link startGeneration}).
|
|
908
|
+
*
|
|
909
|
+
* A teardown still in flight is awaited rather than read as "busy": ending a call and starting the
|
|
910
|
+
* next one back to back is normal host behaviour (a retry, a re-mounted surface), and the flag it
|
|
911
|
+
* used to read flips only at the END of teardown, so the start was dropped with nothing to show
|
|
912
|
+
* for it — a bare `undefined`, which hosts had to guess at (MJ#4629). A call that is genuinely
|
|
913
|
+
* live is refused with an error the host can show; it is never silently ignored, and never
|
|
914
|
+
* overwritten, since a driver no slot points at can never be stopped again.
|
|
915
|
+
*
|
|
916
|
+
* Deliberately synchronous: the start's state must be set in the tick the host called it in, or an
|
|
917
|
+
* End issued straight after the Start (a user tapping back during the mint) finds nothing to abandon.
|
|
918
|
+
*/
|
|
919
|
+
private claimSessionSlot;
|
|
804
920
|
/**
|
|
805
921
|
* Start a client-direct voice session fronting `targetAgentId`.
|
|
806
922
|
*
|
|
@@ -831,7 +947,7 @@ export declare class RealtimeSessionRuntime {
|
|
|
831
947
|
* pickers, and never synthesize overrides beyond what the user explicitly chose.
|
|
832
948
|
* Omit/`null` for the server's defaults (today's behavior).
|
|
833
949
|
* @param recordingConsent Optional EXPLICIT "record this call" consent for THIS session. When
|
|
834
|
-
* `true`, the browser records a mic + agent-audio mix and uploads it
|
|
950
|
+
* `true`, the browser records a mic + agent-audio mix and uploads it as 5 s Opus shards during the call. When
|
|
835
951
|
* omitted/`null`, the per-user persisted preference (`mj.realtimeVoice.recordingConsent.v1`
|
|
836
952
|
* via {@link UserInfoEngine}) is read as the default. `false` never records.
|
|
837
953
|
* @param mediaCollectionId Optional per-session media-kit override (`MJ: Collections.ID`). When set,
|
|
@@ -895,8 +1011,8 @@ export declare class RealtimeSessionRuntime {
|
|
|
895
1011
|
/**
|
|
896
1012
|
* Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
|
|
897
1013
|
* publish the agent name, reset per-session state, and flip the session live (which is ALSO what
|
|
898
|
-
* makes
|
|
899
|
-
* runs before minting, not after). Returns the resolved recording consent, which the mint half
|
|
1014
|
+
* makes {@link claimSessionSlot} refuse a second start with {@link RealtimeSessionAlreadyActiveError}
|
|
1015
|
+
* while the mint is still in flight — hence it runs before minting, not after). Returns the resolved recording consent, which the mint half
|
|
900
1016
|
* reports to the server and the run half uses to decide whether to record.
|
|
901
1017
|
*/
|
|
902
1018
|
private beginSessionStart;
|
|
@@ -905,6 +1021,8 @@ export declare class RealtimeSessionRuntime {
|
|
|
905
1021
|
* the provider connection, and go live. `inputConversationId` is the conversation the START asked
|
|
906
1022
|
* for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
|
|
907
1023
|
* whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
|
|
1024
|
+
* `generation` is the {@link startGeneration} the entry point captured before its first await; every
|
|
1025
|
+
* await below is a window in which the host can end the session, and so was every await before this.
|
|
908
1026
|
*/
|
|
909
1027
|
private runMintedSession;
|
|
910
1028
|
/**
|
|
@@ -1058,6 +1176,15 @@ export declare class RealtimeSessionRuntime {
|
|
|
1058
1176
|
* overlay into 'error', and unwind whatever the half-built session already opened.
|
|
1059
1177
|
*/
|
|
1060
1178
|
private failSessionStart;
|
|
1179
|
+
/**
|
|
1180
|
+
* Ends the call over a failure: a failed start, an unusable provider, or a fatal provider error.
|
|
1181
|
+
*
|
|
1182
|
+
* The teardown is claimed BEFORE `'error'` is published. `ConnectionState$` emits synchronously, and a
|
|
1183
|
+
* host may end the call the moment it sees `'error'` (MobileApp's voice screen does); claimed first, that
|
|
1184
|
+
* host's End joins this teardown instead of starting its own as `'explicit'`, so the row still closes as
|
|
1185
|
+
* `Error` and `SessionEnded$` still says `'error'`. Teardown keeps a terminal `'error'` state.
|
|
1186
|
+
*/
|
|
1187
|
+
private endOnFailure;
|
|
1061
1188
|
/**
|
|
1062
1189
|
* End the active session: stop the mic, tear down the provider connection, and close
|
|
1063
1190
|
* the server-side agent session. Safe to call when no session is active.
|
|
@@ -1155,10 +1282,60 @@ export declare class RealtimeSessionRuntime {
|
|
|
1155
1282
|
* nothing (the recorder disables itself).
|
|
1156
1283
|
*/
|
|
1157
1284
|
private startRecording;
|
|
1158
|
-
/**
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1285
|
+
/**
|
|
1286
|
+
* Builds the transport for shard uploads and terminal requests. The default posts to MJAPI's REST
|
|
1287
|
+
* recording routes, derived from the session's GraphQL provider configuration, with credentials
|
|
1288
|
+
* read per request so a refreshed token is used. Returns null — shards then wait in the local
|
|
1289
|
+
* store — when the provider has no URL (a test double, a provider that is not GraphQL, or one not
|
|
1290
|
+
* configured yet). Override to supply another transport.
|
|
1291
|
+
*/
|
|
1292
|
+
protected createRecordingTransport(): IRecordingTransport | null;
|
|
1293
|
+
/**
|
|
1294
|
+
* Builds the durable local shard store. The default is IndexedDB where the host has it and a no-op
|
|
1295
|
+
* store elsewhere (Node, some React Native runtimes). Called once per runtime. Override to supply
|
|
1296
|
+
* another store.
|
|
1297
|
+
*/
|
|
1298
|
+
protected createRecordingShardStore(): IRecordingShardStore;
|
|
1299
|
+
/** The provider's signed-in user, or null (no provider yet, or an anonymous page). */
|
|
1300
|
+
private signedInUserID;
|
|
1301
|
+
/** The runtime's shard store, created on first use. */
|
|
1302
|
+
private ensureShardStore;
|
|
1303
|
+
/**
|
|
1304
|
+
* Re-uploads shards an earlier, interrupted call left in the local store (crash, closed tab, dead
|
|
1305
|
+
* network), each under its OWN session id and index, oldest first, at most
|
|
1306
|
+
* {@link MAX_DRAINED_SHARDS_PER_LOAD} per runtime. A shard is deleted only on the server's ack.
|
|
1307
|
+
*
|
|
1308
|
+
* Runs once per runtime instance, re-armed whenever a recording ends (which may leave shards behind
|
|
1309
|
+
* for the next session start on the same page). When there is no transport yet (no provider configuration, e.g.
|
|
1310
|
+
* before login) it returns 0 WITHOUT marking itself done, so a later call — session start — drains
|
|
1311
|
+
* instead of the shards being skipped for this page. Concurrent calls share one run. Never rejects.
|
|
1312
|
+
*
|
|
1313
|
+
* @returns the number of shards the server acknowledged in this call.
|
|
1314
|
+
*/
|
|
1315
|
+
DrainPendingRecordingShards(): Promise<number>;
|
|
1316
|
+
/** Makes the next {@link DrainPendingRecordingShards} call run again, even one already in flight. */
|
|
1317
|
+
private rearmPendingShardDrain;
|
|
1318
|
+
/**
|
|
1319
|
+
* The body of {@link DrainPendingRecordingShards}, up to {@link maxDrainRunsPerCall} runs. It runs
|
|
1320
|
+
* again after a re-arm while a run is in flight (a recovery after an outage gave up on more shards),
|
|
1321
|
+
* because the running pass listed the store before those shards were there; and after a run deleted
|
|
1322
|
+
* refused shards, because those filled the page that the shards behind them now get.
|
|
1323
|
+
*/
|
|
1324
|
+
private drainPendingShards;
|
|
1325
|
+
/** One drain pass over the signed-in user's stored shards (see {@link DrainStoredRecordingShardsOnce}). */
|
|
1326
|
+
private drainStoredShardsOnce;
|
|
1327
|
+
/** Builds this recording's shard pipeline and starts the 5 s flush. */
|
|
1328
|
+
private startShardFlushing;
|
|
1329
|
+
/**
|
|
1330
|
+
* Uploads succeed again after failing: re-sends, from the local store, the shards the queue gave up
|
|
1331
|
+
* on during the outage. The queue's attempts run out after about 10 s of a dead network (or an
|
|
1332
|
+
* expired token), and without this those shards would wait for a later page load — which a widget
|
|
1333
|
+
* guest never makes — leaving a permanent gap. Runs in the background, outside any page-lifecycle
|
|
1334
|
+
* handler, so the IndexedDB read is allowed. Never rejects.
|
|
1335
|
+
*/
|
|
1336
|
+
private recoverStoredShardsAfterOutage;
|
|
1337
|
+
/** Stops the periodic shard flush. */
|
|
1338
|
+
private stopShardFlushing;
|
|
1162
1339
|
/**
|
|
1163
1340
|
* Starts telling the server this session is still in use (#3533).
|
|
1164
1341
|
*
|
|
@@ -1190,37 +1367,109 @@ export declare class RealtimeSessionRuntime {
|
|
|
1190
1367
|
*/
|
|
1191
1368
|
private pulseLiveness;
|
|
1192
1369
|
/**
|
|
1193
|
-
*
|
|
1194
|
-
*
|
|
1195
|
-
*
|
|
1196
|
-
*
|
|
1370
|
+
* One 5 s tick: cuts the packets encoded since the last tick into shards, stores each one, then
|
|
1371
|
+
* uploads them. Best-effort — never disturbs the call and never rejects. Overlapping ticks are
|
|
1372
|
+
* safe: `Append` and `SealTail` are synchronous, so indexes are assigned in order, and the queue
|
|
1373
|
+
* bounds how many flushes upload at once. A failed shard is retained by the queue and resent under
|
|
1374
|
+
* its own index on later ticks (#5197), and stays in the store until acknowledged.
|
|
1375
|
+
*
|
|
1376
|
+
* Every tick seals whatever it took, even under a full window (the window's 250-packet and byte caps
|
|
1377
|
+
* still split a larger batch). Capture starts slightly after this timer, so a tick almost never
|
|
1378
|
+
* holds exactly 250 packets; sealing only full windows would leave nearly 5 s of encoded audio in
|
|
1379
|
+
* memory at every moment, unstored and unsent, which a killed tab loses.
|
|
1380
|
+
*/
|
|
1381
|
+
private flushRecordingShards;
|
|
1382
|
+
/** Writes shards to the local store, in order, before their upload starts. Never rejects. */
|
|
1383
|
+
private storeShards;
|
|
1384
|
+
/** Removes an acknowledged shard from the local store. Never rejects. */
|
|
1385
|
+
private deleteStoredShard;
|
|
1386
|
+
/**
|
|
1387
|
+
* Tells the server, right now, how much of the recording exists — for a host's `pagehide` and
|
|
1388
|
+
* `visibilitychange → hidden` handlers, where the page may be gone a moment later. Synchronous:
|
|
1389
|
+
* it seals the pending partial shard and fires the `keepalive` terminal with it, then re-sends
|
|
1390
|
+
* with `keepalive` whatever the server has not acknowledged yet (see
|
|
1391
|
+
* {@link dispatchRecordingTerminal}), and returns without waiting. Never stops the recorder: after
|
|
1392
|
+
* a `hidden` the call may well continue, and later audio simply starts the next shard index (the
|
|
1393
|
+
* tail is sealed, so no index is ever re-sent with different bytes). Also works while End is still
|
|
1394
|
+
* stopping the recorder, which is exactly when a host that does not await End navigates away.
|
|
1395
|
+
* No-op when nothing is recording, there is no session id or no transport, or nothing was
|
|
1396
|
+
* captured. Never throws.
|
|
1397
|
+
*
|
|
1398
|
+
* Deliberately opens NO IndexedDB transaction inside the handler: a `pagehide` handler gets a few
|
|
1399
|
+
* milliseconds, and a store write started there could not complete during unload anyway. The
|
|
1400
|
+
* shards it sealed are written to the store from a `setTimeout(0)` task after the handler returns:
|
|
1401
|
+
* on a page that is merely hidden (the call goes on) that keeps the durable copy until the server
|
|
1402
|
+
* acknowledges it; on a page that is unloading the task just never runs.
|
|
1403
|
+
*/
|
|
1404
|
+
SendRecordingTerminal(options?: RecordingTerminalOptions): void;
|
|
1405
|
+
/**
|
|
1406
|
+
* Fires the terminal request, then re-sends with `keepalive` every shard the server has not
|
|
1407
|
+
* acknowledged, then stores the shards sealed for this terminal. Shared by End and the page-lifecycle
|
|
1408
|
+
* handlers. Everything that needs the network is dispatched synchronously, before any storage work,
|
|
1409
|
+
* because the page may close at any moment from here on: an ordinary `fetch` is aborted on unload,
|
|
1410
|
+
* and a `keepalive` one is not.
|
|
1411
|
+
*
|
|
1412
|
+
* - The terminal goes first so it always gets the keepalive budget. The server waits for the shards
|
|
1413
|
+
* the terminal says exist, and assembles as soon as the last missing one arrives.
|
|
1414
|
+
* - The re-sent shards are the queue's unacknowledged ones (on the wire, waiting in a batch, or
|
|
1415
|
+
* retained after a failure) plus `sealedNow`, oldest first, within
|
|
1416
|
+
* {@link recordingKeepaliveBudgetBytes} together with the tail. A `sealedNow` shard that does not
|
|
1417
|
+
* fit, or whose keepalive upload fails, goes through the ordinary queue instead (it may still
|
|
1418
|
+
* land if the page stays). The queue keeps its own shards either way; a duplicate is acked as one.
|
|
1419
|
+
* - `sealedNow` and the tail are stored after the dispatch, from a `setTimeout(0)` task when called
|
|
1420
|
+
* from a page-lifecycle handler (`deferStore`). A shard's stored copy is deleted on its ack, after
|
|
1421
|
+
* the store write, so a fast ack can never leave an orphan row.
|
|
1422
|
+
*
|
|
1423
|
+
* Resolves true when the terminal was acknowledged. Never rejects.
|
|
1197
1424
|
*/
|
|
1198
|
-
private
|
|
1425
|
+
private dispatchRecordingTerminal;
|
|
1426
|
+
/** Counts `bytes` against the page's keepalive budget until `request` settles. */
|
|
1427
|
+
private trackKeepalive;
|
|
1199
1428
|
/**
|
|
1200
|
-
*
|
|
1201
|
-
*
|
|
1202
|
-
*
|
|
1203
|
-
* of a throw.
|
|
1429
|
+
* The `keepalive` re-send part of {@link dispatchRecordingTerminal}, budgeted against everything this
|
|
1430
|
+
* page still has in flight (the terminal just sent included). A shard whose `keepalive` copy is already
|
|
1431
|
+
* on the wire is not sent again. Never throws.
|
|
1204
1432
|
*/
|
|
1205
|
-
private
|
|
1433
|
+
private resendUnacknowledgedWithKeepalive;
|
|
1206
1434
|
/**
|
|
1207
|
-
*
|
|
1208
|
-
*
|
|
1209
|
-
*
|
|
1435
|
+
* Stores shards from a `setTimeout(0)` task rather than now — the path for shards sealed inside a
|
|
1436
|
+
* page-lifecycle handler, which must not open an IndexedDB transaction itself. Resolves once they
|
|
1437
|
+
* are stored (never, if the page unloads first). Never rejects.
|
|
1210
1438
|
*/
|
|
1211
|
-
private
|
|
1439
|
+
private storeShardsLater;
|
|
1212
1440
|
/**
|
|
1213
|
-
*
|
|
1214
|
-
*
|
|
1441
|
+
* Sends the terminal request for a pipeline whose tail is already sealed. Resolves true on a server
|
|
1442
|
+
* ack and logs the outcome with the session id. Never rejects.
|
|
1215
1443
|
*/
|
|
1216
|
-
private
|
|
1444
|
+
private sendRecordingTerminal;
|
|
1217
1445
|
/**
|
|
1218
|
-
*
|
|
1219
|
-
*
|
|
1220
|
-
*
|
|
1221
|
-
*
|
|
1446
|
+
* Ends the recording at teardown and STARTS its terminal request: stops the recorder (flushing the
|
|
1447
|
+
* encoder), cuts the remaining packets into shards with the tail trimmed to the recorder's captured
|
|
1448
|
+
* sample count, and dispatches the `keepalive` terminal plus `keepalive` re-sends of everything not
|
|
1449
|
+
* yet acknowledged, before storing anything ({@link dispatchRecordingTerminal}). Teardown awaits the
|
|
1450
|
+
* returned terminal only after `CloseAgentSession` ({@link awaitRecordingTerminal}), so the small,
|
|
1451
|
+
* critical close never waits behind the recording (#5195).
|
|
1452
|
+
*
|
|
1453
|
+
* The recorder and pipeline stay visible to {@link SendRecordingTerminal} until the recorder has
|
|
1454
|
+
* stopped: a `pagehide` in that window (a host that navigates away without awaiting End) still
|
|
1455
|
+
* sends what was captured, and End then continues at the next index.
|
|
1456
|
+
*
|
|
1457
|
+
* Returns null when nothing was recording or nothing was captured (no terminal is sent then).
|
|
1458
|
+
* Best-effort: never throws, never blocks teardown on the network.
|
|
1222
1459
|
*/
|
|
1223
|
-
private
|
|
1460
|
+
private finishRecording;
|
|
1461
|
+
/**
|
|
1462
|
+
* Stops the recorder, bounded by {@link RECORDER_STOP_TIMEOUT_MS}. Resolves the captured sample
|
|
1463
|
+
* count, or null when the recorder threw or did not finish in time (logged) — the caller then ends
|
|
1464
|
+
* the recording at the packets already encoded. Never rejects.
|
|
1465
|
+
*/
|
|
1466
|
+
private stopRecorder;
|
|
1467
|
+
/**
|
|
1468
|
+
* Waits for the terminal {@link finishRecording} started, bounded by
|
|
1469
|
+
* {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}, logs the outcome, then closes the recording's queue
|
|
1470
|
+
* (anything it had not delivered stays in the local store for the next-load drain). Never throws.
|
|
1471
|
+
*/
|
|
1472
|
+
private awaitRecordingTerminal;
|
|
1224
1473
|
/** Whether the channel catalog note still has to be sent (see {@link flushChannelCatalogNote}). */
|
|
1225
1474
|
private catalogNotePending;
|
|
1226
1475
|
/** Plugins resolved for the current start but not yet initialized. Consumed by {@link activateChannels}. */
|
|
@@ -1624,18 +1873,38 @@ export declare class RealtimeSessionRuntime {
|
|
|
1624
1873
|
* - NORMAL turns only — NARRATION interims are ephemeral and never persisted, so they must not
|
|
1625
1874
|
* claim the next real turn's start slot.
|
|
1626
1875
|
*
|
|
1627
|
-
*
|
|
1628
|
-
*
|
|
1629
|
-
*
|
|
1630
|
-
*
|
|
1631
|
-
* assistant answer, which always has interims.
|
|
1876
|
+
* Stamped on a turn's first interim transcript (all drivers for the assistant — the post-tool-gap
|
|
1877
|
+
* assistant answer — and user-interim drivers like Gemini/AssemblyAI). Final-only user turns
|
|
1878
|
+
* (OpenAI-protocol drivers) take their start from {@link markUserSpeechStart}'s own slot
|
|
1879
|
+
* instead; one with no speech-start signal either (ElevenLabs) relays `null` as-is.
|
|
1632
1880
|
*/
|
|
1633
1881
|
private markTurnAudioStart;
|
|
1882
|
+
/**
|
|
1883
|
+
* Stamps the user speech onset when the client signals that the user began to speak. When the
|
|
1884
|
+
* driver supplies a conversation `itemId`, records the FIRST onset for that item in
|
|
1885
|
+
* {@link userSpeechStartByItemId} so a mid-sentence pause that re-fires with the same id keeps the
|
|
1886
|
+
* original start. When no `itemId` is supplied, stamps the single-slot {@link userSpeechStartMs}
|
|
1887
|
+
* fallback (keeping the first onset unless {@link userSpeechStartSuperseded} is set).
|
|
1888
|
+
*/
|
|
1889
|
+
private markUserSpeechStart;
|
|
1890
|
+
/**
|
|
1891
|
+
* Ends the held user onset: a final user transcript (empty or not) closed the utterance it began.
|
|
1892
|
+
* When `itemId` is supplied, deletes only that item's entry in {@link userSpeechStartByItemId};
|
|
1893
|
+
* otherwise clears the unkeyed single slot.
|
|
1894
|
+
*/
|
|
1895
|
+
private releaseUserSpeechStart;
|
|
1896
|
+
/**
|
|
1897
|
+
* Consumes and deletes the onset recorded for `itemId` in {@link userSpeechStartByItemId}. When a
|
|
1898
|
+
* matching onset is found, also deletes any remaining map entry with an earlier offset so a VAD
|
|
1899
|
+
* false positive whose transcription never arrived cannot accumulate in the map.
|
|
1900
|
+
*/
|
|
1901
|
+
private takeKeyedUserSpeechStart;
|
|
1634
1902
|
/**
|
|
1635
1903
|
* The current per-turn offset in ms — the RECORDER's clock when one runs (an offset into a
|
|
1636
1904
|
* seekable file), else the SESSION clock (#3832: orderable and displayable, not seekable),
|
|
1637
1905
|
* else `null` before any call is live. One function so the two stamp sites cannot disagree
|
|
1638
|
-
* about which clock a session is on.
|
|
1906
|
+
* about which clock a session is on. Always whole milliseconds: the offsets travel as GraphQL
|
|
1907
|
+
* `Int`, which refuses a fraction, and a 48 kHz sample clock rarely lands on a whole millisecond.
|
|
1639
1908
|
*/
|
|
1640
1909
|
private nowTurnOffsetMs;
|
|
1641
1910
|
/**
|
|
@@ -1644,7 +1913,13 @@ export declare class RealtimeSessionRuntime {
|
|
|
1644
1913
|
* client's caption window).
|
|
1645
1914
|
*/
|
|
1646
1915
|
private replaceLastCaption;
|
|
1647
|
-
/**
|
|
1916
|
+
/**
|
|
1917
|
+
* Finalizes the user turn: push a caption + relay the final transcript.
|
|
1918
|
+
*
|
|
1919
|
+
* @param source `'typed'` for a {@link SendText} turn: it has no speech onset, so it neither takes
|
|
1920
|
+
* nor clears the held one (a spoken transcript may still be pending for it).
|
|
1921
|
+
* @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
|
|
1922
|
+
*/
|
|
1648
1923
|
private onUserTranscript;
|
|
1649
1924
|
/**
|
|
1650
1925
|
* Routes a provider tool call: names matching a registered client-tool prefix execute
|
|
@@ -1738,20 +2013,28 @@ export declare class RealtimeSessionRuntime {
|
|
|
1738
2013
|
* Relays a final transcript turn to MJ via `RelayRealtimeTranscript`.
|
|
1739
2014
|
*
|
|
1740
2015
|
* When the session is being recorded, per-turn timing rides along: `utteranceEndMs` is the
|
|
1741
|
-
* recording-relative offset at finalization, and `utteranceStartMs` is
|
|
1742
|
-
*
|
|
1743
|
-
*
|
|
1744
|
-
*
|
|
1745
|
-
*
|
|
1746
|
-
*
|
|
2016
|
+
* recording-relative offset at finalization, and `utteranceStartMs` is where THIS turn's audio
|
|
2017
|
+
* actually began — NOT inherited from the previous turn's end. For a user turn whose onset the
|
|
2018
|
+
* client signalled, that is {@link userSpeechStartMs}; otherwise it is the offset captured by
|
|
2019
|
+
* {@link markTurnAudioStart} on the turn's first interim. That distinction is the timing fix:
|
|
2020
|
+
* when a tool-call / silence gap sits between turns, the post-gap turn's audio starts much
|
|
2021
|
+
* later, so inheriting the prior turn's end stamped the cue ~gap-length too early. Both are
|
|
2022
|
+
* omitted (left `null`) when the session isn't being recorded. `utteranceStartMs` is also
|
|
2023
|
+
* `null` when no start was measured for this turn; it is never replaced with a placeholder 0.
|
|
1747
2024
|
*
|
|
1748
2025
|
* A correction (`replacesPrevious`) doesn't open a new turn, so it carries no start and doesn't
|
|
1749
2026
|
* reset the per-turn start guard. After a normal finalization the guard is cleared so the NEXT
|
|
1750
|
-
* turn re-stamps its start from where ITS audio begins
|
|
2027
|
+
* turn re-stamps its start from where ITS audio begins — except for a user turn that used its
|
|
2028
|
+
* own speech-start slot, which leaves the shared slot alone (it may hold the in-flight
|
|
2029
|
+
* assistant turn's start). Every final user transcript, corrections included, clears the
|
|
2030
|
+
* speech-start slot.
|
|
1751
2031
|
*
|
|
1752
2032
|
* @param replacesPrevious CORRECTION semantics: the server updates the session's most
|
|
1753
2033
|
* recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
|
|
1754
2034
|
* post-barge-in `agent_response_correction`).
|
|
2035
|
+
* @param source `'typed'` for a user turn sent with {@link SendText}: it does not read or clear
|
|
2036
|
+
* {@link userSpeechStartMs} or {@link userSpeechStartByItemId}, which belong to the user's speech.
|
|
2037
|
+
* @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
|
|
1755
2038
|
*/
|
|
1756
2039
|
private relayTranscript;
|
|
1757
2040
|
/**
|
|
@@ -1870,18 +2153,34 @@ export declare class RealtimeSessionRuntime {
|
|
|
1870
2153
|
/** Tears down the delegation progress subscription and resets the narration throttle. */
|
|
1871
2154
|
private teardownDelegationProgress;
|
|
1872
2155
|
/**
|
|
1873
|
-
* Tears down all client resources and
|
|
1874
|
-
*
|
|
2156
|
+
* Tears down all client resources and closes the server session when one was minted.
|
|
2157
|
+
*
|
|
2158
|
+
* The server row is closed whichever way the call ended: a start that failed after the mint used to
|
|
2159
|
+
* leave its row `Active` until the janitor's staleness sweep, 15 minutes later, so every host that
|
|
2160
|
+
* counts live sessions saw a phantom call (MJ#6013). The janitor remains the fallback for a client
|
|
2161
|
+
* that dies before it can say anything; it is never the plan.
|
|
2162
|
+
*
|
|
2163
|
+
* @param end why the call ended — `'explicit'` when the host or user ended it, `'error'` when the
|
|
2164
|
+
* runtime ended it because the start or the call failed. Reported on `SessionEnded$` and stamped on
|
|
2165
|
+
* the server row as its `CloseReason` (`Explicit` / `Error`).
|
|
1875
2166
|
*/
|
|
1876
2167
|
private teardown;
|
|
1877
2168
|
/** The body of {@link teardown}; never called concurrently with itself. */
|
|
1878
2169
|
private runTeardown;
|
|
2170
|
+
/** The teardown's work: stop the call, hand back what it held, and close the server session. */
|
|
2171
|
+
private releaseCallResources;
|
|
2172
|
+
/** Clears the slot and tells the host the call ended; runs even when {@link releaseCallResources} threw. */
|
|
2173
|
+
private releaseSessionSlot;
|
|
1879
2174
|
/**
|
|
1880
2175
|
* Calls the host's {@link IRealtimeMediaHost.ReleaseMicrophone}. Best-effort by contract: a failure
|
|
1881
2176
|
* is logged and never thrown, because it must never block ending a call.
|
|
1882
2177
|
*/
|
|
1883
2178
|
private releaseMicrophone;
|
|
1884
|
-
/**
|
|
2179
|
+
/**
|
|
2180
|
+
* Calls the `CloseAgentSession` mutation (provisioned in P4b) with why the client ended the call, which
|
|
2181
|
+
* the server stamps as the row's `CloseReason`. Requires a server that accepts the `closeReason`
|
|
2182
|
+
* argument (MJ#6013); the runtime and the server ship together.
|
|
2183
|
+
*/
|
|
1885
2184
|
private closeServerSession;
|
|
1886
2185
|
/** Pushes a caption onto the live list (immutable update for change detection). */
|
|
1887
2186
|
private appendCaption;
|