@memberjunction/realtime-runtime 6.2.0-edge.4 → 6.2.0-edge.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/dist/hosts/IRealtimeMediaHost.d.ts +25 -30
  2. package/dist/hosts/IRealtimeMediaHost.d.ts.map +1 -1
  3. package/dist/index.d.ts +6 -2
  4. package/dist/index.d.ts.map +1 -1
  5. package/dist/index.js +5 -2
  6. package/dist/index.js.map +1 -1
  7. package/dist/session/RealtimeSessionRuntime.d.ts +372 -73
  8. package/dist/session/RealtimeSessionRuntime.d.ts.map +1 -1
  9. package/dist/session/RealtimeSessionRuntime.js +919 -212
  10. package/dist/session/RealtimeSessionRuntime.js.map +1 -1
  11. package/dist/session/RecordingShardDrain.d.ts +59 -0
  12. package/dist/session/RecordingShardDrain.d.ts.map +1 -0
  13. package/dist/session/RecordingShardDrain.js +135 -0
  14. package/dist/session/RecordingShardDrain.js.map +1 -0
  15. package/dist/session/RecordingShardQueue.d.ts +64 -38
  16. package/dist/session/RecordingShardQueue.d.ts.map +1 -1
  17. package/dist/session/RecordingShardQueue.js +106 -84
  18. package/dist/session/RecordingShardQueue.js.map +1 -1
  19. package/dist/session/RecordingShardStore.d.ts +57 -0
  20. package/dist/session/RecordingShardStore.d.ts.map +1 -0
  21. package/dist/session/RecordingShardStore.js +184 -0
  22. package/dist/session/RecordingShardStore.js.map +1 -0
  23. package/dist/session/RecordingShardWindow.d.ts +42 -0
  24. package/dist/session/RecordingShardWindow.d.ts.map +1 -0
  25. package/dist/session/RecordingShardWindow.js +88 -0
  26. package/dist/session/RecordingShardWindow.js.map +1 -0
  27. package/dist/session/RecordingTransport.d.ts +67 -0
  28. package/dist/session/RecordingTransport.d.ts.map +1 -0
  29. package/dist/session/RecordingTransport.js +61 -0
  30. package/dist/session/RecordingTransport.js.map +1 -0
  31. package/package.json +10 -9
@@ -15,6 +15,31 @@ import type { RealtimeContextActionResult } from '../channels/channel-contract-t
15
15
  import { type AppClientToolRegistration } from './app-client-tool-registry.js';
16
16
  import { type IChannelExposurePreferences } from './channel-exposure-preferences.js';
17
17
  import { type RealtimeSessionStartOptions } from './channel-session-scope.js';
18
+ import { type IRecordingShardStore } from './RecordingShardStore.js';
19
+ import type { IRecordingTransport } from './RecordingTransport.js';
20
+ /**
21
+ * How long teardown waits for the server to acknowledge the recording's terminal request, measured
22
+ * from after `CloseAgentSession`. The terminal is `keepalive`, so it completes even if the page goes
23
+ * away; the wait only keeps `SavingRecording$` (and so a host's "don't close yet" guard) up while the
24
+ * server has not confirmed. Bounded so a dead server can never hold the End of a call hostage —
25
+ * unacknowledged shards stay in the local store and the server janitor assembles what arrived.
26
+ */
27
+ export declare const RECORDING_TERMINAL_ACK_TIMEOUT_MS = 3000;
28
+ /**
29
+ * How long teardown waits for the recorder to stop and flush its encoder. `Stop()` runs BEFORE
30
+ * `CloseAgentSession`, so a wedged encoder must not be able to hold the End of a call (and the
31
+ * server session) open; past this the recording ends at the packets already encoded.
32
+ */
33
+ export declare const RECORDER_STOP_TIMEOUT_MS = 2000;
34
+ /** Options of {@link RealtimeSessionRuntime.SendRecordingTerminal}. */
35
+ export interface RecordingTerminalOptions {
36
+ /**
37
+ * False when the call may continue — the host's `visibilitychange → hidden` handler. The server then
38
+ * stamps the counts but assembles only after its grace period, so switching tabs does not rewrite the
39
+ * recording each time. Defaults to true (`pagehide`: the page is going away).
40
+ */
41
+ Final?: boolean;
42
+ }
18
43
  /**
19
44
  * `MJ: User Settings` key for the per-user "record this voice call" consent toggle. Stored as
20
45
  * the literal string `'true'`/`'false'` (read with `=== 'true'`), cross-device via
@@ -224,6 +249,15 @@ export interface RealtimeSessionRunOptions {
224
249
  */
225
250
  readonly appContext?: AppContextSnapshot | null;
226
251
  }
252
+ /** Why a call ended, as the runtime reports it on `SessionEnded$`. */
253
+ export type SessionEndReason = 'explicit' | 'error';
254
+ /** The `CloseReason` values a client may stamp on its own session row; the others belong to the server. */
255
+ export type ClientCloseReason = 'Explicit' | 'Error';
256
+ /** Thrown by a start while a call is live: the runtime holds one driver, and never replaces a live one. */
257
+ export declare class RealtimeSessionAlreadyActiveError extends Error {
258
+ readonly ActiveAgentSessionId: string | null;
259
+ constructor(ActiveAgentSessionId: string | null);
260
+ }
227
261
  /**
228
262
  * Drives a **client-direct** real-time voice session: the browser mints an ephemeral
229
263
  * token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
@@ -250,9 +284,9 @@ export declare class RealtimeSessionRuntime {
250
284
  protected readonly mediaHost: IRealtimeMediaHost;
251
285
  /**
252
286
  * @param mediaHost the platform's media capabilities. The runtime never touches `navigator`,
253
- * `Blob` or `FileReader` itself — microphone acquisition and audio recording
254
- * are the host's, because both are platform-specific product decisions
255
- * (permission UX, container format, where the bytes live).
287
+ * `Blob` or `FileReader` itself — microphone acquisition and audio capture/encoding
288
+ * are the host's, because both are platform-specific (permission UX, which
289
+ * encoder the platform offers).
256
290
  */
257
291
  constructor(mediaHost: IRealtimeMediaHost);
258
292
  private _connectionState$;
@@ -322,10 +356,11 @@ export declare class RealtimeSessionRuntime {
322
356
  readonly Minimized$: Observable<boolean>;
323
357
  /**
324
358
  * True while the end-of-call recording is being saved: from the first statement of a teardown
325
- * that holds a recorder until that teardown settles. False at construction and for unrecorded
326
- * sessions. Exists for #5195: the page must stay open until the consolidated upload lands, or
327
- * the recording is lost. The runtime is platform-free, so hosts own the reaction (a browser
328
- * `beforeunload` guard, a "saving" indicator, etc.).
359
+ * that holds a recorder until that teardown settles — which includes waiting (at most
360
+ * {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}) for the server to acknowledge the recording's terminal
361
+ * request. False at construction and for unrecorded sessions. Exists for #5195; the runtime is
362
+ * platform-free, so hosts own the reaction (a "saving" indicator, a disabled navigation button,
363
+ * etc.).
329
364
  */
330
365
  readonly SavingRecording$: Observable<boolean>;
331
366
  /** Synchronous read of {@link SavingRecording$}. */
@@ -416,13 +451,13 @@ export declare class RealtimeSessionRuntime {
416
451
  * Fired EXACTLY ONCE per session as teardown begins, with the prior
417
452
  * `agentSessionId` (so subscribers can correlate against `SessionStarted$`'s
418
453
  * sessionId) and the client-distinguishable reason — `'explicit'` when the
419
- * user called `EndRealtimeSession`, `'error'` when teardown ran from a catch
420
- * block. Server-side close paths (janitor, shutdown) do NOT propagate here —
454
+ * user called `EndRealtimeSession`, `'error'` when the runtime ended the call over a failure: a
455
+ * failed start, or a fatal error from the provider mid-call. Server-side close paths (janitor, shutdown) do NOT propagate here —
421
456
  * they happen out-of-process and have no client push channel today.
422
457
  */
423
458
  readonly SessionEnded$: Observable<{
424
459
  sessionId: string;
425
- reason: 'explicit' | 'error';
460
+ reason: SessionEndReason;
426
461
  }>;
427
462
  /**
428
463
  * Fires with the channel PLUGIN every time the agent ACTS on that channel (a tool call
@@ -536,25 +571,54 @@ export declare class RealtimeSessionRuntime {
536
571
  private narrationTemplate;
537
572
  /**
538
573
  * The active session's audio recorder (mic + agent mix), or `null` when the user didn't
539
- * consent or the browser can't record. Created after the client connects; stopped + uploaded
540
- * at teardown.
574
+ * consent or the browser can't record. Created after the client connects; its packets are
575
+ * shipped as 5 s shards during the call, and it is stopped at teardown.
541
576
  */
542
577
  private recorder;
543
578
  /** ISO timestamp of when recording started — sent to the server on session start. */
544
579
  private recordingStartedAtIso;
545
- /** Interval that flushes ~15s crash-recovery shards to the server during a recording. */
546
- private segmentTimer;
580
+ /** Interval that cuts and uploads a shard every {@link recordingShardFlushMs} during a recording. */
581
+ private shardFlushTimer;
547
582
  /**
548
- * Owns shard indexing and retry for the current recording: it assigns each shard its index and
549
- * keeps failed shards to resend under that same index. A fresh queue per recording keeps a flush
550
- * still in flight from a previous session off this session's indexes. Null when not recording.
551
- * A recording that ends discards its queue, and any retained shard with it: the consolidated
552
- * end-of-call upload supersedes them, and if that upload is lost too, server-side recovery
553
- * (#5196) fills the gap with silence.
583
+ * The current recording's shard pipeline (window → store → queue → transport), or null when not
584
+ * recording. A fresh pipeline per recording keeps a flush still in flight from a previous session
585
+ * off this session's indexes. Its queue is closed when the recording ends; anything it had not
586
+ * delivered stays in the local store for the next-load drain.
554
587
  */
555
- private shardQueue;
556
- /** How often crash-recovery shards are flushed during a recording. */
557
- private static readonly segmentFlushMs;
588
+ private recordingShards;
589
+ /** How often packets are cut into shards: the 5 s shard window, so each tick seals about one window. */
590
+ private static readonly recordingShardFlushMs;
591
+ /**
592
+ * Bytes of `keepalive` request bodies one terminal may use, the tail included. Browsers share about
593
+ * 64 KiB of in-flight keepalive bodies per page; the brief reserves 48 KiB of it for the recording,
594
+ * leaving room for a host's own close beacon (Caliber sends one from `pagehide` too).
595
+ */
596
+ private static readonly recordingKeepaliveBudgetBytes;
597
+ /**
598
+ * Bytes of this page's recording `keepalive` requests still in flight. The browser's keepalive budget
599
+ * is per page, and closing a tab fires `visibilitychange` and `pagehide` back to back, so each dispatch
600
+ * budgets against what the previous one has not finished sending.
601
+ */
602
+ private keepaliveBytesInFlight;
603
+ /** `sessionId/index` of shards whose `keepalive` copy is still in flight; never sent twice at once. */
604
+ private readonly keepaliveShardsInFlight;
605
+ /**
606
+ * Most runs one drain call makes (each run is itself bounded): it runs again when it was re-armed while
607
+ * running, or when a run deleted shards the server refused for good, which frees the page for the
608
+ * shards behind them.
609
+ */
610
+ private static readonly maxDrainRunsPerCall;
611
+ /** The durable local shard store, created on first use and shared by every recording of this runtime. */
612
+ private shardStore;
613
+ /**
614
+ * True once a drain ran with a transport; the drain then does nothing until a recording ends and
615
+ * re-arms it ({@link rearmPendingShardDrain}).
616
+ */
617
+ private pendingShardsDrained;
618
+ /** Bumped by every re-arm, so a drain already in flight cannot mark itself done over a re-arm. */
619
+ private drainGeneration;
620
+ /** The drain in flight, so overlapping callers share it instead of uploading the same shards twice. */
621
+ private drainInFlight;
558
622
  /**
559
623
  * Interval that tells the server this session is still in use, or `null` when no session is
560
624
  * running. See {@link startLivenessPulse} for why the server cannot work this out itself.
@@ -571,12 +635,40 @@ export declare class RealtimeSessionRuntime {
571
635
  * Recording-relative ms offset at which the IN-FLIGHT (not-yet-finalized) turn's audio
572
636
  * actually BEGAN — captured the moment that turn's audio/text starts flowing (its first
573
637
  * interim transcript), NOT inherited from the previous turn's end. `null` before the first
574
- * turn / between turns (until the next turn's audio starts). Sent as `utteranceStartMs` on
575
- * the turn's final transcript so per-turn timing lines up with the recording even when a
576
- * tool-call / silence gap sits between turns (the inherit-previous-end model mis-stamped
577
- * the post-gap turn at the pre-gap offset). See {@link markTurnAudioStart}.
638
+ * turn / between turns (until the next turn's audio starts), and it stays `null` for a turn
639
+ * whose start was never measured. Sent as `utteranceStartMs` on the turn's final transcript
640
+ * AS-IS — `null` is relayed as `null`, never replaced with 0 — so per-turn timing lines up
641
+ * with the recording even when a tool-call / silence gap sits between turns (the
642
+ * inherit-previous-end model mis-stamped the post-gap turn at the pre-gap offset). See
643
+ * {@link markTurnAudioStart}. A user speech onset signalled by the client does NOT land here;
644
+ * it has its own slot, {@link userSpeechStartMs}.
578
645
  */
579
646
  private currentTurnStartMs;
647
+ /**
648
+ * Recording-relative ms offset of the first user speech onset the client signalled
649
+ * (`OnUserSpeechStart`) since the last final user transcript, or `null` when none was signalled.
650
+ * It has its own slot, apart from {@link currentTurnStartMs}, so a user onset never claims the
651
+ * assistant turn's start: OpenAI user transcripts arrive asynchronously, often after the
652
+ * assistant's interims (or its final) for the reply have already flowed. See
653
+ * {@link markUserSpeechStart}.
654
+ */
655
+ private userSpeechStartMs;
656
+ /**
657
+ * `true` once an assistant turn has finalized after {@link userSpeechStartMs} was stamped BEFORE
658
+ * that turn began. The held onset may still be claimed by a late user transcript (OpenAI
659
+ * transcribes asynchronously), but the NEXT onset replaces it: an onset no transcript ever claimed
660
+ * (a VAD false positive, a failed transcription) must not become the start of the next real user
661
+ * turn. A barge-in onset (stamped during the assistant turn) is not superseded by that turn's
662
+ * final. An EMPTY user transcript releases the onset outright (see onClientTranscript).
663
+ */
664
+ private userSpeechStartSuperseded;
665
+ /**
666
+ * Recording-relative ms offsets of user speech onsets keyed by the provider's conversation item id
667
+ * (`input_audio_buffer.speech_started.item_id`), paired with the matching final user transcript
668
+ * (`RealtimeClientTranscript.ItemId`) so a late user transcript never claims a later turn's onset
669
+ * (MJ#5902). Unkeyed drivers continue to use {@link userSpeechStartMs}.
670
+ */
671
+ private readonly userSpeechStartByItemId;
580
672
  /**
581
673
  * Wall-anchor of the SESSION clock (#3832): `performance.now()` at the moment the call went
582
674
  * live, or `null` before any call has. Read only through {@link nowTurnOffsetMs}.
@@ -673,6 +765,15 @@ export declare class RealtimeSessionRuntime {
673
765
  * two `SessionEnded$` emissions for one session.
674
766
  */
675
767
  private teardownInFlight;
768
+ /**
769
+ * True from the moment a start finishes (connected, liveness pulse running) until the slot is
770
+ * released. A driver's Fatal ends the call only while this is set: during a start, drivers emit
771
+ * Fatal from their socket handlers while Connect is still pending, and the start's own failure path
772
+ * ({@link failSessionStart}) is what reports that error, whether Connect then rejects or resolves
773
+ * (runMintedSession checks {@link lastSessionError} after Connect). Not {@link isSessionLive}: drivers move the
774
+ * state to 'listening' from inside Connect, and a live call that is resuming reads 'connecting'.
775
+ */
776
+ private callWentLive;
676
777
  /**
677
778
  * The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
678
779
  * back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
@@ -801,6 +902,21 @@ export declare class RealtimeSessionRuntime {
801
902
  * regard to case.
802
903
  */
803
904
  IsActiveFor(conversationId: string | null | undefined): boolean;
905
+ /**
906
+ * Takes the runtime's one driver slot for a new start; the two starts call it after awaiting any
907
+ * teardown in flight, and only when no End arrived during that wait (a changed {@link startGeneration}).
908
+ *
909
+ * A teardown still in flight is awaited rather than read as "busy": ending a call and starting the
910
+ * next one back to back is normal host behaviour (a retry, a re-mounted surface), and the flag it
911
+ * used to read flips only at the END of teardown, so the start was dropped with nothing to show
912
+ * for it — a bare `undefined`, which hosts had to guess at (MJ#4629). A call that is genuinely
913
+ * live is refused with an error the host can show; it is never silently ignored, and never
914
+ * overwritten, since a driver no slot points at can never be stopped again.
915
+ *
916
+ * Deliberately synchronous: the start's state must be set in the tick the host called it in, or an
917
+ * End issued straight after the Start (a user tapping back during the mint) finds nothing to abandon.
918
+ */
919
+ private claimSessionSlot;
804
920
  /**
805
921
  * Start a client-direct voice session fronting `targetAgentId`.
806
922
  *
@@ -831,7 +947,7 @@ export declare class RealtimeSessionRuntime {
831
947
  * pickers, and never synthesize overrides beyond what the user explicitly chose.
832
948
  * Omit/`null` for the server's defaults (today's behavior).
833
949
  * @param recordingConsent Optional EXPLICIT "record this call" consent for THIS session. When
834
- * `true`, the browser records a mic + agent-audio mix and uploads it at session end. When
950
+ * `true`, the browser records a mic + agent-audio mix and uploads it as 5 s Opus shards during the call. When
835
951
  * omitted/`null`, the per-user persisted preference (`mj.realtimeVoice.recordingConsent.v1`
836
952
  * via {@link UserInfoEngine}) is read as the default. `false` never records.
837
953
  * @param mediaCollectionId Optional per-session media-kit override (`MJ: Collections.ID`). When set,
@@ -895,8 +1011,8 @@ export declare class RealtimeSessionRuntime {
895
1011
  /**
896
1012
  * Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
897
1013
  * publish the agent name, reset per-session state, and flip the session live (which is ALSO what
898
- * makes the `IsActive` guard suppress duplicate starts while the mint is still in flight — hence it
899
- * runs before minting, not after). Returns the resolved recording consent, which the mint half
1014
+ * makes {@link claimSessionSlot} refuse a second start with {@link RealtimeSessionAlreadyActiveError}
1015
+ * while the mint is still in flight — hence it runs before minting, not after). Returns the resolved recording consent, which the mint half
900
1016
  * reports to the server and the run half uses to decide whether to record.
901
1017
  */
902
1018
  private beginSessionStart;
@@ -905,6 +1021,8 @@ export declare class RealtimeSessionRuntime {
905
1021
  * the provider connection, and go live. `inputConversationId` is the conversation the START asked
906
1022
  * for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
907
1023
  * whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
1024
+ * `generation` is the {@link startGeneration} the entry point captured before its first await; every
1025
+ * await below is a window in which the host can end the session, and so was every await before this.
908
1026
  */
909
1027
  private runMintedSession;
910
1028
  /**
@@ -1058,6 +1176,15 @@ export declare class RealtimeSessionRuntime {
1058
1176
  * overlay into 'error', and unwind whatever the half-built session already opened.
1059
1177
  */
1060
1178
  private failSessionStart;
1179
+ /**
1180
+ * Ends the call over a failure: a failed start, an unusable provider, or a fatal provider error.
1181
+ *
1182
+ * The teardown is claimed BEFORE `'error'` is published. `ConnectionState$` emits synchronously, and a
1183
+ * host may end the call the moment it sees `'error'` (MobileApp's voice screen does); claimed first, that
1184
+ * host's End joins this teardown instead of starting its own as `'explicit'`, so the row still closes as
1185
+ * `Error` and `SessionEnded$` still says `'error'`. Teardown keeps a terminal `'error'` state.
1186
+ */
1187
+ private endOnFailure;
1061
1188
  /**
1062
1189
  * End the active session: stop the mic, tear down the provider connection, and close
1063
1190
  * the server-side agent session. Safe to call when no session is active.
@@ -1155,10 +1282,60 @@ export declare class RealtimeSessionRuntime {
1155
1282
  * nothing (the recorder disables itself).
1156
1283
  */
1157
1284
  private startRecording;
1158
- /** Begins flushing ~15s crash-recovery shards to the server for the duration of the recording. */
1159
- private startSegmentFlushing;
1160
- /** Stops the periodic crash-recovery shard flush. */
1161
- private stopSegmentFlushing;
1285
+ /**
1286
+ * Builds the transport for shard uploads and terminal requests. The default posts to MJAPI's REST
1287
+ * recording routes, derived from the session's GraphQL provider configuration, with credentials
1288
+ * read per request so a refreshed token is used. Returns null — shards then wait in the local
1289
+ * store — when the provider has no URL (a test double, a provider that is not GraphQL, or one not
1290
+ * configured yet). Override to supply another transport.
1291
+ */
1292
+ protected createRecordingTransport(): IRecordingTransport | null;
1293
+ /**
1294
+ * Builds the durable local shard store. The default is IndexedDB where the host has it and a no-op
1295
+ * store elsewhere (Node, some React Native runtimes). Called once per runtime. Override to supply
1296
+ * another store.
1297
+ */
1298
+ protected createRecordingShardStore(): IRecordingShardStore;
1299
+ /** The provider's signed-in user, or null (no provider yet, or an anonymous page). */
1300
+ private signedInUserID;
1301
+ /** The runtime's shard store, created on first use. */
1302
+ private ensureShardStore;
1303
+ /**
1304
+ * Re-uploads shards an earlier, interrupted call left in the local store (crash, closed tab, dead
1305
+ * network), each under its OWN session id and index, oldest first, at most
1306
+ * {@link MAX_DRAINED_SHARDS_PER_LOAD} per runtime. A shard is deleted only on the server's ack.
1307
+ *
1308
+ * Runs once per runtime instance, re-armed whenever a recording ends (which may leave shards behind
1309
+ * for the next session start on the same page). When there is no transport yet (no provider configuration, e.g.
1310
+ * before login) it returns 0 WITHOUT marking itself done, so a later call — session start — drains
1311
+ * instead of the shards being skipped for this page. Concurrent calls share one run. Never rejects.
1312
+ *
1313
+ * @returns the number of shards the server acknowledged in this call.
1314
+ */
1315
+ DrainPendingRecordingShards(): Promise<number>;
1316
+ /** Makes the next {@link DrainPendingRecordingShards} call run again, even one already in flight. */
1317
+ private rearmPendingShardDrain;
1318
+ /**
1319
+ * The body of {@link DrainPendingRecordingShards}, up to {@link maxDrainRunsPerCall} runs. It runs
1320
+ * again after a re-arm while a run is in flight (a recovery after an outage gave up on more shards),
1321
+ * because the running pass listed the store before those shards were there; and after a run deleted
1322
+ * refused shards, because those filled the page that the shards behind them now get.
1323
+ */
1324
+ private drainPendingShards;
1325
+ /** One drain pass over the signed-in user's stored shards (see {@link DrainStoredRecordingShardsOnce}). */
1326
+ private drainStoredShardsOnce;
1327
+ /** Builds this recording's shard pipeline and starts the 5 s flush. */
1328
+ private startShardFlushing;
1329
+ /**
1330
+ * Uploads succeed again after failing: re-sends, from the local store, the shards the queue gave up
1331
+ * on during the outage. The queue's attempts run out after about 10 s of a dead network (or an
1332
+ * expired token), and without this those shards would wait for a later page load — which a widget
1333
+ * guest never makes — leaving a permanent gap. Runs in the background, outside any page-lifecycle
1334
+ * handler, so the IndexedDB read is allowed. Never rejects.
1335
+ */
1336
+ private recoverStoredShardsAfterOutage;
1337
+ /** Stops the periodic shard flush. */
1338
+ private stopShardFlushing;
1162
1339
  /**
1163
1340
  * Starts telling the server this session is still in use (#3533).
1164
1341
  *
@@ -1190,37 +1367,109 @@ export declare class RealtimeSessionRuntime {
1190
1367
  */
1191
1368
  private pulseLiveness;
1192
1369
  /**
1193
- * Uploads the chunks captured since the last flush as one crash-recovery shard (durability only;
1194
- * the canonical file is still the full upload at teardown). Best-effort — never disturbs the call:
1195
- * a failed shard is retained by the queue and resent under its own index on later flushes, up to a
1196
- * bounded number of attempts (#5197).
1370
+ * One 5 s tick: cuts the packets encoded since the last tick into shards, stores each one, then
1371
+ * uploads them. Best-effort — never disturbs the call and never rejects. Overlapping ticks are
1372
+ * safe: `Append` and `SealTail` are synchronous, so indexes are assigned in order, and the queue
1373
+ * bounds how many flushes upload at once. A failed shard is retained by the queue and resent under
1374
+ * its own index on later ticks (#5197), and stays in the store until acknowledged.
1375
+ *
1376
+ * Every tick seals whatever it took, even under a full window (the window's 250-packet and byte caps
1377
+ * still split a larger batch). Capture starts slightly after this timer, so a tick almost never
1378
+ * holds exactly 250 packets; sealing only full windows would leave nearly 5 s of encoded audio in
1379
+ * memory at every moment, unstored and unsent, which a killed tab loses.
1380
+ */
1381
+ private flushRecordingShards;
1382
+ /** Writes shards to the local store, in order, before their upload starts. Never rejects. */
1383
+ private storeShards;
1384
+ /** Removes an acknowledged shard from the local store. Never rejects. */
1385
+ private deleteStoredShard;
1386
+ /**
1387
+ * Tells the server, right now, how much of the recording exists — for a host's `pagehide` and
1388
+ * `visibilitychange → hidden` handlers, where the page may be gone a moment later. Synchronous:
1389
+ * it seals the pending partial shard and fires the `keepalive` terminal with it, then re-sends
1390
+ * with `keepalive` whatever the server has not acknowledged yet (see
1391
+ * {@link dispatchRecordingTerminal}), and returns without waiting. Never stops the recorder: after
1392
+ * a `hidden` the call may well continue, and later audio simply starts the next shard index (the
1393
+ * tail is sealed, so no index is ever re-sent with different bytes). Also works while End is still
1394
+ * stopping the recorder, which is exactly when a host that does not await End navigates away.
1395
+ * No-op when nothing is recording, there is no session id or no transport, or nothing was
1396
+ * captured. Never throws.
1397
+ *
1398
+ * Deliberately opens NO IndexedDB transaction inside the handler: a `pagehide` handler gets a few
1399
+ * milliseconds, and a store write started there could not complete during unload anyway. The
1400
+ * shards it sealed are written to the store from a `setTimeout(0)` task after the handler returns:
1401
+ * on a page that is merely hidden (the call goes on) that keeps the durable copy until the server
1402
+ * acknowledges it; on a page that is unloading the task just never runs.
1403
+ */
1404
+ SendRecordingTerminal(options?: RecordingTerminalOptions): void;
1405
+ /**
1406
+ * Fires the terminal request, then re-sends with `keepalive` every shard the server has not
1407
+ * acknowledged, then stores the shards sealed for this terminal. Shared by End and the page-lifecycle
1408
+ * handlers. Everything that needs the network is dispatched synchronously, before any storage work,
1409
+ * because the page may close at any moment from here on: an ordinary `fetch` is aborted on unload,
1410
+ * and a `keepalive` one is not.
1411
+ *
1412
+ * - The terminal goes first so it always gets the keepalive budget. The server waits for the shards
1413
+ * the terminal says exist, and assembles as soon as the last missing one arrives.
1414
+ * - The re-sent shards are the queue's unacknowledged ones (on the wire, waiting in a batch, or
1415
+ * retained after a failure) plus `sealedNow`, oldest first, within
1416
+ * {@link recordingKeepaliveBudgetBytes} together with the tail. A `sealedNow` shard that does not
1417
+ * fit, or whose keepalive upload fails, goes through the ordinary queue instead (it may still
1418
+ * land if the page stays). The queue keeps its own shards either way; a duplicate is acked as one.
1419
+ * - `sealedNow` and the tail are stored after the dispatch, from a `setTimeout(0)` task when called
1420
+ * from a page-lifecycle handler (`deferStore`). A shard's stored copy is deleted on its ack, after
1421
+ * the store write, so a fast ack can never leave an orphan row.
1422
+ *
1423
+ * Resolves true when the terminal was acknowledged. Never rejects.
1197
1424
  */
1198
- private flushRecordingSegment;
1425
+ private dispatchRecordingTerminal;
1426
+ /** Counts `bytes` against the page's keepalive budget until `request` settles. */
1427
+ private trackKeepalive;
1199
1428
  /**
1200
- * Sends one shard and reports whether the server stored it. The resolver reports a failed
1201
- * ownership load or storage write as `false`, not as an error — that is how #5197 lost a shard
1202
- * without a single client-side log line — so the typed result must be checked, not just the absence
1203
- * of a throw.
1429
+ * The `keepalive` re-send part of {@link dispatchRecordingTerminal}, budgeted against everything this
1430
+ * page still has in flight (the terminal just sent included). A shard whose `keepalive` copy is already
1431
+ * on the wire is not sent again. Never throws.
1204
1432
  */
1205
- private uploadRecordingShard;
1433
+ private resendUnacknowledgedWithKeepalive;
1206
1434
  /**
1207
- * Stops the active recorder and encodes the captured audio. Split from the upload so teardown can
1208
- * close the server session in between (#5195). Best-effort: returns `null` when nothing was
1209
- * recorded or the stop failed (warned) — it must NEVER block teardown.
1435
+ * Stores shards from a `setTimeout(0)` task rather than now — the path for shards sealed inside a
1436
+ * page-lifecycle handler, which must not open an IndexedDB transaction itself. Resolves once they
1437
+ * are stored (never, if the page unloads first). Never rejects.
1210
1438
  */
1211
- private stopRecording;
1439
+ private storeShardsLater;
1212
1440
  /**
1213
- * Uploads a recording returned by {@link stopRecording}. Never throws. No-op when there is
1214
- * nothing to upload; warns when there is no session id to attach the file to.
1441
+ * Sends the terminal request for a pipeline whose tail is already sealed. Resolves true on a server
1442
+ * ack and logs the outcome with the session id. Never rejects.
1215
1443
  */
1216
- private uploadStoppedRecording;
1444
+ private sendRecordingTerminal;
1217
1445
  /**
1218
- * Runs the `UploadRealtimeRecording` mutation. A `Success:false` payload is logged; a transport
1219
- * rejection propagates to `uploadStoppedRecording`, which catches and logs it. Sends the
1220
- * capture-time waveform `peaks` (max-abs per bucket, normalized 0..1) so the server can persist a
1221
- * `peaks.json` sidecar for fast waveform rendering without re-decoding the audio.
1446
+ * Ends the recording at teardown and STARTS its terminal request: stops the recorder (flushing the
1447
+ * encoder), cuts the remaining packets into shards with the tail trimmed to the recorder's captured
1448
+ * sample count, and dispatches the `keepalive` terminal plus `keepalive` re-sends of everything not
1449
+ * yet acknowledged, before storing anything ({@link dispatchRecordingTerminal}). Teardown awaits the
1450
+ * returned terminal only after `CloseAgentSession` ({@link awaitRecordingTerminal}), so the small,
1451
+ * critical close never waits behind the recording (#5195).
1452
+ *
1453
+ * The recorder and pipeline stay visible to {@link SendRecordingTerminal} until the recorder has
1454
+ * stopped: a `pagehide` in that window (a host that navigates away without awaiting End) still
1455
+ * sends what was captured, and End then continues at the next index.
1456
+ *
1457
+ * Returns null when nothing was recording or nothing was captured (no terminal is sent then).
1458
+ * Best-effort: never throws, never blocks teardown on the network.
1222
1459
  */
1223
- private uploadRecording;
1460
+ private finishRecording;
1461
+ /**
1462
+ * Stops the recorder, bounded by {@link RECORDER_STOP_TIMEOUT_MS}. Resolves the captured sample
1463
+ * count, or null when the recorder threw or did not finish in time (logged) — the caller then ends
1464
+ * the recording at the packets already encoded. Never rejects.
1465
+ */
1466
+ private stopRecorder;
1467
+ /**
1468
+ * Waits for the terminal {@link finishRecording} started, bounded by
1469
+ * {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}, logs the outcome, then closes the recording's queue
1470
+ * (anything it had not delivered stays in the local store for the next-load drain). Never throws.
1471
+ */
1472
+ private awaitRecordingTerminal;
1224
1473
  /** Whether the channel catalog note still has to be sent (see {@link flushChannelCatalogNote}). */
1225
1474
  private catalogNotePending;
1226
1475
  /** Plugins resolved for the current start but not yet initialized. Consumed by {@link activateChannels}. */
@@ -1624,18 +1873,38 @@ export declare class RealtimeSessionRuntime {
1624
1873
  * - NORMAL turns only — NARRATION interims are ephemeral and never persisted, so they must not
1625
1874
  * claim the next real turn's start slot.
1626
1875
  *
1627
- * Works for any role whose driver surfaces interim deltas (all drivers for the assistant; the
1628
- * relevant case here — the post-tool-gap assistant answer — and user-interim drivers like
1629
- * Gemini/AssemblyAI). For final-only user turns (OpenAI/xAI/ElevenLabs) no interim arrives, so
1630
- * {@link relayTranscript} falls back to the seeded/prior start — the gap case that drifts is the
1631
- * assistant answer, which always has interims.
1876
+ * Stamped on a turn's first interim transcript (all drivers for the assistant — the post-tool-gap
1877
+ * assistant answer — and user-interim drivers like Gemini/AssemblyAI). Final-only user turns
1878
+ * (OpenAI-protocol drivers) take their start from {@link markUserSpeechStart}'s own slot
1879
+ * instead; one with no speech-start signal either (ElevenLabs) relays `null` as-is.
1632
1880
  */
1633
1881
  private markTurnAudioStart;
1882
+ /**
1883
+ * Stamps the user speech onset when the client signals that the user began to speak. When the
1884
+ * driver supplies a conversation `itemId`, records the FIRST onset for that item in
1885
+ * {@link userSpeechStartByItemId} so a mid-sentence pause that re-fires with the same id keeps the
1886
+ * original start. When no `itemId` is supplied, stamps the single-slot {@link userSpeechStartMs}
1887
+ * fallback (keeping the first onset unless {@link userSpeechStartSuperseded} is set).
1888
+ */
1889
+ private markUserSpeechStart;
1890
+ /**
1891
+ * Ends the held user onset: a final user transcript (empty or not) closed the utterance it began.
1892
+ * When `itemId` is supplied, deletes only that item's entry in {@link userSpeechStartByItemId};
1893
+ * otherwise clears the unkeyed single slot.
1894
+ */
1895
+ private releaseUserSpeechStart;
1896
+ /**
1897
+ * Consumes and deletes the onset recorded for `itemId` in {@link userSpeechStartByItemId}. When a
1898
+ * matching onset is found, also deletes any remaining map entry with an earlier offset so a VAD
1899
+ * false positive whose transcription never arrived cannot accumulate in the map.
1900
+ */
1901
+ private takeKeyedUserSpeechStart;
1634
1902
  /**
1635
1903
  * The current per-turn offset in ms — the RECORDER's clock when one runs (an offset into a
1636
1904
  * seekable file), else the SESSION clock (#3832: orderable and displayable, not seekable),
1637
1905
  * else `null` before any call is live. One function so the two stamp sites cannot disagree
1638
- * about which clock a session is on.
1906
+ * about which clock a session is on. Always whole milliseconds: the offsets travel as GraphQL
1907
+ * `Int`, which refuses a fraction, and a 48 kHz sample clock rarely lands on a whole millisecond.
1639
1908
  */
1640
1909
  private nowTurnOffsetMs;
1641
1910
  /**
@@ -1644,7 +1913,13 @@ export declare class RealtimeSessionRuntime {
1644
1913
  * client's caption window).
1645
1914
  */
1646
1915
  private replaceLastCaption;
1647
- /** Finalizes the user turn: push a caption + relay the final transcript. */
1916
+ /**
1917
+ * Finalizes the user turn: push a caption + relay the final transcript.
1918
+ *
1919
+ * @param source `'typed'` for a {@link SendText} turn: it has no speech onset, so it neither takes
1920
+ * nor clears the held one (a spoken transcript may still be pending for it).
1921
+ * @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
1922
+ */
1648
1923
  private onUserTranscript;
1649
1924
  /**
1650
1925
  * Routes a provider tool call: names matching a registered client-tool prefix execute
@@ -1738,20 +2013,28 @@ export declare class RealtimeSessionRuntime {
1738
2013
  * Relays a final transcript turn to MJ via `RelayRealtimeTranscript`.
1739
2014
  *
1740
2015
  * When the session is being recorded, per-turn timing rides along: `utteranceEndMs` is the
1741
- * recording-relative offset at finalization, and `utteranceStartMs` is the offset captured by
1742
- * {@link markTurnAudioStart} when THIS turn's audio actually began (its first interim) — NOT
1743
- * inherited from the previous turn's end. That distinction is the timing fix: when a tool-call
1744
- * / silence gap sits between turns, the post-gap turn's audio starts much later, so inheriting
1745
- * the prior turn's end stamped the cue ~gap-length too early. Both are omitted (left `null`)
1746
- * when the session isn't being recorded.
2016
+ * recording-relative offset at finalization, and `utteranceStartMs` is where THIS turn's audio
2017
+ * actually began — NOT inherited from the previous turn's end. For a user turn whose onset the
2018
+ * client signalled, that is {@link userSpeechStartMs}; otherwise it is the offset captured by
2019
+ * {@link markTurnAudioStart} on the turn's first interim. That distinction is the timing fix:
2020
+ * when a tool-call / silence gap sits between turns, the post-gap turn's audio starts much
2021
+ * later, so inheriting the prior turn's end stamped the cue ~gap-length too early. Both are
2022
+ * omitted (left `null`) when the session isn't being recorded. `utteranceStartMs` is also
2023
+ * `null` when no start was measured for this turn; it is never replaced with a placeholder 0.
1747
2024
  *
1748
2025
  * A correction (`replacesPrevious`) doesn't open a new turn, so it carries no start and doesn't
1749
2026
  * reset the per-turn start guard. After a normal finalization the guard is cleared so the NEXT
1750
- * turn re-stamps its start from where ITS audio begins.
2027
+ * turn re-stamps its start from where ITS audio begins — except for a user turn that used its
2028
+ * own speech-start slot, which leaves the shared slot alone (it may hold the in-flight
2029
+ * assistant turn's start). Every final user transcript, corrections included, clears the
2030
+ * speech-start slot.
1751
2031
  *
1752
2032
  * @param replacesPrevious CORRECTION semantics: the server updates the session's most
1753
2033
  * recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
1754
2034
  * post-barge-in `agent_response_correction`).
2035
+ * @param source `'typed'` for a user turn sent with {@link SendText}: it does not read or clear
2036
+ * {@link userSpeechStartMs} or {@link userSpeechStartByItemId}, which belong to the user's speech.
2037
+ * @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
1755
2038
  */
1756
2039
  private relayTranscript;
1757
2040
  /**
@@ -1870,18 +2153,34 @@ export declare class RealtimeSessionRuntime {
1870
2153
  /** Tears down the delegation progress subscription and resets the narration throttle. */
1871
2154
  private teardownDelegationProgress;
1872
2155
  /**
1873
- * Tears down all client resources and (optionally) closes the server session.
1874
- * @param closeServerSession when true, calls `CloseAgentSession` on the server.
2156
+ * Tears down all client resources and closes the server session when one was minted.
2157
+ *
2158
+ * The server row is closed whichever way the call ended: a start that failed after the mint used to
2159
+ * leave its row `Active` until the janitor's staleness sweep, 15 minutes later, so every host that
2160
+ * counts live sessions saw a phantom call (MJ#6013). The janitor remains the fallback for a client
2161
+ * that dies before it can say anything; it is never the plan.
2162
+ *
2163
+ * @param end why the call ended — `'explicit'` when the host or user ended it, `'error'` when the
2164
+ * runtime ended it because the start or the call failed. Reported on `SessionEnded$` and stamped on
2165
+ * the server row as its `CloseReason` (`Explicit` / `Error`).
1875
2166
  */
1876
2167
  private teardown;
1877
2168
  /** The body of {@link teardown}; never called concurrently with itself. */
1878
2169
  private runTeardown;
2170
+ /** The teardown's work: stop the call, hand back what it held, and close the server session. */
2171
+ private releaseCallResources;
2172
+ /** Clears the slot and tells the host the call ended; runs even when {@link releaseCallResources} threw. */
2173
+ private releaseSessionSlot;
1879
2174
  /**
1880
2175
  * Calls the host's {@link IRealtimeMediaHost.ReleaseMicrophone}. Best-effort by contract: a failure
1881
2176
  * is logged and never thrown, because it must never block ending a call.
1882
2177
  */
1883
2178
  private releaseMicrophone;
1884
- /** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
2179
+ /**
2180
+ * Calls the `CloseAgentSession` mutation (provisioned in P4b) with why the client ended the call, which
2181
+ * the server stamps as the row's `CloseReason`. Requires a server that accepts the `closeReason`
2182
+ * argument (MJ#6013); the runtime and the server ship together.
2183
+ */
1885
2184
  private closeServerSession;
1886
2185
  /** Pushes a caption onto the live list (immutable update for change detection). */
1887
2186
  private appendCaption;