@memberjunction/realtime-runtime 6.2.0-edge.5 → 6.2.0-edge.6

package/dist/session/RealtimeSessionRuntime.js

@@ -6,7 +6,7 @@ import { GraphQLRealtimeSessionClient } from '@memberjunction/graphql-dataprovid
 import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
 import { DEFAULT_REALTIME_AUDIO_TRACKS, ParseRealtimeAvatarStatus, ParseRealtimeClientTransport } from '@memberjunction/ai';
 import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, RECORDING_TERMINAL_MAX_BYTES, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
-import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
+import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadElevenRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
 import { DefaultRealtimeSessionLauncher, HasClientCredential } from './session-launcher.js';
 import { RequestsAgentVideo, ResolveAvatarNotice } from './avatar-notice.js';
 import { ClientSessionDeadline } from './client-session-deadline.js';
@@ -72,6 +72,7 @@ LoadGeminiEnterpriseRealtimeClient();
 LoadElevenLabsRealtimeClient();
 LoadAssemblyAIRealtimeClient();
 LoadxAIRealtimeClient();
+LoadElevenRealtimeClient();
 LoadHuggingFaceRealtimeClient();
 /**
  * A registry row's surface placement. A row whose `UIConfig` is not valid JSON places its surface as a row without one
@@ -169,6 +170,14 @@ function microphoneStartError(reason, message) {
     error.name = MICROPHONE_ERROR_NAMES[reason];
     return error;
 }
+/** Thrown by a start while a call is live: the runtime holds one driver, and never replaces a live one. */
+export class RealtimeSessionAlreadyActiveError extends Error {
+    constructor(ActiveAgentSessionId) {
+        super(`A realtime session is already active${ActiveAgentSessionId ? ` (${ActiveAgentSessionId})` : ''}; end it before starting another.`);
+        this.ActiveAgentSessionId = ActiveAgentSessionId;
+        this.name = 'RealtimeSessionAlreadyActiveError';
+    }
+}
 /**
  * Drives a **client-direct** real-time voice session: the browser mints an ephemeral
  * token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
@@ -362,8 +371,8 @@ export class RealtimeSessionRuntime {
          * Fired EXACTLY ONCE per session as teardown begins, with the prior
          * `agentSessionId` (so subscribers can correlate against `SessionStarted$`'s
          * sessionId) and the client-distinguishable reason — `'explicit'` when the
-         * user called `EndRealtimeSession`, `'error'` when teardown ran from a catch
-         * block. Server-side close paths (janitor, shutdown) do NOT propagate here —
+         * user called `EndRealtimeSession`, `'error'` when the runtime ended the call over a failure: a
+         * failed start, or a fatal error from the provider mid-call. Server-side close paths (janitor, shutdown) do NOT propagate here —
          * they happen out-of-process and have no client push channel today.
          */
         this.SessionEnded$ = this._sessionEnded$.asObservable();
@@ -515,6 +524,13 @@ export class RealtimeSessionRuntime {
          * final. An EMPTY user transcript releases the onset outright (see onClientTranscript).
          */
         this.userSpeechStartSuperseded = false;
+        /**
+         * Recording-relative ms offsets of user speech onsets keyed by the provider's conversation item id
+         * (`input_audio_buffer.speech_started.item_id`), paired with the matching final user transcript
+         * (`RealtimeClientTranscript.ItemId`) so a late user transcript never claims a later turn's onset
+         * (MJ#5902). Unkeyed drivers continue to use {@link userSpeechStartMs}.
+         */
+        this.userSpeechStartByItemId = new Map();
         /**
          * Wall-anchor of the SESSION clock (#3832): `performance.now()` at the moment the call went
          * live, or `null` before any call has. Read only through {@link nowTurnOffsetMs}.
@@ -599,6 +615,15 @@ export class RealtimeSessionRuntime {
          * two `SessionEnded$` emissions for one session.
          */
         this.teardownInFlight = null;
+        /**
+         * True from the moment a start finishes (connected, liveness pulse running) until the slot is
+         * released. A driver's Fatal ends the call only while this is set: during a start, drivers emit
+         * Fatal from their socket handlers while Connect is still pending, and the start's own failure path
+         * ({@link failSessionStart}) is what reports that error, whether Connect then rejects or resolves
+         * (runMintedSession checks {@link lastSessionError} after Connect). Not {@link isSessionLive}: drivers move the
+         * state to 'listening' from inside Connect, and a live call that is resuming reads 'connecting'.
+         */
+        this.callWentLive = false;
         /**
          * The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
          * back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
@@ -954,6 +979,25 @@ export class RealtimeSessionRuntime {
     IsActiveFor(conversationId) {
         return this.IsActive && conversationId != null && UUIDsEqual(this.sessionConversationId, conversationId);
     }
+    /**
+     * Takes the runtime's one driver slot for a new start; the two starts call it after awaiting any
+     * teardown in flight, and only when no End arrived during that wait (a changed {@link startGeneration}).
+     *
+     * A teardown still in flight is awaited rather than read as "busy": ending a call and starting the
+     * next one back to back is normal host behaviour (a retry, a re-mounted surface), and the flag it
+     * used to read flips only at the END of teardown, so the start was dropped with nothing to show
+     * for it — a bare `undefined`, which hosts had to guess at (MJ#4629). A call that is genuinely
+     * live is refused with an error the host can show; it is never silently ignored, and never
+     * overwritten, since a driver no slot points at can never be stopped again.
+     *
+     * Deliberately synchronous: the start's state must be set in the tick the host called it in, or an
+     * End issued straight after the Start (a user tapping back during the mint) finds nothing to abandon.
+     */
+    claimSessionSlot() {
+        if (this.IsActive) {
+            throw new RealtimeSessionAlreadyActiveError(this.agentSessionId);
+        }
+    }
     /**
      * Start a client-direct voice session fronting `targetAgentId`.
      *
@@ -1000,13 +1044,22 @@ export class RealtimeSessionRuntime {
      *   ({@link RealtimeSessionStartOptions.ConversationBranchID}; omitted is Main, where its transcript is written).
      */
     async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext, options) {
-        if (this.IsActive) {
-            return; // a session is already running — ignore duplicate starts
+        // Captured BEFORE any wait: a teardown from here on bumps it, and the checks below (and in
+        // runMintedSession) abandon the start. Re-capturing after the wait would absorb an End issued during it.
+        const generation = this.startGeneration;
+        // Synchronous unless a teardown is in flight: a start must take the slot in the same tick it was
+        // called in, so a host's End issued right after it finds a start to abandon (see claimSessionSlot).
+        if (this.teardownInFlight) {
+            await this.teardownInFlight;
+            // An End issued during the wait found IsActive still true (the last call was closing), so it only
+            // joined that teardown and bumped the generation. Nothing is acquired yet, so nothing to release.
+            if (this.startGeneration !== generation) {
+                return;
+            }
         }
+        this.claimSessionSlot();
         const consent = this.beginSessionStart({ agentName, recordingConsent, applicationId, appContext });
         this.sessionConversationId = conversationId ?? null;
-        // A teardown while the channels are prepared or the mint runs bumps this; see the checks below.
-        const generation = this.startGeneration;
         // Captured BEFORE the channels are prepared so the mint carries exactly the snapshot the prologue
         // resolved, whatever a channel plugin may push in the meantime.
         const effectiveAppContext = this._appContext$.value;
@@ -1032,14 +1085,14 @@ export class RealtimeSessionRuntime {
         // The host ended the start while the mint ran. That teardown had no session id to close, so
         // close the minted row here, and touch no shared state: a newer start may already own it.
         if (this.startGeneration !== generation) {
-            await this.closeServerSession(session.AgentSessionId);
+            await this.closeServerSession(session.AgentSessionId, 'Explicit');
             return;
         }
         if (!this.hostCanUseProvider(session.Provider)) {
             await this.abortUnusableSession(session);
             return;
         }
-        await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true);
+        await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true, generation);
     }
     /**
      * Declares which provider keys this host can actually carry audio for.
@@ -1075,8 +1128,7 @@ export class RealtimeSessionRuntime {
         // published them on ActiveChannels$; skipping teardown would leave them undisposed, their tool
         // handlers registered, and their subscriptions live until the next start replaced them.
         this.agentSessionId = session.AgentSessionId ?? this.agentSessionId;
-        this.setConnectionState('error');
-        await this.teardown(true);
+        await this.endOnFailure();
     }
     /**
      * Run a session the HOST has already minted itself — the second half of
@@ -1100,22 +1152,28 @@ export class RealtimeSessionRuntime {
      *   Every field defaults exactly as its {@link StartRealtimeSession} counterpart does.
      */
     async StartRealtimeSessionFromResult(result, options) {
-        if (this.IsActive) {
-            return; // a session is already running — ignore duplicate starts
+        // Same wait, same generation check, same reasons as StartRealtimeSession.
+        const generation = this.startGeneration;
+        if (this.teardownInFlight) {
+            await this.teardownInFlight;
+            if (this.startGeneration !== generation) {
+                return;
+            }
         }
+        this.claimSessionSlot();
         const effectiveOptions = options ?? {};
         if (!this.hostCanUseProvider(result.Provider)) {
             await this.abortUnusableSession(result);
             return;
         }
         const consent = this.beginSessionStart(effectiveOptions);
-        await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false);
+        await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false, generation);
     }
     /**
      * Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
      * publish the agent name, reset per-session state, and flip the session live (which is ALSO what
-     * makes the `IsActive` guard suppress duplicate starts while the mint is still in flight — hence it
-     * runs before minting, not after). Returns the resolved recording consent, which the mint half
+     * makes {@link claimSessionSlot} refuse a second start with {@link RealtimeSessionAlreadyActiveError}
+     * while the mint is still in flight — hence it runs before minting, not after). Returns the resolved recording consent, which the mint half
      * reports to the server and the run half uses to decide whether to record.
      */
     beginSessionStart(options) {
@@ -1155,10 +1213,10 @@ export class RealtimeSessionRuntime {
      * the provider connection, and go live. `inputConversationId` is the conversation the START asked
      * for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
      * whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
+     * `generation` is the {@link startGeneration} the entry point captured before its first await; every
+     * await below is a window in which the host can end the session, and so was every await before this.
      */
-    async runMintedSession(session, inputConversationId, consent, cameraCheck) {
-        // Captured up front: every await below is a window in which the host can end the session.
-        const generation = this.startGeneration;
+    async runMintedSession(session, inputConversationId, consent, cameraCheck, generation) {
         try {
             // Mount the channels the resolved policy puts in the session. Deliberately BEFORE the session
             // id is adopted: `ActiveChannels$` consumers (the sessions adapter) treat an emission that
@@ -1207,6 +1265,13 @@ export class RealtimeSessionRuntime {
             if (connectError) {
                 throw connectError;
             }
+            // A Fatal the driver sent during this start (the fatal handler leaves it to the start, see
+            // callWentLive) fails the start even when Connect resolved: the real drivers keep awaiting mic
+            // capture after their connect-failure channel settles, then report 'listening' and resolve on a
+            // socket that already closed. lastSessionError is reset per start and set only by the current client.
+            if (this.lastSessionError?.Fatal) {
+                throw new Error(this.lastSessionError.Message);
+            }
             // Tracks are negotiated now, so a capture can tell whether the model takes video, and the call whether it shows
             // the avatar its agent asked for. The driver holds the microphone now, so the user may switch it.
             this.openCaptures(client, cameraCheck);
@@ -1258,6 +1323,7 @@ export class RealtimeSessionRuntime {
             // Same place, same reason: the session is connected and its id is known, which is exactly
             // the window in which the server needs to be told it is alive.
             this.startLivenessPulse();
+            this.callWentLive = true;
         }
         catch (error) {
             await this.failSessionStart(error);
@@ -1319,7 +1385,7 @@ export class RealtimeSessionRuntime {
         // session, and clearing the id would leave that teardown no session to report.
         if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId && !this.teardownInFlight) {
             this.agentSessionId = null;
-            await this.closeServerSession(session.AgentSessionId);
+            await this.closeServerSession(session.AgentSessionId, 'Explicit');
         }
     }
     /**
@@ -1630,8 +1696,20 @@ export class RealtimeSessionRuntime {
     async failSessionStart(error) {
         console.error('[RealtimeSession] Failed to start session:', error);
         this.lastStartError = error instanceof Error ? error : new Error(String(error));
+        await this.endOnFailure();
+    }
+    /**
+     * Ends the call over a failure: a failed start, an unusable provider, or a fatal provider error.
+     *
+     * The teardown is claimed BEFORE `'error'` is published. `ConnectionState$` emits synchronously, and a
+     * host may end the call the moment it sees `'error'` (MobileApp's voice screen does); claimed first, that
+     * host's End joins this teardown instead of starting its own as `'explicit'`, so the row still closes as
+     * `Error` and `SessionEnded$` still says `'error'`. Teardown keeps a terminal `'error'` state.
+     */
+    endOnFailure() {
+        const ending = this.teardown('error');
         this.setConnectionState('error');
-        await this.teardown(false);
+        return ending;
     }
     /**
      * End the active session: stop the mic, tear down the provider connection, and close
@@ -1641,7 +1719,7 @@ export class RealtimeSessionRuntime {
         if (!this.IsActive && !this.agentSessionId) {
             return;
         }
-        await this.teardown(true);
+        await this.teardown('explicit');
     }
     /**
      * Inject a typed message into the live session as a user turn.
@@ -1732,7 +1810,17 @@ export class RealtimeSessionRuntime {
         if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
             return false;
         }
-        this.requestChannelSpokenResponse(trimmed);
+        // The opening is part of the conversation, so it goes out as a spoken TURN: captioned, relayed
+        // and persisted like any turn the model takes on its own. It used to ride the delegation
+        // narration channel, whose replies are disposable by contract, so the greeting the participant
+        // heard was never in the transcript (MJ#4591). A driver from before the capability existed
+        // falls back to that channel rather than leaving the session silent.
+        if (this.client.RequestSpokenTurn) {
+            this.client.RequestSpokenTurn(trimmed);
+        }
+        else {
+            this.client.RequestSpokenUpdate(trimmed);
+        }
         return true;
     }
     /**
@@ -1814,6 +1902,7 @@ export class RealtimeSessionRuntime {
             this.turnAudioStartCaptured = false;
             this.userSpeechStartMs = null;
             this.userSpeechStartSuperseded = false;
+            this.userSpeechStartByItemId.clear();
             if (this.recorder) {
                 // WebRTC drivers (OpenAI): the agent's track usually lands AFTER Connect() resolves, so
                 // `remoteStream` above is null here and this handler attaches it later. PCM-playback
@@ -1948,17 +2037,19 @@ export class RealtimeSessionRuntime {
             // Closes over this recording's session id and transport — never the runtime's current ones.
             Queue: new RecordingShardQueue(async (shard) => {
                 if (!transport) {
-                    return false;
+                    return 'Retry';
                 }
-                const acked = (await transport.UploadShard(sessionId, shard)) === 'Acked';
-                if (acked) {
+                // The outcome goes to the queue unchanged: it logs a different reason for each, and stops
+                // retrying a `Refused` shard at once (#6012).
+                const outcome = await transport.UploadShard(sessionId, shard);
+                if (outcome === 'Acked') {
                     await this.deleteStoredShard(sessionId, shard.Index);
                     if (shardsGivenUp) {
                         shardsGivenUp = false;
                         this.recoverStoredShardsAfterOutage(sessionId);
                     }
                 }
-                return acked;
+                return outcome;
             }, () => {
                 shardsGivenUp = true;
             }),
@@ -2292,6 +2383,7 @@ export class RealtimeSessionRuntime {
         this.turnAudioStartCaptured = false;
         this.userSpeechStartMs = null;
         this.userSpeechStartSuperseded = false;
+        this.userSpeechStartByItemId.clear();
         if (!recorder || !pipeline) {
             this.recorder = null;
             this.recordingShards = null;
@@ -3384,6 +3476,27 @@ export class RealtimeSessionRuntime {
             console.error('[RealtimeSession] Provider error event:', JSON.stringify(error), error);
             if (error.Fatal && this.client === client) {
                 this.lastSessionError = error;
+                // While the start is still running, its own failure path reports the error and ends the call
+                // (runMintedSession throws on this recorded Fatal after Connect, even if Connect resolves):
+                // ending it here would bump startGeneration, so the start would read as ended by the host and
+                // never set LastStartError.
+                if (!this.callWentLive) {
+                    return;
+                }
+                // An explicit teardown is already ending the call and the client stays current until its
+                // finally: a driver reporting its own close as Fatal from inside Disconnect() is part of that
+                // clean End, and must not flip it to 'error' or start a second teardown.
+                if (this.teardownInFlight) {
+                    return;
+                }
+                // The driver is dead on the wire; nothing resumes after it says Fatal. End the call the way a
+                // failed start ends: the row closes as Error, the microphone is released, the host is told
+                // through SessionEnded$ (MJ#4629). A host's own End afterwards finds nothing to do.
+                // Captured now: teardown nulls agentSessionId, and the failure it reports must name the session.
+                const failedSessionId = this.agentSessionId;
+                void this.endOnFailure().catch((teardownError) => {
+                    console.error(`[RealtimeSession] Teardown after a fatal provider error failed for session ${failedSessionId}:`, teardownError);
+                });
             }
         });
         // Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
@@ -3403,9 +3516,10 @@ export class RealtimeSessionRuntime {
         });
         // The provider VAD saw the user start speaking. Final-only user-transcript drivers (OpenAI,
         // HuggingFace, xAI) emit no user interim, so this is their only real user-turn start. It goes
-        // to its own slot, not the shared turn slot: the user transcript arrives asynchronously, so the
-        // assistant's reply may already be streaming and must keep its own start.
-        client.OnUserSpeechStart(() => this.markUserSpeechStart());
+        // to its own slot (or item-keyed map entry when `itemId` is present), not the shared turn slot:
+        // the user transcript arrives asynchronously, so the assistant's reply may already be streaming
+        // and must keep its own start.
+        client.OnUserSpeechStart((itemId) => this.markUserSpeechStart(itemId));
         client.OnRemoteVideo((video) => this.onAgentVideo(client, video));
     }
     /**
@@ -3511,7 +3625,7 @@ export class RealtimeSessionRuntime {
         if (transcript.Role === 'User' && transcript.Text.trim().length === 0) {
             // An empty final user transcript (a VAD false positive, an empty transcription) never reaches
             // relayTranscript, so release its onset here: it must not become the next real turn's start.
-            this.releaseUserSpeechStart();
+            this.releaseUserSpeechStart(transcript.ItemId);
         }
         if (transcript.Role === 'Assistant') {
             this.hasActiveInterimUserCaption = false;
@@ -3555,7 +3669,7 @@ export class RealtimeSessionRuntime {
             if (this.firstUserTranscript === null) {
                 this.firstUserTranscript = transcript.Text;
             }
-            await this.relayTranscript('user', transcript.Text);
+            await this.relayTranscript('user', transcript.Text, false, 'speech', transcript.ItemId);
         }
         else if (transcript.ReplacesPrevious) {
             // STREAMING user transcription: providers like Grok and OpenAI Live emit the growing utterance as repeated
@@ -3566,10 +3680,10 @@ export class RealtimeSessionRuntime {
                 return;
             }
             this.replaceLastCaption('User', transcript.Text);
-            await this.relayTranscript('user', transcript.Text, true);
+            await this.relayTranscript('user', transcript.Text, true, 'speech', transcript.ItemId);
         }
         else {
-            await this.onUserTranscript(transcript.Text);
+            await this.onUserTranscript(transcript.Text, 'speech', transcript.ItemId);
         }
     }
     /**
@@ -3604,13 +3718,24 @@ export class RealtimeSessionRuntime {
         this.turnAudioStartCaptured = true;
     }
     /**
-     * Stamps {@link userSpeechStartMs} when the client signals that the user began to speak. Keeps
-     * the FIRST onset since the last final user transcript: the VAD fires again after a
-     * mid-sentence pause, and that second onset is not where the utterance began. An onset that an
-     * assistant turn which began after it has since superseded ({@link userSpeechStartSuperseded}) is
-     * replaced instead.
+     * Stamps the user speech onset when the client signals that the user began to speak. When the
+     * driver supplies a conversation `itemId`, records the FIRST onset for that item in
+     * {@link userSpeechStartByItemId} so a mid-sentence pause that re-fires with the same id keeps the
+     * original start. When no `itemId` is supplied, stamps the single-slot {@link userSpeechStartMs}
+     * fallback (keeping the first onset unless {@link userSpeechStartSuperseded} is set).
      */
-    markUserSpeechStart() {
+    markUserSpeechStart(itemId) {
+        if (itemId !== undefined && itemId.length > 0) {
+            if (this.userSpeechStartByItemId.has(itemId)) {
+                return;
+            }
+            const offset = this.nowTurnOffsetMs();
+            if (offset === null) {
+                return;
+            }
+            this.userSpeechStartByItemId.set(itemId, offset);
+            return;
+        }
         if (this.userSpeechStartMs !== null && !this.userSpeechStartSuperseded) {
             return;
         }
@@ -3621,11 +3746,36 @@ export class RealtimeSessionRuntime {
         this.userSpeechStartMs = offset;
         this.userSpeechStartSuperseded = false;
     }
-    /** Ends the held user onset: a final user transcript (empty or not) closed the utterance it began. */
-    releaseUserSpeechStart() {
+    /**
+     * Ends the held user onset: a final user transcript (empty or not) closed the utterance it began.
+     * When `itemId` is supplied, deletes only that item's entry in {@link userSpeechStartByItemId};
+     * otherwise clears the unkeyed single slot.
+     */
+    releaseUserSpeechStart(itemId) {
+        if (itemId !== undefined && itemId.length > 0) {
+            this.userSpeechStartByItemId.delete(itemId);
+            return;
+        }
         this.userSpeechStartMs = null;
         this.userSpeechStartSuperseded = false;
     }
+    /**
+     * Consumes and deletes the onset recorded for `itemId` in {@link userSpeechStartByItemId}. When a
+     * matching onset is found, also deletes any remaining map entry with an earlier offset so a VAD
+     * false positive whose transcription never arrived cannot accumulate in the map.
+     */
+    takeKeyedUserSpeechStart(itemId) {
+        const onset = this.userSpeechStartByItemId.get(itemId) ?? null;
+        this.userSpeechStartByItemId.delete(itemId);
+        if (onset !== null) {
+            for (const [otherId, otherOnset] of this.userSpeechStartByItemId) {
+                if (otherOnset < onset) {
+                    this.userSpeechStartByItemId.delete(otherId);
+                }
+            }
+        }
+        return onset;
+    }
     /**
      * The current per-turn offset in ms — the RECORDER's clock when one runs (an offset into a
      * seekable file), else the SESSION clock (#3832: orderable and displayable, not seekable),
@@ -3664,8 +3814,9 @@ export class RealtimeSessionRuntime {
      *
      * @param source `'typed'` for a {@link SendText} turn: it has no speech onset, so it neither takes
      *   nor clears the held one (a spoken transcript may still be pending for it).
+     * @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
      */
-    async onUserTranscript(transcript, source = 'speech') {
+    async onUserTranscript(transcript, source = 'speech', itemId) {
         if (transcript.trim().length === 0) {
             return;
         }
@@ -3674,7 +3825,7 @@ export class RealtimeSessionRuntime {
             this.firstUserTranscript = transcript;
         }
         this.appendCaption({ Role: 'User', Text: transcript });
-        await this.relayTranscript('user', transcript, false, source);
+        await this.relayTranscript('user', transcript, false, source, itemId);
     }
     // ── Tool calling ───────────────────────────────────────────────────────────
     /**
@@ -4022,16 +4173,23 @@ export class RealtimeSessionRuntime {
      *   recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
      *   post-barge-in `agent_response_correction`).
      * @param source `'typed'` for a user turn sent with {@link SendText}: it does not read or clear
-     *   {@link userSpeechStartMs}, which belongs to the user's speech.
+     *   {@link userSpeechStartMs} or {@link userSpeechStartByItemId}, which belong to the user's speech.
+     * @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
      */
-    async relayTranscript(role, text, replacesPrevious = false, source = 'speech') {
+    async relayTranscript(role, text, replacesPrevious = false, source = 'speech', itemId) {
         if (!this.agentSessionId) {
             return;
         }
         // Every final user transcript closes the signalled onset — a correction too, since it ends the
         // same utterance — so a later onset (an xAI mid-sentence resume) can't leak into the next turn.
+        // When the transcript carries a provider itemId, consume its keyed onset from the map without
+        // touching the unkeyed single slot or falling back to currentTurnStartMs.
+        const hasItemId = role === 'user' && source === 'speech' && itemId !== undefined && itemId.length > 0;
         let userSpeechStartMs = null;
-        if (role === 'user' && source === 'speech') {
+        if (hasItemId) {
+            userSpeechStartMs = this.takeKeyedUserSpeechStart(itemId);
+        }
+        else if (role === 'user' && source === 'speech') {
             userSpeechStartMs = this.userSpeechStartMs;
             this.releaseUserSpeechStart();
         }
@@ -4042,9 +4200,10 @@ export class RealtimeSessionRuntime {
         const utteranceEndMs = this.nowTurnOffsetMs();
         const opensTurn = utteranceEndMs !== null && !replacesPrevious;
         let utteranceStartMs = null;
-        if (opensTurn && userSpeechStartMs !== null) {
-            // A user turn with a signalled onset. Leave the shared slot alone: the user transcript
-            // arrives asynchronously, so it may already hold the in-flight assistant reply's start.
+        if (opensTurn && (hasItemId || userSpeechStartMs !== null)) {
+            // A user turn with a signalled onset (or a keyed user turn whose itemId was not in the map,
+            // which relays null). Leave the shared slot alone: the user transcript arrives asynchronously,
+            // so it may already hold the in-flight assistant reply's start.
             utteranceStartMs = userSpeechStartMs;
             if (!this.turnAudioStartCaptured) {
                 // No turn has captured a start, so the slot holds only the recording-start seed (0). This
@@ -4460,10 +4619,18 @@ export class RealtimeSessionRuntime {
     }
     // ── Teardown ───────────────────────────────────────────────────────────────
     /**
-     * Tears down all client resources and (optionally) closes the server session.
-     * @param closeServerSession when true, calls `CloseAgentSession` on the server.
+     * Tears down all client resources and closes the server session when one was minted.
+     *
+     * The server row is closed whichever way the call ended: a start that failed after the mint used to
+     * leave its row `Active` until the janitor's staleness sweep, 15 minutes later, so every host that
+     * counts live sessions saw a phantom call (MJ#6013). The janitor remains the fallback for a client
+     * that dies before it can say anything; it is never the plan.
+     *
+     * @param end why the call ended — `'explicit'` when the host or user ended it, `'error'` when the
+     *   runtime ended it because the start or the call failed. Reported on `SessionEnded$` and stamped on
+     *   the server row as its `CloseReason` (`Explicit` / `Error`).
      */
-    async teardown(closeServerSession) {
+    async teardown(end) {
         // Invalidate any start still in flight BEFORE anything else, so it abandons itself at its next
         // await rather than opening a microphone behind a session that is being ended.
         this.startGeneration++;
@@ -4473,7 +4640,7 @@ export class RealtimeSessionRuntime {
             await this.teardownInFlight;
             return;
         }
-        this.teardownInFlight = this.runTeardown(closeServerSession).finally(() => {
+        this.teardownInFlight = this.runTeardown(end).finally(() => {
             if (this._savingRecording$.value) {
                 this._savingRecording$.next(false);
             }
@@ -4482,11 +4649,22 @@ export class RealtimeSessionRuntime {
         await this.teardownInFlight;
     }
     /** The body of {@link teardown}; never called concurrently with itself. */
-    async runTeardown(closeServerSession) {
+    async runTeardown(end) {
         // Arm BEFORE any await so a tab closed in the first milliseconds of teardown is still guarded.
         if (this.recorder) {
             this._savingRecording$.next(true);
         }
+        // Whatever a step below does, the slot is released: a teardown that threw halfway used to leave
+        // `IsActive` set for good, so every later start was refused (MJ#4629).
+        try {
+            await this.releaseCallResources(end);
+        }
+        finally {
+            this.releaseSessionSlot(end);
+        }
+    }
+    /** The teardown's work: stop the call, hand back what it held, and close the server session. */
+    async releaseCallResources(end) {
         // First: stop asserting liveness. A pulse racing the close would re-stamp LastActiveAt on a
         // session we are deliberately ending, leaving an Idle row the janitor then has to age out.
         this.stopLivenessPulse();
@@ -4546,8 +4724,8 @@ export class RealtimeSessionRuntime {
         }
         await this.flushPendingUsage(this.agentSessionId);
         this.resetUsageRelay();
-        if (closeServerSession && this.agentSessionId) {
-            await this.closeServerSession(this.agentSessionId);
+        if (this.agentSessionId) {
+            await this.closeServerSession(this.agentSessionId, end === 'error' ? 'Error' : 'Explicit');
         }
         // Wait for the terminal's ack only AFTER the close: nothing after an aborted await ever runs, so
         // the small, critical close must never queue behind the recording (#5195). Bounded by
@@ -4555,11 +4733,15 @@ export class RealtimeSessionRuntime {
         if (recordingTerminal) {
             await this.awaitRecordingTerminal(recordingTerminal);
         }
+    }
+    /** Clears the slot and tells the host the call ended; runs even when {@link releaseCallResources} threw. */
+    releaseSessionSlot(end) {
         // Capture the session id BEFORE we null it so the lifecycle emit carries it.
         // Skip emitting when there was no live session (defensive — teardown is safe
         // to call without an active session).
         const closedSessionId = this.agentSessionId;
         this.agentSessionId = null;
+        this.callWentLive = false;
         this.sessionConversationId = null;
         this.narrationTemplate = null;
         this.clientToolHandlers.clear();
@@ -4572,14 +4754,9 @@ export class RealtimeSessionRuntime {
         if (this._connectionState$.value !== 'error') {
             this.setConnectionState('closed');
         }
-        // Surface generic session-ended for the conversations runtime bridge.
-        // `closeServerSession=true` means the user explicitly called EndRealtimeSession;
-        // `false` means teardown ran from a catch block (start path error path).
+        // Surface generic session-ended for the conversations runtime bridge, with why it ended.
         if (closedSessionId) {
-            this._sessionEnded$.next({
-                sessionId: closedSessionId,
-                reason: closeServerSession ? 'explicit' : 'error',
-            });
+            this._sessionEnded$.next({ sessionId: closedSessionId, reason: end });
         }
     }
     /**
@@ -4594,18 +4771,22 @@ export class RealtimeSessionRuntime {
             console.error('[RealtimeSession] Media host failed to release the microphone:', error);
         }
     }
-    /** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
-    async closeServerSession(agentSessionId) {
+    /**
+     * Calls the `CloseAgentSession` mutation (provisioned in P4b) with why the client ended the call, which
+     * the server stamps as the row's `CloseReason`. Requires a server that accepts the `closeReason`
+     * argument (MJ#6013); the runtime and the server ship together.
+     */
+    async closeServerSession(agentSessionId, closeReason) {
         try {
             const mutation = `
-        mutation CloseAgentSession($agentSessionId: String!) {
-          CloseAgentSession(agentSessionId: $agentSessionId)
+        mutation CloseAgentSession($agentSessionId: String!, $closeReason: String) {
+          CloseAgentSession(agentSessionId: $agentSessionId, closeReason: $closeReason)
         }
       `;
-            await this.gql().ExecuteGQL(mutation, { agentSessionId });
+            await this.gql().ExecuteGQL(mutation, { agentSessionId, closeReason });
         }
         catch (error) {
-            console.error('[RealtimeSession] Failed to close server session:', error);
+            console.error(`[RealtimeSession] Failed to close server session ${agentSessionId} (closeReason ${closeReason}):`, error);
         }
     }
     // ── Helpers ────────────────────────────────────────────────────────────────
@@ -4629,6 +4810,7 @@ export class RealtimeSessionRuntime {
         this.turnAudioStartCaptured = false;
         this.userSpeechStartMs = null;
         this.userSpeechStartSuperseded = false;
+        this.userSpeechStartByItemId.clear();
         this.usedChannelNames.clear();
         this.microphoneMuted = false;
     }