@memberjunction/realtime-runtime 6.2.0-edge.5 → 6.2.0-edge.7

package/dist/session/RealtimeSessionRuntime.js

@@ -6,7 +6,7 @@ import { GraphQLRealtimeSessionClient } from '@memberjunction/graphql-dataprovid
 import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
 import { DEFAULT_REALTIME_AUDIO_TRACKS, ParseRealtimeAvatarStatus, ParseRealtimeClientTransport } from '@memberjunction/ai';
 import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, RECORDING_TERMINAL_MAX_BYTES, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
-import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
+import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadElevenRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
 import { DefaultRealtimeSessionLauncher, HasClientCredential } from './session-launcher.js';
 import { RequestsAgentVideo, ResolveAvatarNotice } from './avatar-notice.js';
 import { ClientSessionDeadline } from './client-session-deadline.js';
@@ -72,6 +72,7 @@ LoadGeminiEnterpriseRealtimeClient();
 LoadElevenLabsRealtimeClient();
 LoadAssemblyAIRealtimeClient();
 LoadxAIRealtimeClient();
+LoadElevenRealtimeClient();
 LoadHuggingFaceRealtimeClient();
 /**
  * A registry row's surface placement. A row whose `UIConfig` is not valid JSON places its surface as a row without one
@@ -169,6 +170,20 @@ function microphoneStartError(reason, message) {
     error.name = MICROPHONE_ERROR_NAMES[reason];
     return error;
 }
+/** The one mapping from how a call ended to the `CloseReason` the client claims for its row; a new end reason is a compile error here. */
+const CLOSE_REASON_BY_END = {
+    explicit: 'Explicit',
+    error: 'Error',
+    deadline: 'TimeLimit',
+};
+/** Thrown by a start while a call is live: the runtime holds one driver, and never replaces a live one. */
+export class RealtimeSessionAlreadyActiveError extends Error {
+    constructor(ActiveAgentSessionId) {
+        super(`A realtime session is already active${ActiveAgentSessionId ? ` (${ActiveAgentSessionId})` : ''}; end it before starting another.`);
+        this.ActiveAgentSessionId = ActiveAgentSessionId;
+        this.name = 'RealtimeSessionAlreadyActiveError';
+    }
+}
 /**
  * Drives a **client-direct** real-time voice session: the browser mints an ephemeral
  * token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
@@ -362,8 +377,9 @@ export class RealtimeSessionRuntime {
          * Fired EXACTLY ONCE per session as teardown begins, with the prior
          * `agentSessionId` (so subscribers can correlate against `SessionStarted$`'s
          * sessionId) and the client-distinguishable reason — `'explicit'` when the
-         * user called `EndRealtimeSession`, `'error'` when teardown ran from a catch
-         * block. Server-side close paths (janitor, shutdown) do NOT propagate here —
+         * user called `EndRealtimeSession`, `'error'` when the runtime ended the call over a failure: a
+         * failed start, or a fatal error from the provider mid-call; `'deadline'` when the session deadline ended it
+         * (the runtime's auto-end, or `EndRealtimeSession('deadline')`). Server-side close paths (janitor, shutdown) do NOT propagate here —
          * they happen out-of-process and have no client push channel today.
          */
         this.SessionEnded$ = this._sessionEnded$.asObservable();
@@ -515,6 +531,13 @@ export class RealtimeSessionRuntime {
          * final. An EMPTY user transcript releases the onset outright (see onClientTranscript).
          */
         this.userSpeechStartSuperseded = false;
+        /**
+         * Recording-relative ms offsets of user speech onsets keyed by the provider's conversation item id
+         * (`input_audio_buffer.speech_started.item_id`), paired with the matching final user transcript
+         * (`RealtimeClientTranscript.ItemId`) so a late user transcript never claims a later turn's onset
+         * (MJ#5902). Unkeyed drivers continue to use {@link userSpeechStartMs}.
+         */
+        this.userSpeechStartByItemId = new Map();
         /**
          * Wall-anchor of the SESSION clock (#3832): `performance.now()` at the moment the call went
          * live, or `null` before any call has. Read only through {@link nowTurnOffsetMs}.
@@ -599,6 +622,15 @@ export class RealtimeSessionRuntime {
          * two `SessionEnded$` emissions for one session.
          */
         this.teardownInFlight = null;
+        /**
+         * True from the moment a start finishes (connected, liveness pulse running) until the slot is
+         * released. A driver's Fatal ends the call only while this is set: during a start, drivers emit
+         * Fatal from their socket handlers while Connect is still pending, and the start's own failure path
+         * ({@link failSessionStart}) is what reports that error, whether Connect then rejects or resolves
+         * (runMintedSession checks {@link lastSessionError} after Connect). Not {@link isSessionLive}: drivers move the
+         * state to 'listening' from inside Connect, and a live call that is resuming reads 'connecting'.
+         */
+        this.callWentLive = false;
         /**
          * The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
          * back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
@@ -676,6 +708,26 @@ export class RealtimeSessionRuntime {
          */
         this.IdentityVerifiedSpokenResponse = null;
         this.clientDeadline = new ClientSessionDeadline();
+        /**
+         * The deadline the clock reached while no session id existed yet (the host set a deadline that passed, or one nearer
+         * than the mint takes), or `null`. The mint path consumes it as soon as the session id is adopted, so the call still
+         * ends at its deadline; Set, Extend and session teardown drop it, since each replaces or forgets that deadline.
+         */
+        this.pendingDeadlineReached = null;
+        this._sessionDeadlineWarning$ = new Subject();
+        this._sessionDeadlineReached$ = new Subject();
+        /**
+         * Fires once per offset the host passed in `warnBeforeMs` to {@link SetSessionDeadline}, as that moment arrives, with
+         * the live session's id. `remainingMs` is the offset. Emits only while a session is live.
+         */
+        this.SessionDeadlineWarning$ = this._sessionDeadlineWarning$.asObservable();
+        /**
+         * Fires once when the session deadline is reached, with the live session's id — BEFORE the runtime's own auto-end, so
+         * a host that ends the call here (`autoEnd: false`, or a goodbye of its own) is the end that counts; a second end is
+         * coalesced onto the first. Emits only while a session is live; a deadline reached while the session is still being
+         * minted fires (and auto-ends) as soon as the mint adopts the session id.
+         */
+        this.SessionDeadlineReached$ = this._sessionDeadlineReached$.asObservable();
         // ── Interactive channels (registry-driven + host-declared plugins) ─────────
         //
         // A session's channels come from two places — the `MJ: AI Agent Channels` registry and the channels
@@ -721,6 +773,7 @@ export class RealtimeSessionRuntime {
          * Owner-keyed — see {@link AppClientToolRegistry}.
          */
         this.appToolRegistry = new AppClientToolRegistry();
+        this.wireSessionDeadline();
         // Re-upload shards a previous, interrupted call left in the local store. Deferred a tick so a
         // host can assign `Provider` right after construction; when it still has no transport the drain
         // defers itself and session start tries again. Unref'd so it never holds a Node process open.
@@ -873,9 +926,83 @@ export class RealtimeSessionRuntime {
     get SessionDeadline() {
         return this.clientDeadline.Value;
     }
-    /** Sets the baseline deadline a host knows at session start (`null` clears). See {@link SessionDeadline$}. */
-    SetSessionDeadline(deadline) {
-        this.clientDeadline.Set(deadline);
+    /**
+     * Sets the baseline deadline a host knows at session start (`null` clears) and arms the alarm. See
+     * {@link SessionDeadline$}. By default the runtime ends the call with reason `'deadline'` when it is reached; pass
+     * `{ autoEnd: false }` to own the hang-up (then end with `EndRealtimeSession('deadline')`, which claims `TimeLimit`: the
+     * server keeps it only against the session's stored deadline, so a deadline set here alone records `Explicit`), and
+     * `warnBeforeMs` for {@link SessionDeadlineWarning$}. The server's verified deadline (`identity.verified`) also arms the
+     * alarm with the current options (default `autoEnd: true`); a host that owns the hang-up calls `SetSessionDeadline(d, { autoEnd: false })`.
+     * The runtime never arms this by itself.
+     */
+    SetSessionDeadline(deadline, options) {
+        this.pendingDeadlineReached = null;
+        this.clientDeadline.Set(deadline, options);
+    }
+    /**
+     * Re-publishes the deadline clock's warnings and reach with the live session id, and runs the auto-end. Subscribed
+     * once, from the constructor; the clock is owned by this runtime and shares its lifetime, and the runtime has no
+     * dispose step, so the subscriptions are never torn down separately. Session teardown clears the clock, which
+     * cancels every pending timer.
+     */
+    wireSessionDeadline() {
+        this.clientDeadline.Warning$.subscribe((warning) => {
+            const sessionId = this.liveDeadlineSessionId();
+            if (sessionId) {
+                this._sessionDeadlineWarning$.next({ sessionId, deadline: warning.deadline, remainingMs: warning.remainingMs });
+            }
+        });
+        this.clientDeadline.Reached$.subscribe((reached) => this.onSessionDeadlineReached(reached.deadline));
+    }
+    /**
+     * Tells the host first, then auto-ends unless the host opted out, already ended the call, or moved the deadline from
+     * inside the event. Reached before the call went live (before the mint, or while the microphone and connect are still
+     * in flight) and with no teardown running, the reach is kept in {@link pendingDeadlineReached} for the start to
+     * replay once `SessionStarted$` has fired, so a host never hears of a deadline for a call it was not told started.
+     */
+    onSessionDeadlineReached(deadline) {
+        const sessionId = this.liveDeadlineSessionId();
+        if (!sessionId) {
+            if (!this.teardownInFlight && this.holdsDeadline(deadline)) {
+                this.pendingDeadlineReached = deadline;
+            }
+            return;
+        }
+        this._sessionDeadlineReached$.next({ sessionId, deadline });
+        if (this.clientDeadline.Options.autoEnd && this.liveDeadlineSessionId() === sessionId && this.holdsDeadline(deadline)) {
+            this.teardown('deadline').catch((error) => {
+                console.error(`[RealtimeSession] Ending session ${sessionId} at its deadline failed:`, error);
+            });
+        }
+    }
+    /** Whether the clock still holds `deadline`: a host that set or extended it (even from inside `Reached$`) has replaced it. */
+    holdsDeadline(deadline) {
+        return this.clientDeadline.Value?.getTime() === deadline.getTime();
+    }
+    /**
+     * Replays a deadline reached before the call went live, once it is live. On the next tick, so the host's
+     * `SessionDeadlineReached$` handler runs outside the start, after the start has finished.
+     */
+    replayPendingDeadlineReached() {
+        const pending = this.pendingDeadlineReached;
+        if (!pending) {
+            return;
+        }
+        this.pendingDeadlineReached = null;
+        setTimeout(() => {
+            if (this.holdsDeadline(pending)) {
+                this.onSessionDeadlineReached(pending);
+            }
+        }, 0);
+    }
+    /**
+     * The session a deadline event belongs to: the session once its call went live, unless it is already being torn
+     * down. A start still opening the microphone or connecting has none: a host that owns the hang-up must be able to
+     * speak its goodbye when it hears the deadline. {@link callWentLive}, not {@link isSessionLive}: a call that is
+     * resuming reads `'connecting'`, and its deadline still holds. Warnings due before the call went live are dropped.
+     */
+    liveDeadlineSessionId() {
+        return this.teardownInFlight || !this.callWentLive ? null : this.agentSessionId;
     }
     /**
      * Builds the transport that reads the session's events. The default rides the session's GraphQL
@@ -935,7 +1062,9 @@ export class RealtimeSessionRuntime {
             return;
         }
         const payload = event.Payload;
-        this.clientDeadline.Extend(payload.MaxSessionDeadlineIso);
+        if (this.clientDeadline.Extend(payload.MaxSessionDeadlineIso)) {
+            this.pendingDeadlineReached = null;
+        }
         this.SendContextNote(`[identity] verified ${JSON.stringify({ email: payload.VerifiedEmail, name: payload.VerifiedName, method: payload.Method })} ` +
             '(background context — the user has proven they control this email address; treat them as verified. Do not read this note aloud.)');
         const spoken = typeof this.IdentityVerifiedSpokenResponse === 'function' ? this.IdentityVerifiedSpokenResponse(payload) : this.IdentityVerifiedSpokenResponse;
@@ -954,6 +1083,25 @@ export class RealtimeSessionRuntime {
     IsActiveFor(conversationId) {
         return this.IsActive && conversationId != null && UUIDsEqual(this.sessionConversationId, conversationId);
     }
+    /**
+     * Takes the runtime's one driver slot for a new start; the two starts call it after awaiting any
+     * teardown in flight, and only when no End arrived during that wait (a changed {@link startGeneration}).
+     *
+     * A teardown still in flight is awaited rather than read as "busy": ending a call and starting the
+     * next one back to back is normal host behaviour (a retry, a re-mounted surface), and the flag it
+     * used to read flips only at the END of teardown, so the start was dropped with nothing to show
+     * for it — a bare `undefined`, which hosts had to guess at (MJ#4629). A call that is genuinely
+     * live is refused with an error the host can show; it is never silently ignored, and never
+     * overwritten, since a driver no slot points at can never be stopped again.
+     *
+     * Deliberately synchronous: the start's state must be set in the tick the host called it in, or an
+     * End issued straight after the Start (a user tapping back during the mint) finds nothing to abandon.
+     */
+    claimSessionSlot() {
+        if (this.IsActive) {
+            throw new RealtimeSessionAlreadyActiveError(this.agentSessionId);
+        }
+    }
     /**
      * Start a client-direct voice session fronting `targetAgentId`.
      *
@@ -1000,13 +1148,27 @@ export class RealtimeSessionRuntime {
      *   ({@link RealtimeSessionStartOptions.ConversationBranchID}; omitted is Main, where its transcript is written).
      */
     async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext, options) {
-        if (this.IsActive) {
-            return; // a session is already running — ignore duplicate starts
+        // Captured BEFORE any wait: a teardown from here on bumps it, and the checks below (and in
+        // runMintedSession) abandon the start. Re-capturing after the wait would absorb an End issued during it.
+        const generation = this.startGeneration;
+        // Synchronous unless a teardown is in flight: a start must take the slot in the same tick it was
+        // called in, so a host's End issued right after it finds a start to abandon (see claimSessionSlot).
+        if (this.teardownInFlight) {
+            await this.teardownInFlight;
+            // An End issued during the wait found IsActive still true (the last call was closing), so it only
+            // joined that teardown and bumped the generation. Nothing is acquired yet, so nothing to release.
+            if (this.startGeneration !== generation) {
+                return;
+            }
+        }
+        this.claimSessionSlot();
+        // Armed only now that the slot is ours: set before the claim, a refused start would leave its deadline to end
+        // the call that holds the runtime.
+        if (options?.SessionDeadline) {
+            this.SetSessionDeadline(options.SessionDeadline);
         }
         const consent = this.beginSessionStart({ agentName, recordingConsent, applicationId, appContext });
         this.sessionConversationId = conversationId ?? null;
-        // A teardown while the channels are prepared or the mint runs bumps this; see the checks below.
-        const generation = this.startGeneration;
         // Captured BEFORE the channels are prepared so the mint carries exactly the snapshot the prologue
         // resolved, whatever a channel plugin may push in the meantime.
         const effectiveAppContext = this._appContext$.value;
@@ -1032,14 +1194,14 @@ export class RealtimeSessionRuntime {
         // The host ended the start while the mint ran. That teardown had no session id to close, so
         // close the minted row here, and touch no shared state: a newer start may already own it.
         if (this.startGeneration !== generation) {
-            await this.closeServerSession(session.AgentSessionId);
+            await this.closeServerSession(session.AgentSessionId, 'Explicit');
             return;
         }
         if (!this.hostCanUseProvider(session.Provider)) {
             await this.abortUnusableSession(session);
             return;
         }
-        await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true);
+        await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true, generation);
     }
     /**
      * Declares which provider keys this host can actually carry audio for.
@@ -1075,8 +1237,7 @@ export class RealtimeSessionRuntime {
         // published them on ActiveChannels$; skipping teardown would leave them undisposed, their tool
         // handlers registered, and their subscriptions live until the next start replaced them.
         this.agentSessionId = session.AgentSessionId ?? this.agentSessionId;
-        this.setConnectionState('error');
-        await this.teardown(true);
+        await this.endOnFailure();
     }
     /**
      * Run a session the HOST has already minted itself — the second half of
@@ -1100,22 +1261,28 @@ export class RealtimeSessionRuntime {
      *   Every field defaults exactly as its {@link StartRealtimeSession} counterpart does.
      */
     async StartRealtimeSessionFromResult(result, options) {
-        if (this.IsActive) {
-            return; // a session is already running — ignore duplicate starts
+        // Same wait, same generation check, same reasons as StartRealtimeSession.
+        const generation = this.startGeneration;
+        if (this.teardownInFlight) {
+            await this.teardownInFlight;
+            if (this.startGeneration !== generation) {
+                return;
+            }
         }
+        this.claimSessionSlot();
         const effectiveOptions = options ?? {};
         if (!this.hostCanUseProvider(result.Provider)) {
             await this.abortUnusableSession(result);
             return;
         }
         const consent = this.beginSessionStart(effectiveOptions);
-        await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false);
+        await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false, generation);
     }
     /**
      * Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
      * publish the agent name, reset per-session state, and flip the session live (which is ALSO what
-     * makes the `IsActive` guard suppress duplicate starts while the mint is still in flight — hence it
-     * runs before minting, not after). Returns the resolved recording consent, which the mint half
+     * makes {@link claimSessionSlot} refuse a second start with {@link RealtimeSessionAlreadyActiveError}
+     * while the mint is still in flight — hence it runs before minting, not after). Returns the resolved recording consent, which the mint half
      * reports to the server and the run half uses to decide whether to record.
      */
     beginSessionStart(options) {
@@ -1155,10 +1322,10 @@ export class RealtimeSessionRuntime {
      * the provider connection, and go live. `inputConversationId` is the conversation the START asked
      * for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
      * whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
+     * `generation` is the {@link startGeneration} the entry point captured before its first await; every
+     * await below is a window in which the host can end the session, and so was every await before this.
      */
-    async runMintedSession(session, inputConversationId, consent, cameraCheck) {
-        // Captured up front: every await below is a window in which the host can end the session.
-        const generation = this.startGeneration;
+    async runMintedSession(session, inputConversationId, consent, cameraCheck, generation) {
         try {
             // Mount the channels the resolved policy puts in the session. Deliberately BEFORE the session
             // id is adopted: `ActiveChannels$` consumers (the sessions adapter) treat an emission that
@@ -1207,6 +1374,13 @@ export class RealtimeSessionRuntime {
             if (connectError) {
                 throw connectError;
             }
+            // A Fatal the driver sent during this start (the fatal handler leaves it to the start, see
+            // callWentLive) fails the start even when Connect resolved: the real drivers keep awaiting mic
+            // capture after their connect-failure channel settles, then report 'listening' and resolve on a
+            // socket that already closed. lastSessionError is reset per start and set only by the current client.
+            if (this.lastSessionError?.Fatal) {
+                throw new Error(this.lastSessionError.Message);
+            }
             // Tracks are negotiated now, so a capture can tell whether the model takes video, and the call whether it shows
             // the avatar its agent asked for. The driver holds the microphone now, so the user may switch it.
             this.openCaptures(client, cameraCheck);
@@ -1258,6 +1432,8 @@ export class RealtimeSessionRuntime {
             // Same place, same reason: the session is connected and its id is known, which is exactly
             // the window in which the server needs to be told it is alive.
             this.startLivenessPulse();
+            this.callWentLive = true;
+            this.replayPendingDeadlineReached();
         }
         catch (error) {
             await this.failSessionStart(error);
@@ -1319,7 +1495,7 @@ export class RealtimeSessionRuntime {
         // session, and clearing the id would leave that teardown no session to report.
         if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId && !this.teardownInFlight) {
             this.agentSessionId = null;
-            await this.closeServerSession(session.AgentSessionId);
+            await this.closeServerSession(session.AgentSessionId, 'Explicit');
         }
     }
     /**
@@ -1630,18 +1806,37 @@ export class RealtimeSessionRuntime {
     async failSessionStart(error) {
         console.error('[RealtimeSession] Failed to start session:', error);
         this.lastStartError = error instanceof Error ? error : new Error(String(error));
+        await this.endOnFailure();
+    }
+    /**
+     * Ends the call over a failure: a failed start, an unusable provider, or a fatal provider error.
+     *
+     * The teardown is claimed BEFORE `'error'` is published. `ConnectionState$` emits synchronously, and a
+     * host may end the call the moment it sees `'error'` (MobileApp's voice screen does); claimed first, that
+     * host's End joins this teardown instead of starting its own as `'explicit'`, so the row still closes as
+     * `Error` and `SessionEnded$` still says `'error'`. Teardown keeps a terminal `'error'` state.
+     */
+    endOnFailure() {
+        const ending = this.teardown('error');
         this.setConnectionState('error');
-        await this.teardown(false);
+        return ending;
     }
     /**
      * End the active session: stop the mic, tear down the provider connection, and close
      * the server-side agent session. Safe to call when no session is active.
+     *
+     * @param reason `'deadline'` for a host that owns the hang-up at its deadline (`autoEnd: false`): the end is reported
+     *   as `'deadline'` and the row is claimed as `TimeLimit`. The server keeps `TimeLimit` only when the session's own
+     *   stored deadline (`Config_.maxSessionDeadlineIso`, stamped at mint, e.g. a widget guest's voice cap) has passed or is
+     *   within a minute; a deadline known only to this browser is recorded as `Explicit`. Default `'explicit'`.
      */
-    async EndRealtimeSession() {
+    async EndRealtimeSession(reason = 'explicit') {
         if (!this.IsActive && !this.agentSessionId) {
             return;
         }
-        await this.teardown(true);
+        // A JS caller wiring this straight to a click handler passes the event object; anything but 'deadline' is explicit.
+        const end = reason === 'deadline' ? 'deadline' : 'explicit';
+        await this.teardown(end);
     }
     /**
      * Inject a typed message into the live session as a user turn.
@@ -1732,7 +1927,17 @@ export class RealtimeSessionRuntime {
         if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
             return false;
         }
-        this.requestChannelSpokenResponse(trimmed);
+        // The opening is part of the conversation, so it goes out as a spoken TURN: captioned, relayed
+        // and persisted like any turn the model takes on its own. It used to ride the delegation
+        // narration channel, whose replies are disposable by contract, so the greeting the participant
+        // heard was never in the transcript (MJ#4591). A driver from before the capability existed
+        // falls back to that channel rather than leaving the session silent.
+        if (this.client.RequestSpokenTurn) {
+            this.client.RequestSpokenTurn(trimmed);
+        }
+        else {
+            this.client.RequestSpokenUpdate(trimmed);
+        }
         return true;
     }
     /**
@@ -1814,6 +2019,7 @@ export class RealtimeSessionRuntime {
             this.turnAudioStartCaptured = false;
             this.userSpeechStartMs = null;
             this.userSpeechStartSuperseded = false;
+            this.userSpeechStartByItemId.clear();
             if (this.recorder) {
                 // WebRTC drivers (OpenAI): the agent's track usually lands AFTER Connect() resolves, so
                 // `remoteStream` above is null here and this handler attaches it later. PCM-playback
@@ -1948,17 +2154,19 @@ export class RealtimeSessionRuntime {
             // Closes over this recording's session id and transport — never the runtime's current ones.
             Queue: new RecordingShardQueue(async (shard) => {
                 if (!transport) {
-                    return false;
+                    return 'Retry';
                 }
-                const acked = (await transport.UploadShard(sessionId, shard)) === 'Acked';
-                if (acked) {
+                // The outcome goes to the queue unchanged: it logs a different reason for each, and stops
+                // retrying a `Refused` shard at once (#6012).
+                const outcome = await transport.UploadShard(sessionId, shard);
+                if (outcome === 'Acked') {
                     await this.deleteStoredShard(sessionId, shard.Index);
                     if (shardsGivenUp) {
                         shardsGivenUp = false;
                         this.recoverStoredShardsAfterOutage(sessionId);
                     }
                 }
-                return acked;
+                return outcome;
             }, () => {
                 shardsGivenUp = true;
             }),
@@ -2292,6 +2500,7 @@ export class RealtimeSessionRuntime {
         this.turnAudioStartCaptured = false;
         this.userSpeechStartMs = null;
         this.userSpeechStartSuperseded = false;
+        this.userSpeechStartByItemId.clear();
         if (!recorder || !pipeline) {
             this.recorder = null;
             this.recordingShards = null;
@@ -3384,6 +3593,27 @@ export class RealtimeSessionRuntime {
             console.error('[RealtimeSession] Provider error event:', JSON.stringify(error), error);
             if (error.Fatal && this.client === client) {
                 this.lastSessionError = error;
+                // While the start is still running, its own failure path reports the error and ends the call
+                // (runMintedSession throws on this recorded Fatal after Connect, even if Connect resolves):
+                // ending it here would bump startGeneration, so the start would read as ended by the host and
+                // never set LastStartError.
+                if (!this.callWentLive) {
+                    return;
+                }
+                // An explicit teardown is already ending the call and the client stays current until its
+                // finally: a driver reporting its own close as Fatal from inside Disconnect() is part of that
+                // clean End, and must not flip it to 'error' or start a second teardown.
+                if (this.teardownInFlight) {
+                    return;
+                }
+                // The driver is dead on the wire; nothing resumes after it says Fatal. End the call the way a
+                // failed start ends: the row closes as Error, the microphone is released, the host is told
+                // through SessionEnded$ (MJ#4629). A host's own End afterwards finds nothing to do.
+                // Captured now: teardown nulls agentSessionId, and the failure it reports must name the session.
+                const failedSessionId = this.agentSessionId;
+                void this.endOnFailure().catch((teardownError) => {
+                    console.error(`[RealtimeSession] Teardown after a fatal provider error failed for session ${failedSessionId}:`, teardownError);
+                });
             }
         });
         // Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
@@ -3403,9 +3633,10 @@ export class RealtimeSessionRuntime {
         });
         // The provider VAD saw the user start speaking. Final-only user-transcript drivers (OpenAI,
         // HuggingFace, xAI) emit no user interim, so this is their only real user-turn start. It goes
-        // to its own slot, not the shared turn slot: the user transcript arrives asynchronously, so the
-        // assistant's reply may already be streaming and must keep its own start.
-        client.OnUserSpeechStart(() => this.markUserSpeechStart());
+        // to its own slot (or item-keyed map entry when `itemId` is present), not the shared turn slot:
+        // the user transcript arrives asynchronously, so the assistant's reply may already be streaming
+        // and must keep its own start.
+        client.OnUserSpeechStart((itemId) => this.markUserSpeechStart(itemId));
         client.OnRemoteVideo((video) => this.onAgentVideo(client, video));
     }
     /**
@@ -3511,7 +3742,7 @@ export class RealtimeSessionRuntime {
         if (transcript.Role === 'User' && transcript.Text.trim().length === 0) {
             // An empty final user transcript (a VAD false positive, an empty transcription) never reaches
             // relayTranscript, so release its onset here: it must not become the next real turn's start.
-            this.releaseUserSpeechStart();
+            this.releaseUserSpeechStart(transcript.ItemId);
         }
         if (transcript.Role === 'Assistant') {
             this.hasActiveInterimUserCaption = false;
@@ -3555,7 +3786,7 @@ export class RealtimeSessionRuntime {
             if (this.firstUserTranscript === null) {
                 this.firstUserTranscript = transcript.Text;
             }
-            await this.relayTranscript('user', transcript.Text);
+            await this.relayTranscript('user', transcript.Text, false, 'speech', transcript.ItemId);
         }
         else if (transcript.ReplacesPrevious) {
             // STREAMING user transcription: providers like Grok and OpenAI Live emit the growing utterance as repeated
@@ -3566,10 +3797,10 @@ export class RealtimeSessionRuntime {
                 return;
             }
             this.replaceLastCaption('User', transcript.Text);
-            await this.relayTranscript('user', transcript.Text, true);
+            await this.relayTranscript('user', transcript.Text, true, 'speech', transcript.ItemId);
         }
         else {
-            await this.onUserTranscript(transcript.Text);
+            await this.onUserTranscript(transcript.Text, 'speech', transcript.ItemId);
         }
     }
     /**
@@ -3604,13 +3835,24 @@ export class RealtimeSessionRuntime {
         this.turnAudioStartCaptured = true;
     }
     /**
-     * Stamps {@link userSpeechStartMs} when the client signals that the user began to speak. Keeps
-     * the FIRST onset since the last final user transcript: the VAD fires again after a
-     * mid-sentence pause, and that second onset is not where the utterance began. An onset that an
-     * assistant turn which began after it has since superseded ({@link userSpeechStartSuperseded}) is
-     * replaced instead.
+     * Stamps the user speech onset when the client signals that the user began to speak. When the
+     * driver supplies a conversation `itemId`, records the FIRST onset for that item in
+     * {@link userSpeechStartByItemId} so a mid-sentence pause that re-fires with the same id keeps the
+     * original start. When no `itemId` is supplied, stamps the single-slot {@link userSpeechStartMs}
+     * fallback (keeping the first onset unless {@link userSpeechStartSuperseded} is set).
      */
-    markUserSpeechStart() {
+    markUserSpeechStart(itemId) {
+        if (itemId !== undefined && itemId.length > 0) {
+            if (this.userSpeechStartByItemId.has(itemId)) {
+                return;
+            }
+            const offset = this.nowTurnOffsetMs();
+            if (offset === null) {
+                return;
+            }
+            this.userSpeechStartByItemId.set(itemId, offset);
+            return;
+        }
         if (this.userSpeechStartMs !== null && !this.userSpeechStartSuperseded) {
             return;
         }
@@ -3621,11 +3863,36 @@ export class RealtimeSessionRuntime {
         this.userSpeechStartMs = offset;
         this.userSpeechStartSuperseded = false;
     }
-    /** Ends the held user onset: a final user transcript (empty or not) closed the utterance it began. */
-    releaseUserSpeechStart() {
+    /**
+     * Ends the held user onset: a final user transcript (empty or not) closed the utterance it began.
+     * When `itemId` is supplied, deletes only that item's entry in {@link userSpeechStartByItemId};
+     * otherwise clears the unkeyed single slot.
+     */
+    releaseUserSpeechStart(itemId) {
+        if (itemId !== undefined && itemId.length > 0) {
+            this.userSpeechStartByItemId.delete(itemId);
+            return;
+        }
         this.userSpeechStartMs = null;
         this.userSpeechStartSuperseded = false;
     }
+    /**
+     * Consumes and deletes the onset recorded for `itemId` in {@link userSpeechStartByItemId}. When a
+     * matching onset is found, also deletes any remaining map entry with an earlier offset so a VAD
+     * false positive whose transcription never arrived cannot accumulate in the map.
+     */
+    takeKeyedUserSpeechStart(itemId) {
+        const onset = this.userSpeechStartByItemId.get(itemId) ?? null;
+        this.userSpeechStartByItemId.delete(itemId);
+        if (onset !== null) {
+            for (const [otherId, otherOnset] of this.userSpeechStartByItemId) {
+                if (otherOnset < onset) {
+                    this.userSpeechStartByItemId.delete(otherId);
+                }
+            }
+        }
+        return onset;
+    }
     /**
      * The current per-turn offset in ms — the RECORDER's clock when one runs (an offset into a
      * seekable file), else the SESSION clock (#3832: orderable and displayable, not seekable),
@@ -3664,8 +3931,9 @@ export class RealtimeSessionRuntime {
      *
      * @param source `'typed'` for a {@link SendText} turn: it has no speech onset, so it neither takes
      *   nor clears the held one (a spoken transcript may still be pending for it).
+     * @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
      */
-    async onUserTranscript(transcript, source = 'speech') {
+    async onUserTranscript(transcript, source = 'speech', itemId) {
         if (transcript.trim().length === 0) {
             return;
         }
@@ -3674,7 +3942,7 @@ export class RealtimeSessionRuntime {
             this.firstUserTranscript = transcript;
         }
         this.appendCaption({ Role: 'User', Text: transcript });
-        await this.relayTranscript('user', transcript, false, source);
+        await this.relayTranscript('user', transcript, false, source, itemId);
     }
     // ── Tool calling ───────────────────────────────────────────────────────────
     /**
@@ -4022,16 +4290,23 @@ export class RealtimeSessionRuntime {
      *   recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
      *   post-barge-in `agent_response_correction`).
      * @param source `'typed'` for a user turn sent with {@link SendText}: it does not read or clear
-     *   {@link userSpeechStartMs}, which belongs to the user's speech.
+     *   {@link userSpeechStartMs} or {@link userSpeechStartByItemId}, which belong to the user's speech.
+     * @param itemId Provider conversation item id (`RealtimeClientTranscript.ItemId`) when present.
      */
-    async relayTranscript(role, text, replacesPrevious = false, source = 'speech') {
+    async relayTranscript(role, text, replacesPrevious = false, source = 'speech', itemId) {
         if (!this.agentSessionId) {
             return;
         }
         // Every final user transcript closes the signalled onset — a correction too, since it ends the
         // same utterance — so a later onset (an xAI mid-sentence resume) can't leak into the next turn.
+        // When the transcript carries a provider itemId, consume its keyed onset from the map without
+        // touching the unkeyed single slot or falling back to currentTurnStartMs.
+        const hasItemId = role === 'user' && source === 'speech' && itemId !== undefined && itemId.length > 0;
         let userSpeechStartMs = null;
-        if (role === 'user' && source === 'speech') {
+        if (hasItemId) {
+            userSpeechStartMs = this.takeKeyedUserSpeechStart(itemId);
+        }
+        else if (role === 'user' && source === 'speech') {
             userSpeechStartMs = this.userSpeechStartMs;
             this.releaseUserSpeechStart();
         }
@@ -4042,9 +4317,10 @@ export class RealtimeSessionRuntime {
         const utteranceEndMs = this.nowTurnOffsetMs();
         const opensTurn = utteranceEndMs !== null && !replacesPrevious;
         let utteranceStartMs = null;
-        if (opensTurn && userSpeechStartMs !== null) {
-            // A user turn with a signalled onset. Leave the shared slot alone: the user transcript
-            // arrives asynchronously, so it may already hold the in-flight assistant reply's start.
+        if (opensTurn && (hasItemId || userSpeechStartMs !== null)) {
+            // A user turn with a signalled onset (or a keyed user turn whose itemId was not in the map,
+            // which relays null). Leave the shared slot alone: the user transcript arrives asynchronously,
+            // so it may already hold the in-flight assistant reply's start.
             utteranceStartMs = userSpeechStartMs;
             if (!this.turnAudioStartCaptured) {
                 // No turn has captured a start, so the slot holds only the recording-start seed (0). This
@@ -4222,34 +4498,34 @@ export class RealtimeSessionRuntime {
      * delegation path is untouched.
      */
     onDelegationStatusMessage(raw) {
-        const frame = this.parseScreencastFrame(raw);
+        let parsed;
+        try {
+            parsed = JSON.parse(raw);
+        }
+        catch {
+            return;
+        }
+        const frame = this.parseScreencastFrame(parsed);
         if (frame) {
             this.routeScreencastFrame(frame);
             return;
         }
-        const audio = this.parseAudioChunk(raw);
+        const audio = this.parseAudioChunk(parsed);
         if (audio) {
             this.routeAudioChunk(audio);
             return;
         }
-        const progress = this.parseProgress(raw);
+        const progress = this.parseProgress(parsed);
         if (progress) {
             this.dispatchProgress(progress);
         }
     }
     /**
-     * Parses a push-status message and returns it only when it's a Remote Browser screencast frame for the
+     * Matches a parsed push-status payload and returns it only when it's a Remote Browser screencast frame for the
      * active session — otherwise `null` (ignored, so delegation progress falls through). Matched by
      * `resolver` + `type`, then scoped to THIS session by `agentSessionID`.
      */
-    parseScreencastFrame(raw) {
-        let payload;
-        try {
-            payload = JSON.parse(raw);
-        }
-        catch {
-            return null;
-        }
+    parseScreencastFrame(payload) {
         const matches = payload?.resolver === 'RemoteBrowserActionResolver' &&
             payload?.type === 'RemoteBrowserScreencastFrame' &&
             payload?.agentSessionID === this.agentSessionId &&
@@ -4276,18 +4552,11 @@ export class RealtimeSessionRuntime {
         return typeof channel.OnScreencastFrame === 'function';
     }
     /**
-     * Parses a push-status message and returns it only when it's a Remote Browser audio chunk for the active
+     * Matches a parsed push-status payload and returns it only when it's a Remote Browser audio chunk for the active
      * session — otherwise `null` (ignored). Matched by `resolver` + `type`, then scoped to THIS session by
      * `agentSessionID`.
      */
-    parseAudioChunk(raw) {
-        let payload;
-        try {
-            payload = JSON.parse(raw);
-        }
-        catch {
-            return null;
-        }
+    parseAudioChunk(payload) {
         const matches = payload?.resolver === 'RemoteBrowserActionResolver' &&
             payload?.type === 'RemoteBrowserAudioChunk' &&
             payload?.agentSessionID === this.agentSessionId &&
@@ -4318,17 +4587,10 @@ export class RealtimeSessionRuntime {
         return typeof channel.OnAudioChunk === 'function';
     }
     /**
-     * Parses a push-status message and returns it only when it's a delegation
+     * Matches a parsed push-status payload and returns it only when it's a delegation
      * progress event for the active voice session — otherwise `null` (ignored).
      */
-    parseProgress(raw) {
-        let payload;
-        try {
-            payload = JSON.parse(raw);
-        }
-        catch {
-            return null; // non-JSON or unrelated frame
-        }
+    parseProgress(payload) {
         const matches = payload?.resolver === 'RealtimeClientSessionResolver' &&
             payload?.type === 'RealtimeDelegationProgress' &&
             payload?.agentSessionID === this.agentSessionId;
@@ -4460,10 +4722,19 @@ export class RealtimeSessionRuntime {
     }
     // ── Teardown ───────────────────────────────────────────────────────────────
     /**
-     * Tears down all client resources and (optionally) closes the server session.
-     * @param closeServerSession when true, calls `CloseAgentSession` on the server.
+     * Tears down all client resources and closes the server session when one was minted.
+     *
+     * The server row is closed whichever way the call ended: a start that failed after the mint used to
+     * leave its row `Active` until the janitor's staleness sweep, 15 minutes later, so every host that
+     * counts live sessions saw a phantom call (MJ#6013). The janitor remains the fallback for a client
+     * that dies before it can say anything; it is never the plan.
+     *
+     * @param end why the call ended — `'explicit'` when the host or user ended it, `'error'` when the
+     *   runtime ended it because the start or the call failed, `'deadline'` when the session deadline ended it.
+     *   Reported on `SessionEnded$` and stamped on the server row as its `CloseReason` through
+     *   {@link CLOSE_REASON_BY_END} (`Explicit` / `Error` / `TimeLimit`).
      */
-    async teardown(closeServerSession) {
+    async teardown(end) {
         // Invalidate any start still in flight BEFORE anything else, so it abandons itself at its next
         // await rather than opening a microphone behind a session that is being ended.
         this.startGeneration++;
@@ -4473,7 +4744,7 @@ export class RealtimeSessionRuntime {
             await this.teardownInFlight;
             return;
         }
-        this.teardownInFlight = this.runTeardown(closeServerSession).finally(() => {
+        this.teardownInFlight = this.runTeardown(end).finally(() => {
             if (this._savingRecording$.value) {
                 this._savingRecording$.next(false);
             }
@@ -4482,17 +4753,29 @@ export class RealtimeSessionRuntime {
         await this.teardownInFlight;
     }
     /** The body of {@link teardown}; never called concurrently with itself. */
-    async runTeardown(closeServerSession) {
+    async runTeardown(end) {
         // Arm BEFORE any await so a tab closed in the first milliseconds of teardown is still guarded.
         if (this.recorder) {
             this._savingRecording$.next(true);
         }
+        // Whatever a step below does, the slot is released: a teardown that threw halfway used to leave
+        // `IsActive` set for good, so every later start was refused (MJ#4629).
+        try {
+            await this.releaseCallResources(end);
+        }
+        finally {
+            this.releaseSessionSlot(end);
+        }
+    }
+    /** The teardown's work: stop the call, hand back what it held, and close the server session. */
+    async releaseCallResources(end) {
         // First: stop asserting liveness. A pulse racing the close would re-stamp LastActiveAt on a
         // session we are deliberately ending, leaving an Idle row the janitor then has to age out.
         this.stopLivenessPulse();
         this.teardownDelegationProgress();
         this.stopSessionEvents();
         this.clientDeadline.Clear();
+        this.pendingDeadlineReached = null;
         // Channels first: flush any unsaved channel state WHILE the live session id is still
         // set (the captured per-save id covers the race anyway), then dispose the plugins.
         this.flushAllChannelSaves();
@@ -4546,8 +4829,8 @@ export class RealtimeSessionRuntime {
         }
         await this.flushPendingUsage(this.agentSessionId);
         this.resetUsageRelay();
-        if (closeServerSession && this.agentSessionId) {
-            await this.closeServerSession(this.agentSessionId);
+        if (this.agentSessionId) {
+            await this.closeServerSession(this.agentSessionId, CLOSE_REASON_BY_END[end]);
         }
         // Wait for the terminal's ack only AFTER the close: nothing after an aborted await ever runs, so
         // the small, critical close must never queue behind the recording (#5195). Bounded by
@@ -4555,11 +4838,15 @@ export class RealtimeSessionRuntime {
         if (recordingTerminal) {
             await this.awaitRecordingTerminal(recordingTerminal);
         }
+    }
+    /** Clears the slot and tells the host the call ended; runs even when {@link releaseCallResources} threw. */
+    releaseSessionSlot(end) {
         // Capture the session id BEFORE we null it so the lifecycle emit carries it.
         // Skip emitting when there was no live session (defensive — teardown is safe
         // to call without an active session).
         const closedSessionId = this.agentSessionId;
         this.agentSessionId = null;
+        this.callWentLive = false;
         this.sessionConversationId = null;
         this.narrationTemplate = null;
         this.clientToolHandlers.clear();
@@ -4572,14 +4859,9 @@ export class RealtimeSessionRuntime {
         if (this._connectionState$.value !== 'error') {
             this.setConnectionState('closed');
         }
-        // Surface generic session-ended for the conversations runtime bridge.
-        // `closeServerSession=true` means the user explicitly called EndRealtimeSession;
-        // `false` means teardown ran from a catch block (start path error path).
+        // Surface generic session-ended for the conversations runtime bridge, with why it ended.
         if (closedSessionId) {
-            this._sessionEnded$.next({
-                sessionId: closedSessionId,
-                reason: closeServerSession ? 'explicit' : 'error',
-            });
+            this._sessionEnded$.next({ sessionId: closedSessionId, reason: end });
         }
     }
     /**
@@ -4594,18 +4876,22 @@ export class RealtimeSessionRuntime {
             console.error('[RealtimeSession] Media host failed to release the microphone:', error);
         }
     }
-    /** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
-    async closeServerSession(agentSessionId) {
+    /**
+     * Calls the `CloseAgentSession` mutation (provisioned in P4b) with why the client ended the call, which
+     * the server stamps as the row's `CloseReason`. Requires a server that accepts the `closeReason`
+     * argument (MJ#6013); the runtime and the server ship together.
+     */
+    async closeServerSession(agentSessionId, closeReason) {
         try {
             const mutation = `
-        mutation CloseAgentSession($agentSessionId: String!) {
-          CloseAgentSession(agentSessionId: $agentSessionId)
+        mutation CloseAgentSession($agentSessionId: String!, $closeReason: String) {
+          CloseAgentSession(agentSessionId: $agentSessionId, closeReason: $closeReason)
         }
       `;
-            await this.gql().ExecuteGQL(mutation, { agentSessionId });
+            await this.gql().ExecuteGQL(mutation, { agentSessionId, closeReason });
         }
         catch (error) {
-            console.error('[RealtimeSession] Failed to close server session:', error);
+            console.error(`[RealtimeSession] Failed to close server session ${agentSessionId} (closeReason ${closeReason}):`, error);
         }
     }
     // ── Helpers ────────────────────────────────────────────────────────────────
@@ -4629,6 +4915,7 @@ export class RealtimeSessionRuntime {
         this.turnAudioStartCaptured = false;
         this.userSpeechStartMs = null;
         this.userSpeechStartSuperseded = false;
+        this.userSpeechStartByItemId.clear();
         this.usedChannelNames.clear();
         this.microphoneMuted = false;
     }