@codingame/monaco-vscode-chat-service-override 34.1.3 → 35.0.0

package/vscode/src/vs/workbench/contrib/chat/browser/voiceClient/voiceSessionController.d.ts

@@ -0,0 +1,321 @@
+import { Disposable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle";
+import { IObservable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/observable";
+import { URI } from "@codingame/monaco-vscode-api/vscode/vs/base/common/uri";
+import { ICommandService } from "@codingame/monaco-vscode-api/vscode/vs/platform/commands/common/commands.service";
+import { ILogService } from "@codingame/monaco-vscode-api/vscode/vs/platform/log/common/log.service";
+import { IAuthenticationService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/services/authentication/common/authentication.service";
+import { IVoiceTranscriptStore } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/agentsVoice/common/voiceTranscriptStore.service";
+import { IVoiceClientService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/common/voiceClient/voiceClientService.service";
+import { IMicCaptureService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/voiceClient/micCaptureService.service";
+import { ITtsPlaybackService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/voiceClient/ttsPlaybackService.service";
+import { IVoiceToolDispatchService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/voiceClient/voiceToolDispatchService.service";
+import { IVoicePlaybackService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/common/voicePlaybackService.service";
+import { IAgentSessionsService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/agentSessions/agentSessionsService.service";
+import { IChatService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/common/chatService/chatService.service";
+import { IWorkbenchEnvironmentService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/services/environment/common/environmentService.service";
+import { ITelemetryService } from "@codingame/monaco-vscode-api/vscode/vs/platform/telemetry/common/telemetry.service";
+import { IConfigurationService } from "@codingame/monaco-vscode-api/vscode/vs/platform/configuration/common/configuration.service";
+import { IAccessibilitySignalService } from "@codingame/monaco-vscode-api/vscode/vs/platform/accessibilitySignal/browser/accessibilitySignalService.service";
+import { IAccessibilityService } from "@codingame/monaco-vscode-api/vscode/vs/platform/accessibility/common/accessibility.service";
+import { IVoiceSessionController } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/voiceClient/voiceSessionController.service";
+export type VoiceState = "idle" | "listening" | "processing" | "speaking" | "error";
+export interface IPendingToolConfirmation {
+    readonly type: "approval" | "input";
+    readonly sessionLabel: string;
+    readonly sessionResource: URI;
+    readonly description: string;
+    approve(): void;
+    deny(): void;
+}
+export interface ITranscriptTurn {
+    readonly speaker: "user" | "assistant";
+    readonly text: string;
+    /** Stable-recognition prefix of `text`. User turns only; empty otherwise. */
+    readonly committed: string;
+    /** True while the user is still speaking (live recognition). */
+    readonly isPartial: boolean;
+}
+export declare class VoiceSessionController extends Disposable implements IVoiceSessionController {
+    private readonly voiceClientService;
+    private readonly micCaptureService;
+    private readonly ttsPlaybackService;
+    private readonly voiceToolDispatchService;
+    private readonly voicePlaybackService;
+    private readonly agentSessionsService;
+    private readonly chatService;
+    private readonly commandService;
+    private readonly authenticationService;
+    private readonly voiceTranscriptStore;
+    private readonly logService;
+    private readonly environmentService;
+    private readonly telemetryService;
+    private readonly configurationService;
+    private readonly accessibilitySignalService;
+    private readonly accessibilityService;
+    readonly _serviceBrand: undefined;
+    private readonly _voiceState;
+    readonly voiceState: IObservable<VoiceState>;
+    private readonly _statusText;
+    readonly statusText: IObservable<string>;
+    private static readonly _MAX_TURNS;
+    private readonly _transcriptTurns;
+    readonly transcriptTurns: IObservable<readonly ITranscriptTurn[]>;
+    private readonly _isConnected;
+    readonly isConnected: IObservable<boolean>;
+    private readonly _isConnecting;
+    readonly isConnecting: IObservable<boolean>;
+    private readonly _isReconnecting;
+    readonly isReconnecting: IObservable<boolean>;
+    private readonly _pendingToolConfirmations;
+    readonly pendingToolConfirmations: IObservable<readonly IPendingToolConfirmation[]>;
+    private readonly _targetSession;
+    readonly targetSession: IObservable<URI | undefined>;
+    private _pttHeld;
+    private _pttToggleMode;
+    private _pttCurrentTurnId;
+    private _window;
+    private readonly _voiceEventDisposables;
+    private readonly _voiceAutorunDisposable;
+    private readonly _autoApprovedSessions;
+    private _transcriptFadeTimer;
+    private _pttMaxDurationTimer;
+    private static readonly _PTT_MAX_DURATION_MS;
+    /** Short-tap threshold: if the key is held for less than this, enter
+     *  toggle mode where a second tap finishes the recording. */
+    private static readonly _PTT_TOGGLE_THRESHOLD_MS;
+    /** Debounce before re-entering listening after assistant stops speaking. */
+    private static readonly _AUTO_LISTEN_QUIET_MS;
+    private _delayedMicStopTimer;
+    private _autoSendSilenceTimer;
+    private _autoListenTimer;
+    private _pttWaitingForPlayback;
+    /** Guards auto re-listen: only re-arm after a reply has actually played. */
+    private _replyPlayedSinceSend;
+    /** Set after send_to_chat; blocks auto-listen until the reply TTS starts. */
+    private _awaitingReplyAudio;
+    private _awaitingReplyWatchdog;
+    /** Enter listening immediately after greeting finishes (no debounce). */
+    private _autoListenAfterGreeting;
+    /** Tracks whether the initial listen cue has been played after connecting. */
+    private _hasPlayedInitialListenCue;
+    private readonly _audioQueue;
+    private _currentPlaybackSessionId;
+    private _isProcessingQueue;
+    private _suppressIncomingAudio;
+    private readonly _sessionAudioCache;
+    private _replaySourceNode;
+    private readonly _prevSessionStates;
+    private readonly _userCancelledSessions;
+    private static readonly _USER_CANCEL_SUPPRESS_MS;
+    private readonly _confirmationFlushWatchdogs;
+    private static readonly _CONFIRMATION_FLUSH_DELAY_MS;
+    /** Model refs eagerly loaded for sessions awaiting input (no UI focus needed). */
+    private readonly _eagerModelRefs;
+    /** Sessions with an in-flight eager model load, to dedupe concurrent loads. */
+    private readonly _eagerModelLoading;
+    /**
+     * Sessions whose ``idle`` transition is being deferred until their chat
+     * model loads, so the narration can include ``last_response_summary``.
+     * While a session id is in this set we suppress emitting a premature,
+     * summary-less ``idle`` to the backend (see _buildSessionContext).
+     */
+    private readonly _pendingIdleNarration;
+    private _telemetrySessionIndex;
+    private _telemetrySessionStart;
+    private _telemetryTurnCount;
+    private _telemetryReconnectCount;
+    private _telemetryFirstConnect;
+    private _telemetryConnectStartMs;
+    private _telemetryLastConnectMs;
+    private _telemetryPttDownMs;
+    private _telemetryPttUpMs;
+    private _telemetryFirstTranscriptionMs;
+    private _telemetryTtsInterrupted;
+    /** Cached GitHub login resolved on connect; used as transcript partition key. */
+    private _userLogin;
+    /** Locally-persisted turn id of the last assistant turn we appended.
+     * Used as the ancestor of the next user turn we persist. */
+    private _lastPersistedTurnId;
+    /** Last-N cross-session timeline entries — voice turns, voice tool
+     * calls, coding-session events, plus a synthesized first-2-sentences
+     * summary of the latest Copilot reply per active session. Sent to the
+     * BE on the next start_session and then cleared — single-shot recall. */
+    private _pendingPriorTimeline;
+    /**
+     * How many of the most recent persisted timeline entries we forward
+     * to the BE (across all kinds). Coding-agent reply synthesis happens
+     * on top of this — we add one entry per active coding session.
+     */
+    private static readonly PRIOR_TIMELINE_ENTRY_LIMIT;
+    /**
+     * Max sentences of Copilot's last reply we include per active coding
+     * session when synthesizing ``coding_agent_reply`` entries. Bounded
+     * because the full reply can be arbitrarily long.
+     */
+    private static readonly CODING_AGENT_REPLY_SENTENCE_LIMIT;
+    constructor(voiceClientService: IVoiceClientService, micCaptureService: IMicCaptureService, ttsPlaybackService: ITtsPlaybackService, voiceToolDispatchService: IVoiceToolDispatchService, voicePlaybackService: IVoicePlaybackService, agentSessionsService: IAgentSessionsService, chatService: IChatService, commandService: ICommandService, authenticationService: IAuthenticationService, voiceTranscriptStore: IVoiceTranscriptStore, logService: ILogService, environmentService: IWorkbenchEnvironmentService, telemetryService: ITelemetryService, configurationService: IConfigurationService, accessibilitySignalService: IAccessibilitySignalService, accessibilityService: IAccessibilityService);
+    connect(window: Window & typeof globalThis): Promise<void>;
+    disconnect(): void;
+    /** DEV ONLY: Simulate a connected session with fake transcript for UI testing. */
+    simulateConnection(): void;
+    private _onConnectionLost;
+    pttDown(): void;
+    pttUp(): void;
+    private _finishPtt;
+    markUserCancelled(sessionId: string): void;
+    setTargetSession(resource: URI | undefined): void;
+    newSessionAsTarget(): void;
+    private _scheduleDelayedMicStop;
+    private _isAutoSendEnabled;
+    /**
+     * Check if the transcript text ends with the configured send keyword.
+     * Returns the text with the keyword stripped if found, or undefined if not.
+     */
+    private _checkSendKeyword;
+    /** Re-enter listening via synthetic short tap. */
+    private _enterAutoListen;
+    /** Debounced re-listen after assistant stops speaking. */
+    private _scheduleAutoListen;
+    private _clearAutoListenTimer;
+    /** Auto-finish recording after configured silence in toggle mode. */
+    private _scheduleAutoSendOnSilence;
+    private _clearAutoSendSilenceTimer;
+    /** Block auto-listen until reply audio arrives (with 30s watchdog). */
+    private _setAwaitingReply;
+    private _clearAwaitingReply;
+    /**
+     * Send transcription text to the target session or active chat.
+     * If a target session is selected, sends directly via chatService.
+     * Otherwise sends to whatever is currently active via the view pane command.
+     */
+    private _sendTranscriptionToChat;
+    /**
+     * Watch a session's latest response and surface it in the floating window
+     * transcript. Called when voice sends to a non-visible session so the user
+     * can see the reply without switching the chat panel.
+     */
+    private _watchResponseForFloatingWindow;
+    private _pushTurn;
+    /**
+     * Start a new user turn at the tail of the buffer. If the previous tail is
+     * already an empty user turn (rapid PTT toggle before any transcription
+     * landed), reuse it instead of pushing a duplicate empty entry.
+     */
+    private _startUserTurn;
+    private _updateUserTurn;
+    /**
+     * Update the assistant turn at the tail of the buffer with `text`.
+     *
+     * The streaming TTS pipeline pushes a monotonically-growing transcript
+     * with each audio chunk of a response. `startNewTurn` distinguishes
+     * the first chunk of a NEW response (push a fresh assistant turn)
+     * from continuation chunks of the SAME response (replace the tail's
+     * text as the transcript grows). This prevents two distinct
+     * assistant responses from collapsing into one when they happen
+     * back-to-back without an intervening user turn (e.g. proactive
+     * narration followed by a command reply).
+     */
+    private _setAssistantTurn;
+    private _cancelTranscriptFade;
+    /**
+     * Append a final entry to the on-disk transcript store.
+     *
+     * Entry ids are generated locally — voice_code's backend has no persistent
+     * conversation memory today, so there's no server-issued id to defer to.
+     * Each new entry chains off the previous one via ``ancestorIds`` so a UI
+     * can show the linear conversation order.
+     *
+     * ``user_voice`` and ``agent_voice`` are user-visible in the transcripts
+     * pane. ``agent_tool_call`` and ``coding_event`` are persisted only so we
+     * can replay them as cross-session context to the backend on reconnect.
+     */
+    private _persistEntry;
+    /** Back-compat thin shim for the two existing voice call sites. */
+    private _persistTurn;
+    /**
+     * One-line, human/LLM-readable summary of a voice tool call for the
+     * timeline. Backend's prior_timeline renderer expects this format — keep
+     * it stable.
+     *
+     *   send_to_chat(text="Open a new terminal and cd into the current directory.")
+     *   new_sessions(sessions=[{"text": "Refactor upload service"}])
+     *   approve_confirmation(...)
+     */
+    private _renderToolCallSummary;
+    /**
+     * Convert persisted transcript turns into typed timeline entries for
+     * the BE, then top up with a synthesized ``coding_agent_reply`` per
+     * active coding session (first ~2 sentences of the latest Copilot
+     * response). The synthetic entries are *not* persisted — they read
+     * live ``IChatModel`` state so the summary stays fresh on every
+     * reconnect.
+     *
+     * Output is chronological (oldest first), matching what the BE
+     * renders into its ``[PRIOR_CONTEXT]`` block. Synthetic
+     * ``coding_agent_reply`` entries are appended at the end since they
+     * represent the *current* state of coding sessions at reconnect.
+     */
+    private _buildPriorTimeline;
+    /**
+     * Return the first ``n`` sentences of ``text``. Cheap regex split —
+     * good enough for a prompt-prefix summary; we don't need perfect NLP
+     * boundaries here. Falls back to a hard char cap if no terminator
+     * shows up in the first 600 chars.
+     */
+    private _firstSentences;
+    private _enqueueAudio;
+    private _playChunk;
+    private _processQueue;
+    private _replaySessionAudio;
+    private _stopReplay;
+    private _sendContext;
+    /**
+     * Paranoid mitigation for the "confirmation narration not fired while user
+     * is on the same session" symptom. Even though the autorun calls
+     * `_sendContext + flushSessionContext` at the transition, in practice
+     * users observed that the BE-side narration ("I need approval to run X")
+     * only fires after they navigate AWAY from the session.
+     *
+     * As a guarded re-flush we schedule a single delayed `_sendContext + flush`
+     * per session that's awaiting confirmation. The merge-patch in
+     * `_sendDelta` short-circuits when no fields changed (see lines 393-395),
+     * so a no-op re-send is silent on the BE — but if the FIRST send was
+     * dropped (race condition, debounce hiccup, WS coalescing), this second
+     * send pushes the state through.
+     *
+     * The watchdog auto-clears once the autorun observes the session has left
+     * `waiting_for_confirmation`.
+     */
+    private _armConfirmationFlushWatchdog;
+    /**
+     * Check all sessions for state changes and send notifications to backend.
+     * This catches state transitions for sessions without a loaded chat model
+     * (which the autorun can't track via observables), and also regular chat
+     * sessions that are not agent sessions.
+     */
+    private _checkSessionStateChanges;
+    private _buildSessionContext;
+    /**
+     * Eagerly load a chat model for a session that needs input but hasn't been
+     * opened in the UI yet. Once loaded, the autorun observables will re-fire
+     * with full confirmation detail so the backend can narrate properly.
+     */
+    private _ensureModelLoaded;
+    /**
+     * Defer narrating a session's ``idle`` transition until its chat model is
+     * resident, so the narration can include ``last_response_summary``. Remote/
+     * Copilot sessions don't keep their model loaded, so without this the
+     * backend would only ever see a summary-less completion. Eagerly loads the
+     * model; once it resolves the autorun re-fires and narrates with the summary.
+     */
+    private _deferIdleNarrationUntilModelLoaded;
+    private _getAgentStateInfo;
+    private _classifyPendingType;
+    private _getConfirmationDescription;
+    private _autoApproveCheck;
+    private _getMachineId;
+    submitFeedback(feedbackText: string): Promise<{
+        ok: boolean;
+        error?: string;
+    }>;
+}