@codingame/monaco-vscode-chat-service-override 34.1.3 → 35.0.1

package/vscode/src/vs/workbench/contrib/chat/browser/voiceClient/ttsPlaybackService.js

@@ -0,0 +1,198 @@
+
+import { __decorate, __param } from '@codingame/monaco-vscode-api/external/tslib/tslib.es6';
+import { Disposable } from '@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle';
+import { Emitter } from '@codingame/monaco-vscode-api/vscode/vs/base/common/event';
+import '@codingame/monaco-vscode-api/vscode/vs/platform/instantiation/common/extensions';
+import { ILogService } from '@codingame/monaco-vscode-api/vscode/vs/platform/log/common/log.service';
+import '@codingame/monaco-vscode-api/vscode/vs/platform/instantiation/common/instantiation';
+
+const PLAYBACK_SAMPLE_RATE = 24000;
+const MAX_PLAYBACK_SAMPLES = PLAYBACK_SAMPLE_RATE * 180;
+let TtsPlaybackService = class TtsPlaybackService extends Disposable {
+    constructor(logService) {
+        super();
+        this.logService = logService;
+        this._playbackTurn = null;
+        this._playbackGen = 0;
+        this._isPlaying = false;
+        this._lastPlayedSamples = null;
+        this._onPlaybackStarted = this._register(( new Emitter()));
+        this.onPlaybackStarted = this._onPlaybackStarted.event;
+        this._onPlaybackStopped = this._register(( new Emitter()));
+        this.onPlaybackStopped = this._onPlaybackStopped.event;
+    }
+    get isPlaying() {
+        return this._isPlaying;
+    }
+    get analyserNode() {
+        return this._analyserNode;
+    }
+    getLastPlayedSamples() {
+        return this._lastPlayedSamples;
+    }
+    ensureContext(window) {
+        this._window = window;
+        if (!this._playbackCtx) {
+            this._playbackCtx = new window.AudioContext({
+                sampleRate: PLAYBACK_SAMPLE_RATE
+            });
+        }
+        if (this._playbackCtx.state === "suspended") {
+            this._playbackCtx.resume().catch(() => {});
+        }
+        return this._playbackCtx;
+    }
+    playAudioChunk(audio, isFinal, window) {
+        this._window = window;
+        if (!audio && isFinal) {
+            const turn = this._ensurePlayTurn(window);
+            turn.writeChain = turn.writeChain.then(() => this._schedulePlayStop());
+            return;
+        }
+        if (!audio) {
+            return;
+        }
+        const turn = this._ensurePlayTurn(window);
+        const gen = this._playbackGen;
+        const binary = window.atob(audio);
+        const bytes = ( new Uint8Array(binary.length));
+        for (let i = 0; i < binary.length; i++) {
+            bytes[i] = binary.charCodeAt(i);
+        }
+        const arrayBuf = bytes.buffer;
+        turn.writeChain = turn.writeChain.then(async () => {
+            if (gen !== this._playbackGen) {
+                return;
+            }
+            try {
+                const ctx = this.ensureContext(this._window);
+                const decoded = await ctx.decodeAudioData(arrayBuf);
+                if (gen !== this._playbackGen) {
+                    return;
+                }
+                this._writeToPlayBuffer(decoded);
+                if (!this._playbackTurn?.started) {
+                    this._startPlayback();
+                }
+            } catch (err) {
+                this.logService.error("[voice] TTS decode error", err);
+            }
+        });
+        if (isFinal) {
+            turn.writeChain = turn.writeChain.then(() => this._schedulePlayStop());
+        }
+    }
+    stopPlayback() {
+        this._playbackGen++;
+        if (this._playbackTurn) {
+            this._captureSamples(this._playbackTurn);
+        }
+        try {
+            this._playbackTurn?.sourceNode?.stop();
+        } catch {}
+        this._playbackTurn = null;
+        this._analyserNode = undefined;
+        if (this._isPlaying) {
+            this._isPlaying = false;
+            this._onPlaybackStopped.fire();
+        }
+    }
+    closeContext() {
+        this.stopPlayback();
+        if (this._playbackCtx) {
+            this._playbackCtx.close();
+            this._playbackCtx = undefined;
+        }
+    }
+    _ensurePlayTurn(window) {
+        const ctx = this.ensureContext(window);
+        if (this._playbackTurn) {
+            return this._playbackTurn;
+        }
+        const turn = {
+            buffer: ctx.createBuffer(1, MAX_PLAYBACK_SAMPLES, PLAYBACK_SAMPLE_RATE),
+            sourceNode: null,
+            writeOffset: 0,
+            startCtxTime: 0,
+            started: false,
+            writeChain: Promise.resolve()
+        };
+        this._playbackTurn = turn;
+        return turn;
+    }
+    _writeToPlayBuffer(decoded) {
+        if (!this._playbackTurn) {
+            return;
+        }
+        const src = decoded.getChannelData(0);
+        const dst = this._playbackTurn.buffer.getChannelData(0);
+        const toWrite = Math.min(src.length, MAX_PLAYBACK_SAMPLES - this._playbackTurn.writeOffset);
+        for (let i = 0; i < toWrite; i++) {
+            dst[this._playbackTurn.writeOffset + i] = src[i];
+        }
+        this._playbackTurn.writeOffset += toWrite;
+    }
+    _startPlayback() {
+        const ctx = this._playbackCtx;
+        const turn = this._playbackTurn;
+        if (!ctx || !turn || turn.started) {
+            return;
+        }
+        turn.started = true;
+        const node = ctx.createBufferSource();
+        node.buffer = turn.buffer;
+        turn.sourceNode = node;
+        const analyser = ctx.createAnalyser();
+        analyser.fftSize = 256;
+        node.connect(analyser);
+        analyser.connect(ctx.destination);
+        this._analyserNode = analyser;
+        turn.startCtxTime = ctx.currentTime;
+        node.start(0);
+        if (!this._isPlaying) {
+            this._isPlaying = true;
+            this._onPlaybackStarted.fire();
+        }
+    }
+    _schedulePlayStop() {
+        const ctx = this._playbackCtx;
+        const turn = this._playbackTurn;
+        if (!ctx || !turn) {
+            return;
+        }
+        if (!turn.started) {
+            this._startPlayback();
+        }
+        const node = turn.sourceNode;
+        if (!node) {
+            return;
+        }
+        const stopAt = turn.startCtxTime + turn.writeOffset / PLAYBACK_SAMPLE_RATE;
+        const endedTurn = turn;
+        node.stop(Math.max(stopAt, ctx.currentTime));
+        node.onended = () => {
+            if (this._playbackTurn !== endedTurn) {
+                return;
+            }
+            this._captureSamples(endedTurn);
+            this._playbackTurn = null;
+            this._analyserNode = undefined;
+            if (this._isPlaying) {
+                this._isPlaying = false;
+                this._onPlaybackStopped.fire();
+            }
+        };
+    }
+    _captureSamples(turn) {
+        if (turn.writeOffset > 0) {
+            this._lastPlayedSamples = turn.buffer.getChannelData(0).slice(0, turn.writeOffset);
+        }
+    }
+    dispose() {
+        this.closeContext();
+        super.dispose();
+    }
+};
+TtsPlaybackService = ( __decorate([( __param(0, ILogService))], TtsPlaybackService));
+
+export { TtsPlaybackService };

package/vscode/src/vs/workbench/contrib/chat/browser/voiceClient/voiceClientService.d.ts

@@ -0,0 +1,85 @@
+import { Disposable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle";
+import { Event } from "@codingame/monaco-vscode-api/vscode/vs/base/common/event";
+import { IConfigurationService } from "@codingame/monaco-vscode-api/vscode/vs/platform/configuration/common/configuration.service";
+import { ILogService } from "@codingame/monaco-vscode-api/vscode/vs/platform/log/common/log.service";
+import { IProductService } from "@codingame/monaco-vscode-api/vscode/vs/platform/product/common/productService.service";
+import { IVoicePriorTimelineEntry, IVoiceSessionContext, IVoiceTranscription, IVoiceAudioResponse, IVoiceToolCall, IVoiceSpeechStarted, IVoiceSessionInit, IVoiceFeedbackPayload } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/common/voiceClient/voiceClientService";
+import { IVoiceClientService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/common/voiceClient/voiceClientService.service";
+export declare class VoiceClientService extends Disposable implements IVoiceClientService {
+    private readonly _configurationService;
+    private readonly _logService;
+    private readonly _productService;
+    readonly _serviceBrand: undefined;
+    private _ws;
+    private _reconnectAttempts;
+    private _reconnectStartedAt;
+    private _reconnectTimer;
+    private _isConnected;
+    private _isResuming;
+    private _window;
+    private _lastSessionId;
+    private _pingTimer;
+    private _pongTimer;
+    private _contextSendTimer;
+    private _pendingContext;
+    private _lastSentById;
+    private _lastSentActive;
+    private readonly _onTranscription;
+    readonly onTranscription: Event<IVoiceTranscription>;
+    private readonly _onAudioResponse;
+    readonly onAudioResponse: Event<IVoiceAudioResponse>;
+    private readonly _onToolCall;
+    readonly onToolCall: Event<IVoiceToolCall>;
+    private readonly _onSpeechStarted;
+    readonly onSpeechStarted: Event<IVoiceSpeechStarted>;
+    private readonly _onSessionInit;
+    readonly onSessionInit: Event<IVoiceSessionInit>;
+    private readonly _onError;
+    readonly onError: Event<string>;
+    private readonly _onDidChangeConnectionState;
+    readonly onDidChangeConnectionState: Event<boolean>;
+    get isConnected(): boolean;
+    get isResuming(): boolean;
+    get currentSessionId(): string | undefined;
+    private _authToken;
+    constructor(_configurationService: IConfigurationService, _logService: ILogService, _productService: IProductService);
+    private _getWsUrl;
+    connect(window: Window & typeof globalThis, authToken?: string): Promise<void>;
+    private _connectWebSocket;
+    disconnect(): void;
+    private _cleanup;
+    private _startPing;
+    private _stopPing;
+    private _clearPongTimeout;
+    private _setConnected;
+    sendPttStart(turnId: string): void;
+    sendPttAudioChunk(audio: string): void;
+    sendPttEnd(): void;
+    sendPttDiagnostic(turnId: string, metrics: Record<string, unknown>): void;
+    sendSessionContext(context: IVoiceSessionContext): void;
+    flushSessionContext(): void;
+    invalidateSessionCache(sessionId: string): void;
+    private _sendDelta;
+    private _seedTracking;
+    sendToolResult(callId: string, result: string): void;
+    sendSessionStateChange(sessionId: string, newState: string, _label: string, detail?: string, lastResponseSummary?: string): void;
+    stopSpeaking(): void;
+    /**
+     * Send the start_session message with the given context.
+     * Called by the consumer after connect() resolves and AudioContext is ready.
+     *
+     * ``priorTimeline`` carries an ordered slice of cross-session entries
+     * (voice turns, voice tool calls, coding-session events, and a synthesized
+     * coding-agent-reply summary per active session) from the previous voice
+     * session. The BE consumes it once on the first command turn so the model
+     * can answer recall questions across reconnects without backend
+     * persistence. See ``IVoicePriorTimelineEntry``.
+     */
+    sendStartSession(context: IVoiceSessionContext, machineId: string, priorTimeline?: readonly IVoicePriorTimelineEntry[]): void;
+    sendResumeSession(context: IVoiceSessionContext, machineId: string): void;
+    submitFeedback(payload: IVoiceFeedbackPayload): Promise<{
+        ok: boolean;
+        error?: string;
+    }>;
+    dispose(): void;
+}