@codingame/monaco-vscode-chat-service-override 34.1.3 → 35.0.0

package/vscode/src/vs/workbench/contrib/chat/browser/voiceClient/micCaptureService.d.ts

@@ -0,0 +1,125 @@
+import { Disposable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle";
+import { Event } from "@codingame/monaco-vscode-api/vscode/vs/base/common/event";
+import { IStorageService } from "@codingame/monaco-vscode-api/vscode/vs/platform/storage/common/storage.service";
+import { INotificationService } from "@codingame/monaco-vscode-api/vscode/vs/platform/notification/common/notification.service";
+import { ILogService } from "@codingame/monaco-vscode-api/vscode/vs/platform/log/common/log.service";
+import { IMicCaptureService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/voiceClient/micCaptureService.service";
+/**
+ * Per-PTT-press diagnostic emitted after `pttUp` once the diagnostic
+ * window closes. Logged + sent to backend so we can correlate frontend
+ * audio bookkeeping with backend ASR results via `turnId`.
+ *
+ * Drain model: after `pttUp` the service keeps streaming audio chunks
+ * for a fixed "drain window" (~500ms by default). The drain ends as
+ * soon as it has shipped enough samples to cover the window (or a
+ * fallback timer trips if `onaudioprocess` stops firing). Only AFTER
+ * the drain has closed does `_onPttEnd` fire. The diagnostic window is
+ * intentionally LONGER than the drain window so any audio still
+ * produced after drain end (within the diagnostic window) is counted
+ * as `postReleaseCallbacks` -- a direct signal that the drain window
+ * is too short for this device/load and the fix needs to extend it.
+ *
+ * Field interpretation:
+ *  - `drainChunks` / `drainSamples` => audio captured during the drain
+ *    window and shipped to the backend. Non-zero in normal operation.
+ *  - `postReleaseCallbacks > 0` => the WebAudio pipeline produced more
+ *    audio AFTER the drain window closed but before the diagnostic
+ *    window. This audio was DROPPED; if it happens often the drain
+ *    window needs to grow.
+ *  - `drainSkippedBy*` > 0 => the drain was muted or AEC-suppressed.
+ *    Tail audio for that press was lost; investigate the mute / AEC
+ *    suppression path rather than the drain window.
+ *  - `pttUpWithoutCapture` => pttUp arrived while mic was not capturing.
+ *  - `releasedDuringAcquire` => user released while mic was still being
+ *    acquired; no audio was ever recorded for this press.
+ */
+export interface IPttDiagnostic {
+    readonly turnId: string;
+    readonly msHeld: number;
+    readonly chunksSent: number;
+    readonly samplesSent: number;
+    readonly drainFired: boolean;
+    readonly drainChunks: number;
+    readonly drainSamples: number;
+    readonly drainWindowMs: number;
+    readonly drainSkippedByMute: number;
+    readonly drainSkippedBySuppression: number;
+    readonly postReleaseCallbacks: number;
+    readonly postReleaseSamples: number;
+    readonly postReleaseSkippedByMute: number;
+    readonly postReleaseSkippedBySuppression: number;
+    readonly postReleaseWindowMs: number;
+    readonly releasedDuringAcquire: boolean;
+    readonly pttUpWithoutCapture: boolean;
+}
+export declare class MicCaptureService extends Disposable implements IMicCaptureService {
+    private readonly storageService;
+    private readonly notificationService;
+    private readonly logService;
+    readonly _serviceBrand: undefined;
+    constructor(storageService: IStorageService, notificationService: INotificationService, logService: ILogService);
+    private _window;
+    private _micStream;
+    private _micCtx;
+    private _scriptNode;
+    private _analyserNode;
+    private _isCapturing;
+    private _pttHeld;
+    private _pttStreaming;
+    private _isMuted;
+    private _suppressUntilTs;
+    private _pttAcquiring;
+    private _pttReleasedDuringAcquire;
+    private static readonly _PTT_DRAIN_WINDOW_MS;
+    private _pttDrainTargetSamples;
+    private _pttDrainSamplesSent;
+    private _pttDrainFallbackTimer;
+    private static readonly _DIAG_POST_RELEASE_WINDOW_MS;
+    private _diagTurnId;
+    private _diagPttDownTs;
+    private _diagPttUpTs;
+    private _diagChunksSent;
+    private _diagSamplesSent;
+    private _diagDrainFired;
+    private _diagDrainChunks;
+    private _diagDrainSamples;
+    private _diagDrainSkippedByMute;
+    private _diagDrainSkippedBySuppression;
+    private _diagPostReleaseCallbacks;
+    private _diagPostReleaseSamples;
+    private _diagPostReleaseSkippedByMute;
+    private _diagPostReleaseSkippedBySuppression;
+    private _diagReleasedDuringAcquire;
+    private _diagPttUpWithoutCapture;
+    private _diagFireTimer;
+    private readonly _onPttStart;
+    readonly onPttStart: Event<void>;
+    private readonly _onPttAudioChunk;
+    readonly onPttAudioChunk: Event<string>;
+    private readonly _onPttEnd;
+    readonly onPttEnd: Event<void>;
+    private readonly _onPttDiagnostic;
+    readonly onPttDiagnostic: Event<IPttDiagnostic>;
+    get isCapturing(): boolean;
+    get analyserNode(): AnalyserNode | undefined;
+    get isMuted(): boolean;
+    set isMuted(value: boolean);
+    suppressUntil(timestamp: number): void;
+    prepare(window: Window & typeof globalThis): void;
+    pttDown(turnId: string): Promise<void>;
+    pttUp(): void;
+    startCapture(window: Window & typeof globalThis): Promise<void>;
+    private _notifyMicPermissionDenied;
+    stopCapture(): void;
+    dispose(): void;
+    /**
+     * End the post-release drain phase: stop accepting more audio for
+     * this turn and fire `_onPttEnd`. Idempotent. Safe to call when no
+     * drain is in progress.
+     */
+    private _finishDrain;
+    private _resetDiagnosticCounters;
+    private _scheduleDiagnosticFire;
+    private _flushPendingDiagnostic;
+    private _emitDiagnostic;
+}

package/vscode/src/vs/workbench/contrib/chat/browser/voiceClient/micCaptureService.js

@@ -0,0 +1,399 @@
+
+import { __decorate, __param } from '@codingame/monaco-vscode-api/external/tslib/tslib.es6';
+import { Disposable } from '@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle';
+import { Emitter } from '@codingame/monaco-vscode-api/vscode/vs/base/common/event';
+import '@codingame/monaco-vscode-api/vscode/vs/platform/instantiation/common/extensions';
+import { StorageScope } from '@codingame/monaco-vscode-api/vscode/vs/platform/storage/common/storage';
+import { IStorageService } from '@codingame/monaco-vscode-api/vscode/vs/platform/storage/common/storage.service';
+import '@codingame/monaco-vscode-api/vscode/vs/platform/notification/common/notification';
+import { INotificationService } from '@codingame/monaco-vscode-api/vscode/vs/platform/notification/common/notification.service';
+import { ILogService } from '@codingame/monaco-vscode-api/vscode/vs/platform/log/common/log.service';
+import { localize } from '@codingame/monaco-vscode-api/vscode/vs/nls';
+import { AgentsVoiceStorageKeys } from '../../../agentsVoice/common/agentsVoice.js';
+import '@codingame/monaco-vscode-api/vscode/vs/platform/instantiation/common/instantiation';
+import Severity from '@codingame/monaco-vscode-api/vscode/vs/base/common/severity';
+
+var MicCaptureService_1;
+let MicCaptureService = class MicCaptureService extends Disposable {
+    static {
+        MicCaptureService_1 = this;
+    }
+    constructor(storageService, notificationService, logService) {
+        super();
+        this.storageService = storageService;
+        this.notificationService = notificationService;
+        this.logService = logService;
+        this._micStream = null;
+        this._isCapturing = false;
+        this._pttHeld = false;
+        this._pttStreaming = false;
+        this._isMuted = false;
+        this._suppressUntilTs = 0;
+        this._pttAcquiring = false;
+        this._pttReleasedDuringAcquire = false;
+        this._pttDrainTargetSamples = 0;
+        this._pttDrainSamplesSent = 0;
+        this._diagTurnId = "";
+        this._diagPttDownTs = 0;
+        this._diagPttUpTs = 0;
+        this._diagChunksSent = 0;
+        this._diagSamplesSent = 0;
+        this._diagDrainFired = false;
+        this._diagDrainChunks = 0;
+        this._diagDrainSamples = 0;
+        this._diagDrainSkippedByMute = 0;
+        this._diagDrainSkippedBySuppression = 0;
+        this._diagPostReleaseCallbacks = 0;
+        this._diagPostReleaseSamples = 0;
+        this._diagPostReleaseSkippedByMute = 0;
+        this._diagPostReleaseSkippedBySuppression = 0;
+        this._diagReleasedDuringAcquire = false;
+        this._diagPttUpWithoutCapture = false;
+        this._onPttStart = this._register(( new Emitter()));
+        this.onPttStart = this._onPttStart.event;
+        this._onPttAudioChunk = this._register(( new Emitter()));
+        this.onPttAudioChunk = this._onPttAudioChunk.event;
+        this._onPttEnd = this._register(( new Emitter()));
+        this.onPttEnd = this._onPttEnd.event;
+        this._onPttDiagnostic = this._register(( new Emitter()));
+        this.onPttDiagnostic = this._onPttDiagnostic.event;
+    }
+    static {
+        this._PTT_DRAIN_WINDOW_MS = 500;
+    }
+    static {
+        this._DIAG_POST_RELEASE_WINDOW_MS = 1000;
+    }
+    get isCapturing() {
+        return this._isCapturing;
+    }
+    get analyserNode() {
+        return this._analyserNode;
+    }
+    get isMuted() {
+        return this._isMuted;
+    }
+    set isMuted(value) {
+        this._isMuted = value;
+    }
+    suppressUntil(timestamp) {
+        this._suppressUntilTs = timestamp;
+    }
+    prepare(window) {
+        this._window = window;
+    }
+    async pttDown(turnId) {
+        if (this._pttHeld) {
+            return;
+        }
+        this._finishDrain();
+        this._flushPendingDiagnostic();
+        this._resetDiagnosticCounters(turnId);
+        this._pttHeld = true;
+        this._pttStreaming = true;
+        this._pttReleasedDuringAcquire = false;
+        this._isMuted = false;
+        if (this._isCapturing) {
+            this._onPttStart.fire();
+            return;
+        }
+        if (!this._window) {
+            return;
+        }
+        if (this._pttAcquiring) {
+            return;
+        }
+        this._pttAcquiring = true;
+        try {
+            await this.startCapture(this._window);
+        } catch (err) {
+            this._pttHeld = false;
+            this._pttStreaming = false;
+            this._pttAcquiring = false;
+            this._pttReleasedDuringAcquire = false;
+            throw err;
+        }
+        this._pttAcquiring = false;
+        this._onPttStart.fire();
+        if (this._pttReleasedDuringAcquire) {
+            this._pttReleasedDuringAcquire = false;
+            this._pttStreaming = false;
+            this._diagReleasedDuringAcquire = true;
+            this._onPttEnd.fire();
+            this.stopCapture();
+            this._scheduleDiagnosticFire();
+        }
+    }
+    pttUp() {
+        if (!this._pttHeld) {
+            return;
+        }
+        if (this._pttAcquiring) {
+            this._pttReleasedDuringAcquire = true;
+            this._diagReleasedDuringAcquire = true;
+            this._diagPttUpTs = Date.now();
+            this._scheduleDiagnosticFire();
+            return;
+        }
+        if (!this._isCapturing) {
+            this._pttHeld = false;
+            this._pttStreaming = false;
+            this._diagPttUpWithoutCapture = true;
+            this._diagPttUpTs = Date.now();
+            this._scheduleDiagnosticFire();
+            return;
+        }
+        this._pttHeld = false;
+        this._diagPttUpTs = Date.now();
+        const sampleRate = this._micCtx?.sampleRate ?? 16000;
+        this._pttDrainTargetSamples = Math.ceil(sampleRate * MicCaptureService_1._PTT_DRAIN_WINDOW_MS / 1000);
+        this._pttDrainSamplesSent = 0;
+        this._pttDrainFallbackTimer = setTimeout(() => {
+            this._pttDrainFallbackTimer = undefined;
+            this._finishDrain();
+        }, MicCaptureService_1._PTT_DRAIN_WINDOW_MS + 250);
+        this._scheduleDiagnosticFire();
+    }
+    async startCapture(window) {
+        this._window = window;
+        if (this._isCapturing) {
+            return;
+        }
+        const deviceId = this.storageService.get(AgentsVoiceStorageKeys.MicrophoneDevice, StorageScope.APPLICATION);
+        const audioConstraints = {
+            channelCount: 1,
+            sampleRate: 16000,
+            echoCancellation: true,
+            noiseSuppression: true
+        };
+        if (deviceId) {
+            audioConstraints.deviceId = {
+                exact: deviceId
+            };
+        }
+        let micStream;
+        try {
+            micStream = await window.navigator.mediaDevices.getUserMedia({
+                audio: audioConstraints
+            });
+        } catch (err) {
+            const isDeviceError = deviceId && err instanceof DOMException && (err.name === "OverconstrainedError" || err.name === "NotFoundError");
+            if (isDeviceError) {
+                this.logService.warn(
+                    `[mic] Preferred device ${deviceId.slice(0, 8)}… unavailable, falling back to default`
+                );
+                delete audioConstraints.deviceId;
+                try {
+                    micStream = await window.navigator.mediaDevices.getUserMedia({
+                        audio: audioConstraints
+                    });
+                } catch (retryErr) {
+                    this._notifyMicPermissionDenied(retryErr);
+                    throw retryErr;
+                }
+            } else {
+                this._notifyMicPermissionDenied(err);
+                throw err;
+            }
+        }
+        this._micStream = micStream;
+        if (!this._micCtx) {
+            this._micCtx = new window.AudioContext({
+                sampleRate: 16000
+            });
+        }
+        const ctx = this._micCtx;
+        const source = ctx.createMediaStreamSource(micStream);
+        const analyser = ctx.createAnalyser();
+        analyser.fftSize = 256;
+        source.connect(analyser);
+        this._analyserNode = analyser;
+        const processor = ctx.createScriptProcessor(2048, 1, 1);
+        this._scriptNode = processor;
+        processor.onaudioprocess = e => {
+            const nowTs = Date.now();
+            const ptUpTs = this._diagPttUpTs;
+            const isDrainCallback = this._pttStreaming && !this._pttHeld;
+            const inDiagWindow = ptUpTs > 0 && !this._pttHeld && nowTs <= ptUpTs + MicCaptureService_1._DIAG_POST_RELEASE_WINDOW_MS;
+            const isPostReleaseCallback = !this._pttStreaming && inDiagWindow;
+            if (this._isMuted) {
+                if (isDrainCallback) {
+                    this._diagDrainSkippedByMute++;
+                }
+                if (isPostReleaseCallback) {
+                    this._diagPostReleaseSkippedByMute++;
+                }
+                return;
+            }
+            if (nowTs < this._suppressUntilTs) {
+                if (isDrainCallback) {
+                    this._diagDrainSkippedBySuppression++;
+                }
+                if (isPostReleaseCallback) {
+                    this._diagPostReleaseSkippedBySuppression++;
+                }
+                return;
+            }
+            if (!this._pttStreaming) {
+                if (isPostReleaseCallback) {
+                    this._diagPostReleaseCallbacks++;
+                    this._diagPostReleaseSamples += e.inputBuffer.length;
+                }
+                return;
+            }
+            const channelData = e.inputBuffer.getChannelData(0);
+            const samples = ( new Float32Array(channelData));
+            const b64 = encodeRawPcm16Base64(samples, this._window);
+            this._diagChunksSent++;
+            this._diagSamplesSent += samples.length;
+            if (isDrainCallback) {
+                this._diagDrainFired = true;
+                this._diagDrainChunks++;
+                this._diagDrainSamples += samples.length;
+                this._pttDrainSamplesSent += samples.length;
+            }
+            this._onPttAudioChunk.fire(b64);
+            if (isDrainCallback && this._pttDrainSamplesSent >= this._pttDrainTargetSamples) {
+                this._finishDrain();
+            }
+        };
+        source.connect(processor);
+        processor.connect(ctx.destination);
+        this._isCapturing = true;
+    }
+    _notifyMicPermissionDenied(err) {
+        if (err instanceof DOMException && err.name === "NotAllowedError") {
+            this.notificationService.notify({
+                severity: Severity.Error,
+                message: ( localize(
+                    7762,
+                    "Microphone access was denied. Grant microphone permission in your system settings to use Voice Mode."
+                ))
+            });
+        }
+    }
+    stopCapture() {
+        if (this._pttDrainFallbackTimer) {
+            clearTimeout(this._pttDrainFallbackTimer);
+            this._pttDrainFallbackTimer = undefined;
+        }
+        this._pttDrainTargetSamples = 0;
+        this._pttDrainSamplesSent = 0;
+        if (this._scriptNode) {
+            this._scriptNode.disconnect();
+            this._scriptNode = undefined;
+        }
+        this._analyserNode = undefined;
+        this._micCtx?.close();
+        this._micCtx = undefined;
+        if (this._micStream) {
+            this._micStream.getTracks().forEach(t => t.stop());
+            this._micStream = null;
+        }
+        this._isCapturing = false;
+        this._pttHeld = false;
+        this._pttStreaming = false;
+        this._pttReleasedDuringAcquire = false;
+    }
+    dispose() {
+        if (this._diagFireTimer) {
+            clearTimeout(this._diagFireTimer);
+            this._diagFireTimer = undefined;
+        }
+        if (this._pttDrainFallbackTimer) {
+            clearTimeout(this._pttDrainFallbackTimer);
+            this._pttDrainFallbackTimer = undefined;
+        }
+        this.stopCapture();
+        super.dispose();
+    }
+    _finishDrain() {
+        if (this._pttDrainFallbackTimer) {
+            clearTimeout(this._pttDrainFallbackTimer);
+            this._pttDrainFallbackTimer = undefined;
+        }
+        this._pttDrainTargetSamples = 0;
+        this._pttDrainSamplesSent = 0;
+        if (this._pttStreaming && !this._pttHeld) {
+            this._pttStreaming = false;
+            this._onPttEnd.fire();
+        }
+    }
+    _resetDiagnosticCounters(turnId) {
+        this._diagTurnId = turnId;
+        this._diagPttDownTs = Date.now();
+        this._diagPttUpTs = 0;
+        this._diagChunksSent = 0;
+        this._diagSamplesSent = 0;
+        this._diagDrainFired = false;
+        this._diagDrainChunks = 0;
+        this._diagDrainSamples = 0;
+        this._diagDrainSkippedByMute = 0;
+        this._diagDrainSkippedBySuppression = 0;
+        this._diagPostReleaseCallbacks = 0;
+        this._diagPostReleaseSamples = 0;
+        this._diagPostReleaseSkippedByMute = 0;
+        this._diagPostReleaseSkippedBySuppression = 0;
+        this._diagReleasedDuringAcquire = false;
+        this._diagPttUpWithoutCapture = false;
+    }
+    _scheduleDiagnosticFire() {
+        if (this._diagFireTimer) {
+            clearTimeout(this._diagFireTimer);
+            this._diagFireTimer = undefined;
+        }
+        this._diagFireTimer = setTimeout(() => {
+            this._diagFireTimer = undefined;
+            this._emitDiagnostic();
+        }, MicCaptureService_1._DIAG_POST_RELEASE_WINDOW_MS);
+    }
+    _flushPendingDiagnostic() {
+        if (this._diagFireTimer) {
+            clearTimeout(this._diagFireTimer);
+            this._diagFireTimer = undefined;
+            this._emitDiagnostic();
+        }
+    }
+    _emitDiagnostic() {
+        if (!this._diagTurnId && this._diagPttDownTs === 0) {
+            return;
+        }
+        const msHeld = this._diagPttUpTs > 0 ? this._diagPttUpTs - this._diagPttDownTs : 0;
+        this._onPttDiagnostic.fire({
+            turnId: this._diagTurnId,
+            msHeld,
+            chunksSent: this._diagChunksSent,
+            samplesSent: this._diagSamplesSent,
+            drainFired: this._diagDrainFired,
+            drainChunks: this._diagDrainChunks,
+            drainSamples: this._diagDrainSamples,
+            drainWindowMs: MicCaptureService_1._PTT_DRAIN_WINDOW_MS,
+            drainSkippedByMute: this._diagDrainSkippedByMute,
+            drainSkippedBySuppression: this._diagDrainSkippedBySuppression,
+            postReleaseCallbacks: this._diagPostReleaseCallbacks,
+            postReleaseSamples: this._diagPostReleaseSamples,
+            postReleaseSkippedByMute: this._diagPostReleaseSkippedByMute,
+            postReleaseSkippedBySuppression: this._diagPostReleaseSkippedBySuppression,
+            postReleaseWindowMs: MicCaptureService_1._DIAG_POST_RELEASE_WINDOW_MS,
+            releasedDuringAcquire: this._diagReleasedDuringAcquire,
+            pttUpWithoutCapture: this._diagPttUpWithoutCapture
+        });
+    }
+};
+MicCaptureService = MicCaptureService_1 = ( __decorate([( __param(0, IStorageService)), ( __param(1, INotificationService)), ( __param(2, ILogService))], MicCaptureService));
+function encodeRawPcm16Base64(samples, win) {
+    const buf = ( new ArrayBuffer(samples.length * 2));
+    const view = ( new DataView(buf));
+    for (let i = 0; i < samples.length; i++) {
+        const s = Math.max(-1, Math.min(1, samples[i]));
+        view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
+    }
+    const bytes = ( new Uint8Array(buf));
+    let binaryStr = "";
+    for (let i = 0; i < bytes.length; i++) {
+        binaryStr += String.fromCharCode(bytes[i]);
+    }
+    return win.btoa(binaryStr);
+}
+
+export { MicCaptureService };

package/vscode/src/vs/workbench/contrib/chat/browser/voiceClient/ttsPlaybackService.d.ts

@@ -0,0 +1,34 @@
+import { Disposable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle";
+import { Event } from "@codingame/monaco-vscode-api/vscode/vs/base/common/event";
+import { ILogService } from "@codingame/monaco-vscode-api/vscode/vs/platform/log/common/log.service";
+import { ITtsPlaybackService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/voiceClient/ttsPlaybackService.service";
+export declare class TtsPlaybackService extends Disposable implements ITtsPlaybackService {
+    private readonly logService;
+    readonly _serviceBrand: undefined;
+    constructor(logService: ILogService);
+    private _window;
+    private _playbackCtx;
+    private _playbackTurn;
+    private _playbackGen;
+    private _analyserNode;
+    private _isPlaying;
+    private _lastPlayedSamples;
+    private readonly _onPlaybackStarted;
+    readonly onPlaybackStarted: Event<void>;
+    private readonly _onPlaybackStopped;
+    readonly onPlaybackStopped: Event<void>;
+    get isPlaying(): boolean;
+    get analyserNode(): AnalyserNode | undefined;
+    getLastPlayedSamples(): Float32Array | null;
+    ensureContext(window: Window & typeof globalThis): AudioContext;
+    playAudioChunk(audio: string, isFinal: boolean, window: Window & typeof globalThis): void;
+    stopPlayback(): void;
+    /** Close the AudioContext entirely (for full teardown). */
+    closeContext(): void;
+    private _ensurePlayTurn;
+    private _writeToPlayBuffer;
+    private _startPlayback;
+    private _schedulePlayStop;
+    private _captureSamples;
+    dispose(): void;
+}