@codingame/monaco-vscode-api 36.2.7 → 37.0.0

package/vscode/src/vs/workbench/contrib/chat/browser/speechToText/dictationSession.js

@@ -0,0 +1,408 @@
+
+import { registerCss } from '../../../../../../../../css.js';
+import * as dictationSession from './media/dictationSession.css';
+import { status } from '../../../../../base/browser/ui/aria/aria.js';
+import { Emitter } from '../../../../../base/common/event.js';
+import { DisposableStore, toDisposable } from '../../../../../base/common/lifecycle.js';
+import { EditorOption } from '../../../../../editor/common/config/editorOptions.js';
+import { TrackedRangeStickiness } from '../../../../../editor/common/model.js';
+import { Position } from '../../../../../editor/common/core/position.js';
+import { Range } from '../../../../../editor/common/core/range.js';
+import { Selection } from '../../../../../editor/common/core/selection.js';
+import { localize } from '../../../../../nls.js';
+import { ChatSpeechToTextState } from './chatSpeechToTextService.js';
+
+registerCss(dictationSession);
+const INTERIM_PROCESSING_CLASS = "dictation-interim-processing";
+const LOG_PREFIX = "[chat-stt-dictation]";
+const MAX_DICTATION_DURATION_MS = 20 * 60 * 1000;
+function isRecording(service) {
+    return service.state === ChatSpeechToTextState.Recording;
+}
+class LiveTranscriptInserter {
+    constructor(_editor, _logService) {
+        this._editor = _editor;
+        this._logService = _logService;
+        this._needsLeadingSpace = false;
+        this._finalized = false;
+        this._isApplyingEdit = false;
+        this._userModified = false;
+        this._lastCumulativeText = "";
+        this._committedText = "";
+    }
+    update(fullText, interim = true) {
+        this._logService.trace(
+            `${LOG_PREFIX} inserter.update interim=${interim} finalized=${this._finalized} userModified=${this._userModified} len=${fullText.length}`
+        );
+        if (this._finalized && interim) {
+            this._logService.trace(`${LOG_PREFIX} inserter.update ignored (already finalized)`);
+            return;
+        }
+        if (!interim) {
+            this._finalized = true;
+        }
+        const model = this._editor.getModel();
+        if (!model) {
+            this._logService.trace(`${LOG_PREFIX} inserter.update no model`);
+            return;
+        }
+        this._lastCumulativeText = fullText;
+        let renderText = fullText;
+        if (this._committedText) {
+            const committedLength = this._committedText.length;
+            renderText = fullText.slice(committedLength);
+            renderText = renderText.replace(/^\s+/, "");
+            if (renderText.length === 0) {
+                this._logService.trace(`${LOG_PREFIX} inserter.update nothing new after user edit`);
+                return;
+            }
+        }
+        if (!this._anchor) {
+            const selection = this._editor.getSelection() ?? model.getFullModelRange().collapseToEnd();
+            const start = selection.getStartPosition();
+            this._anchor = start;
+            this._end = start;
+            this._revertAnchor ??= start;
+            this._needsLeadingSpace = start.column > 1 && !/\s$/.test(model.getValueInRange(( new Range(
+                start.lineNumber,
+                Math.max(1, start.column - 1),
+                start.lineNumber,
+                start.column
+            ))));
+        }
+        const text = (this._needsLeadingSpace ? " " : "") + renderText;
+        const replaceRange = Range.fromPositions(this._anchor, this._end ?? this._anchor);
+        const lines = text.split("\n");
+        const endLine = this._anchor.lineNumber + lines.length - 1;
+        const endColumn = lines.length === 1 ? this._anchor.column + lines[0].length : lines[lines.length - 1].length + 1;
+        this._end = ( new Position(endLine, endColumn));
+        const caret = this._end;
+        this._isApplyingEdit = true;
+        try {
+            this._editor.executeEdits("chatSpeechToText", [{
+                range: replaceRange,
+                text,
+                forceMoveMarkers: true
+            }], [Selection.fromPositions(caret)]);
+        } finally {
+            this._isApplyingEdit = false;
+        }
+        this._updateInterimDecorations(interim);
+    }
+    onDidChangeModelContent(event) {
+        if (this._isApplyingEdit || !this._anchor || !this._end) {
+            return;
+        }
+        const affectsTranscript = ( event.changes.some(change => Position.isBeforeOrEqual(( new Position(change.range.startLineNumber, change.range.startColumn)), this._end)));
+        if (!affectsTranscript) {
+            return;
+        }
+        this._logService.trace(`${LOG_PREFIX} transcript invalidated by user edit`);
+        this._userModified = true;
+        this._committedText = this._lastCumulativeText;
+        this._revertAnchor ??= this._anchor;
+        this._revertEnd = this._end;
+        this._anchor = undefined;
+        this._end = undefined;
+        this.clearInterimDecorations();
+    }
+    _updateInterimDecorations(interim) {
+        if (!interim || !this._anchor || !this._end || Position.equals(this._anchor, this._end)) {
+            this._logService.trace(`${LOG_PREFIX} interim decorations clear (interim=${interim})`);
+            this._processingDecorations?.clear();
+            return;
+        }
+        this._processingDecorations ??= this._editor.createDecorationsCollection();
+        this._logService.trace(
+            `${LOG_PREFIX} interim decorations ${this._anchor.lineNumber}:${this._anchor.column} -> ${this._end.lineNumber}:${this._end.column}`
+        );
+        this._processingDecorations.set([{
+            range: Range.fromPositions(this._anchor, this._end),
+            options: {
+                description: "chatSpeechToText-interim",
+                inlineClassName: INTERIM_PROCESSING_CLASS
+            }
+        }]);
+    }
+    clearInterimDecorations() {
+        this._logService.trace(`${LOG_PREFIX} clearInterimDecorations`);
+        this._processingDecorations?.clear();
+    }
+    beginFinalize() {
+        this._logService.trace(`${LOG_PREFIX} beginFinalize`);
+        this._finalized = true;
+        this._processingDecorations?.clear();
+    }
+    finalizedRange() {
+        if (this._userModified || !this._anchor || !this._end) {
+            return undefined;
+        }
+        const start = this._needsLeadingSpace ? ( new Position(this._anchor.lineNumber, this._anchor.column + 1)) : this._anchor;
+        return Range.fromPositions(start, this._end);
+    }
+    revert() {
+        this._processingDecorations?.clear();
+        const model = this._editor.getModel();
+        const anchor = this._revertAnchor ?? this._anchor;
+        const end = this._end ?? this._revertEnd;
+        if (!model || !anchor || !end) {
+            return;
+        }
+        this._editor.executeEdits("chatSpeechToText", [{
+            range: Range.fromPositions(anchor, end),
+            text: "",
+            forceMoveMarkers: true
+        }]);
+        this._editor.setPosition(anchor);
+        this._anchor = undefined;
+        this._end = undefined;
+        this._revertAnchor = undefined;
+        this._revertEnd = undefined;
+    }
+}
+let _active;
+const _onDidChangeDictationEditor = ( new Emitter());
+const onDidChangeDictationEditor = _onDidChangeDictationEditor.event;
+function setActiveDictation(active) {
+    if (_active !== active) {
+        _active = active;
+        _onDidChangeDictationEditor.fire();
+    }
+}
+let _finalizing;
+function setFinalizingDictation(finalizing) {
+    if (_finalizing !== finalizing) {
+        _finalizing = finalizing;
+        _onDidChangeDictationEditor.fire();
+    }
+}
+function isDictating() {
+    return !!_active;
+}
+function activeDictationEditor() {
+    return _active?.editor;
+}
+function isDictationActiveForEditor(editor) {
+    return _active?.editor === editor || _finalizing?.editor === editor;
+}
+async function startDictation(service, editor, window, logService, surface = "chat") {
+    if (_active?.editor === editor) {
+        return;
+    }
+    if (_active || service.isBusy) {
+        await service.cancel();
+    }
+    if (service.state !== ChatSpeechToTextState.Idle) {
+        return;
+    }
+    const inserter = ( new LiveTranscriptInserter(editor, logService));
+    const disposables = ( new DisposableStore());
+    const HIDE_CURSOR_CLASS = "dictation-hide-cursor";
+    editor.getDomNode()?.classList.add(HIDE_CURSOR_CLASS);
+    disposables.add(
+        toDisposable(() => editor.getDomNode()?.classList.remove(HIDE_CURSOR_CLASS))
+    );
+    const previousPlaceholder = editor.getOption(EditorOption.placeholder);
+    const listeningPlaceholder = ( localize(8990, "Listening…"));
+    let appliedPlaceholder;
+    const applyPlaceholder = () => {
+        if (!editor.getModel()) {
+            return;
+        }
+        const recording = service.state === ChatSpeechToTextState.Recording;
+        const desired = recording && !service.isPreparingModel ? listeningPlaceholder : undefined;
+        if (desired !== undefined) {
+            if (appliedPlaceholder !== desired) {
+                editor.updateOptions({
+                    placeholder: desired
+                });
+                appliedPlaceholder = desired;
+            }
+        } else if (appliedPlaceholder !== undefined) {
+            editor.updateOptions({
+                placeholder: previousPlaceholder
+            });
+            appliedPlaceholder = undefined;
+        }
+    };
+    disposables.add(toDisposable(() => {
+        inserter.clearInterimDecorations();
+        if (!editor.getModel() || appliedPlaceholder === undefined) {
+            return;
+        }
+        editor.updateOptions({
+            placeholder: previousPlaceholder
+        });
+        appliedPlaceholder = undefined;
+    }));
+    disposables.add(service.onDidUpdateTranscript(update => {
+        logService.trace(
+            `${LOG_PREFIX} onDidUpdateTranscript len=${update.text.length} finalized=${update.finalizedText.length} state=${service.state}`
+        );
+        if (!service.showTranscriptWhileDictating) {
+            inserter.clearInterimDecorations();
+            return;
+        }
+        inserter.update(update.text);
+    }));
+    disposables.add(
+        editor.onDidChangeModelContent(event => inserter.onDidChangeModelContent(event))
+    );
+    disposables.add(service.onDidChangePreparingModel(() => applyPlaceholder()));
+    disposables.add(service.onDidChangeState(state => {
+        logService.trace(`${LOG_PREFIX} onDidChangeState ${state}`);
+        if (state === ChatSpeechToTextState.Idle && _active?.service === service) {
+            setActiveDictation(undefined);
+            disposables.dispose();
+            return;
+        }
+        applyPlaceholder();
+    }));
+    disposables.add(editor.onDidDispose(() => cancelDictation()));
+    const activeDictation = {
+        service,
+        editor,
+        inserter,
+        disposables,
+        logService,
+        surface
+    };
+    setActiveDictation(activeDictation);
+    try {
+        await service.start(window, surface);
+        if (_active === activeDictation && isRecording(service)) {
+            const durationLimit = window.setTimeout(() => {
+                logService.info(`${LOG_PREFIX} stopping after maximum duration`);
+                status(( localize(8991, "Dictation stopped after 20 minutes.")));
+                void stopDictation();
+            }, MAX_DICTATION_DURATION_MS);
+            disposables.add(toDisposable(() => window.clearTimeout(durationLimit)));
+        }
+    } catch {
+        if (_active === activeDictation) {
+            setActiveDictation(undefined);
+        }
+        disposables.dispose();
+    }
+}
+async function stopDictation() {
+    const active = _active;
+    if (!active) {
+        await _finalizing?.promise;
+        return;
+    }
+    setActiveDictation(undefined);
+    const promise = finalizeDictation(active);
+    setFinalizingDictation({
+        editor: active.editor,
+        promise
+    });
+    try {
+        await promise;
+    } finally {
+        if (_finalizing?.promise === promise) {
+            setFinalizingDictation(undefined);
+        }
+    }
+}
+async function finalizeDictation(active) {
+    active.logService.trace(`${LOG_PREFIX} stopDictation begin, state=${active.service.state}`);
+    active.inserter.beginFinalize();
+    try {
+        const text = await active.service.stopAndTranscribe({
+            preserveLiveTranscript: active.service.showTranscriptWhileDictating
+        });
+        active.logService.trace(
+            `${LOG_PREFIX} stopAndTranscribe resolved text=${text === undefined ? "undefined" : `len=${text.length}`}`
+        );
+        if (text !== undefined) {
+            active.inserter.update(text, false);
+            trackDictationAccuracy(active, text);
+        } else {
+            active.inserter.clearInterimDecorations();
+        }
+    } finally {
+        active.logService.trace(`${LOG_PREFIX} stopDictation dispose`);
+        active.disposables.dispose();
+        active.editor.focus();
+    }
+}
+async function stopDictationForEditor(editor) {
+    if (_active?.editor === editor) {
+        await stopDictation();
+    } else if (_finalizing?.editor === editor) {
+        await _finalizing.promise;
+    }
+}
+function cancelDictation() {
+    const active = _active;
+    if (!active) {
+        return;
+    }
+    setActiveDictation(undefined);
+    active.inserter.revert();
+    active.disposables.dispose();
+    void active.service.cancel();
+}
+const _accuracyTrackers = ( new Set());
+function notifyDictationSubmitted(editor) {
+    for (const tracker of [..._accuracyTrackers]) {
+        if (tracker.editor === editor) {
+            tracker.measure(true);
+        }
+    }
+}
+function trackDictationAccuracy(active, dictatedText) {
+    const {
+        editor,
+        inserter,
+        service,
+        surface
+    } = active;
+    const model = editor.getModel();
+    const range = inserter.finalizedRange();
+    if (!model || !range || !dictatedText) {
+        return;
+    }
+    const backend = service.currentBackend;
+    const collection = editor.createDecorationsCollection([{
+        range,
+        options: {
+            description: "chatSpeechToText-accuracy",
+            stickiness: TrackedRangeStickiness.NeverGrowsWhenTypingAtEdges
+        }
+    }]);
+    const store = ( new DisposableStore());
+    let measured = false;
+    const tracker = {
+        editor,
+        measure(submitted) {
+            if (measured) {
+                return;
+            }
+            measured = true;
+            const current = collection.getRange(0);
+            const submittedText = current ? model.getValueInRange(current) : "";
+            service.logDictationAccuracy({
+                dictatedText,
+                submittedText,
+                backend,
+                surface,
+                submitted
+            });
+            collection.clear();
+            store.dispose();
+            _accuracyTrackers.delete(tracker);
+        }
+    };
+    store.add(model.onDidChangeContent(() => {
+        if (model.getValueLength() === 0) {
+            tracker.measure(false);
+        }
+    }));
+    store.add(model.onWillDispose(() => tracker.measure(false)));
+    store.add(editor.onDidDispose(() => tracker.measure(false)));
+    _accuracyTrackers.add(tracker);
+}
+
+export { activeDictationEditor, cancelDictation, isDictating, isDictationActiveForEditor, notifyDictationSubmitted, onDidChangeDictationEditor, startDictation, stopDictation, stopDictationForEditor };

package/vscode/src/vs/workbench/contrib/chat/browser/speechToText/media/dictationSession.css

@@ -0,0 +1,22 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/* While a dictation session is active the caret is parked at the end of the
+ * dictated region, so a blinking cursor there is distracting as transcript text
+ * streams in. Hide it for the duration of the session. */
+.monaco-editor.dictation-hide-cursor .cursors-layer {
+	display: none !important;
+}
+
+/* The not-yet-final transcript is rendered in the input's placeholder color to
+ * signal it is provisional; the decoration is removed once the transcript is
+ * finalized, leaving solid text.
+ *
+ * The selector has to out-specify Monaco's syntax-token rules (`.monaco-editor
+ * .mtkN`), which are injected into the document after this stylesheet and would
+ * otherwise win the tie and paint the text in the normal foreground color. */
+.monaco-editor .view-line span.dictation-interim-processing {
+	color: var(--vscode-input-placeholderForeground);
+}

package/vscode/src/vs/workbench/contrib/chat/browser/speechToText/voiceCodeTranscriptionClient.service.d.ts

@@ -0,0 +1,16 @@
+import { Event } from "../../../../../base/common/event.js";
+import { IVoiceCodeTranscription, IVoiceCodeTranscriptionError } from "@codingame/monaco-vscode-chat-service-override/vscode/vs/workbench/contrib/chat/browser/speechToText/voiceCodeTranscriptionClient";
+export declare const IVoiceCodeTranscriptionClient: import("../../../../../platform/instantiation/common/instantiation.js").ServiceIdentifier<IVoiceCodeTranscriptionClient>;
+export interface IVoiceCodeTranscriptionClient {
+    readonly _serviceBrand: undefined;
+    readonly onTranscription: Event<IVoiceCodeTranscription>;
+    readonly onError: Event<IVoiceCodeTranscriptionError>;
+    readonly onDidClose: Event<number>;
+    readonly isConnected: boolean;
+    connect(window: Window & typeof globalThis, authToken: string): Promise<void>;
+    startSession(): Promise<void>;
+    sendPttStart(turnId: string): void;
+    sendPttAudioChunk(turnId: string, audio: string): void;
+    sendPttEnd(turnId: string): void;
+    disconnect(): void;
+}

package/vscode/src/vs/workbench/contrib/chat/browser/speechToText/voiceCodeTranscriptionClient.service.js

@@ -0,0 +1,6 @@
+
+import { createDecorator } from '../../../../../platform/instantiation/common/instantiation.js';
+
+const IVoiceCodeTranscriptionClient = ( createDecorator("voiceCodeTranscriptionClient"));
+
+export { IVoiceCodeTranscriptionClient };

package/vscode/src/vs/workbench/contrib/chat/browser/voiceClient/micCaptureService.d.ts

@@ -0,0 +1,139 @@
+import { Disposable } from "../../../../../base/common/lifecycle.js";
+import { Event } from "../../../../../base/common/event.js";
+import { IStorageService } from "../../../../../platform/storage/common/storage.service.js";
+import { INotificationService } from "../../../../../platform/notification/common/notification.service.js";
+import { ILogService } from "../../../../../platform/log/common/log.service.js";
+import { IMicCaptureService } from "./micCaptureService.service.js";
+export declare function getMediaCaptureWindow(targetWindow: Window & typeof globalThis): Window & typeof globalThis;
+/** Number of samples buffered per 32 ms voice capture chunk at 16 kHz, matching one Silero VAD frame. */
+export declare const MIC_CAPTURE_CHUNK_SIZE = 512;
+export declare function isMicrophonePermissionDeniedError(error: unknown): boolean;
+/**
+ * Per-PTT-press diagnostic emitted after `pttUp` once the diagnostic
+ * window closes. Logged + sent to backend so we can correlate frontend
+ * audio bookkeeping with backend ASR results via `turnId`.
+ *
+ * Drain model: after `pttUp` the service keeps streaming audio chunks
+ * for a fixed "drain window" (~500ms by default). The drain ends as
+ * soon as it has shipped enough samples to cover the window (or a
+ * fallback timer trips if `onaudioprocess` stops firing). Only AFTER
+ * the drain has closed does `_onPttEnd` fire. The diagnostic window is
+ * intentionally LONGER than the drain window so any audio still
+ * produced after drain end (within the diagnostic window) is counted
+ * as `postReleaseCallbacks` -- a direct signal that the drain window
+ * is too short for this device/load and the fix needs to extend it.
+ *
+ * Field interpretation:
+ *  - `drainChunks` / `drainSamples` => audio captured during the drain
+ *    window and shipped to the backend. Non-zero in normal operation.
+ *  - `postReleaseCallbacks > 0` => the WebAudio pipeline produced more
+ *    audio AFTER the drain window closed but before the diagnostic
+ *    window. This audio was DROPPED; if it happens often the drain
+ *    window needs to grow.
+ *  - `drainSkippedBy*` > 0 => the drain was muted or AEC-suppressed.
+ *    Tail audio for that press was lost; investigate the mute / AEC
+ *    suppression path rather than the drain window.
+ *  - `pttUpWithoutCapture` => pttUp arrived while mic was not capturing.
+ *  - `releasedDuringAcquire` => user released while mic was still being
+ *    acquired; no audio was ever recorded for this press.
+ */
+export interface IPttDiagnostic {
+    readonly turnId: string;
+    readonly msHeld: number;
+    readonly chunksSent: number;
+    readonly samplesSent: number;
+    readonly drainFired: boolean;
+    readonly drainChunks: number;
+    readonly drainSamples: number;
+    readonly drainWindowMs: number;
+    readonly drainSkippedByMute: number;
+    readonly drainSkippedBySuppression: number;
+    readonly postReleaseCallbacks: number;
+    readonly postReleaseSamples: number;
+    readonly postReleaseSkippedByMute: number;
+    readonly postReleaseSkippedBySuppression: number;
+    readonly postReleaseWindowMs: number;
+    readonly releasedDuringAcquire: boolean;
+    readonly pttUpWithoutCapture: boolean;
+}
+export declare class MicCaptureService extends Disposable implements IMicCaptureService {
+    private readonly storageService;
+    private readonly notificationService;
+    private readonly logService;
+    readonly _serviceBrand: undefined;
+    constructor(storageService: IStorageService, notificationService: INotificationService, logService: ILogService);
+    private _window;
+    private _micStream;
+    private _micCtx;
+    private _workletNode;
+    private _analyserNode;
+    private _isCapturing;
+    private _captureGeneration;
+    private _capturePromise;
+    private _pttGeneration;
+    private _pttHeld;
+    private _pttStreaming;
+    private _isMuted;
+    private _suppressUntilTs;
+    private _pttAcquiring;
+    private _pttReleasedDuringAcquire;
+    private readonly _micTrackListeners;
+    private _micMutedNotified;
+    private static readonly _PTT_DRAIN_WINDOW_MS;
+    private _pttDrainTargetSamples;
+    private _pttDrainSamplesSent;
+    private _pttDrainFallbackTimer;
+    private static readonly _DIAG_POST_RELEASE_WINDOW_MS;
+    private _diagTurnId;
+    private _diagPttDownTs;
+    private _diagPttUpTs;
+    private _diagChunksSent;
+    private _diagSamplesSent;
+    private _diagDrainFired;
+    private _diagDrainChunks;
+    private _diagDrainSamples;
+    private _diagDrainSkippedByMute;
+    private _diagDrainSkippedBySuppression;
+    private _diagPostReleaseCallbacks;
+    private _diagPostReleaseSamples;
+    private _diagPostReleaseSkippedByMute;
+    private _diagPostReleaseSkippedBySuppression;
+    private _diagReleasedDuringAcquire;
+    private _diagPttUpWithoutCapture;
+    private _diagFireTimer;
+    private readonly _onPttStart;
+    readonly onPttStart: Event<boolean>;
+    private readonly _onPttAudioChunk;
+    readonly onPttAudioChunk: Event<string>;
+    private readonly _onPttEnd;
+    readonly onPttEnd: Event<void>;
+    private readonly _onPttDiagnostic;
+    readonly onPttDiagnostic: Event<IPttDiagnostic>;
+    get isCapturing(): boolean;
+    get analyserNode(): AnalyserNode | undefined;
+    get isMuted(): boolean;
+    set isMuted(value: boolean);
+    suppressUntil(timestamp: number): void;
+    protected getMediaCaptureWindow(targetWindow: Window & typeof globalThis): Window & typeof globalThis;
+    prepare(window: Window & typeof globalThis): void;
+    pttDown(turnId: string, passive?: boolean): Promise<void>;
+    pttUp(): void;
+    abortPtt(): void;
+    startCapture(window: Window & typeof globalThis): Promise<void>;
+    private _startCapture;
+    private _notifyMicPermissionDenied;
+    private _notifyMicrophoneMuted;
+    private _stopCaptureResources;
+    stopCapture(): void;
+    dispose(): void;
+    /**
+     * End the post-release drain phase: stop accepting more audio for
+     * this turn and fire `_onPttEnd`. Idempotent. Safe to call when no
+     * drain is in progress.
+     */
+    private _finishDrain;
+    private _resetDiagnosticCounters;
+    private _scheduleDiagnosticFire;
+    private _flushPendingDiagnostic;
+    private _emitDiagnostic;
+}