@nextpa/chat-core 0.1.0

package/dist/core/voice-state-manager.js

@@ -0,0 +1,317 @@
+import { logger } from '../utils/logger';
+import { getToolStatusLabel, getSynthesizingStatus, resolveToolCallState } from '../utils/tool-status';
+import { startThinkingAudio, stopThinkingAudio } from '../services/thinking-audio-pulse';
+const STOP_CONVERSATION_TIMEOUT_MS = 30_000;
+export class VoiceStateManager {
+    isUserSpeaking = false;
+    isAiThinking = false;
+    isMuted = false;
+    isMicrophoneReady = true;
+    isMicrophoneBlocked = false;
+    isTranscribing = false;
+    transcribingContent = '';
+    isStreaming = false;
+    isAssistantSpeaking = false;
+    streamContent = '';
+    agentStatus = null;
+    audioLevel = 0;
+    audioLevelLastTime = 0;
+    isDelegationActive = false;
+    connectionManager = null;
+    apiClient = null;
+    logPrefix = '';
+    language;
+    latestUsage = null;
+    accumulatedUsage = null;
+    pendingToolCalls = [];
+    thinkingStartedAt = null;
+    pendingStreamingUpdate = false;
+    toolAbortController = null;
+    onStateChanged;
+    constructor(onStateChanged) {
+        this.onStateChanged = onStateChanged;
+    }
+    setupEventHandlers(deps) {
+        this.connectionManager = deps.connectionManager;
+        this.apiClient = deps.apiClient;
+        this.logPrefix = deps.logPrefix;
+        this.setupSpeechHandlers(deps);
+        this.setupConnectionHandlers(deps);
+        this.setupActionHandlers(deps);
+    }
+    setLanguage(language) {
+        this.language = language;
+    }
+    toggleMute() {
+        if (!this.connectionManager)
+            return;
+        this.isMuted = !this.isMuted;
+        this.connectionManager.setMuted(this.isMuted);
+        this.onStateChanged();
+    }
+    /**
+     * User pressed stop while the assistant was busy. If a backend tool (e.g. a
+     * long web research) is in flight, abort its HTTP call so the server cancels
+     * it (RequestAborted) and let the model wrap up gracefully. Otherwise the
+     * assistant is thinking/talking — interrupt the response and drop the audio.
+     */
+    stop() {
+        if (this.toolAbortController) {
+            this.toolAbortController.abort();
+            return;
+        }
+        this.connectionManager?.interrupt();
+        this.isStreaming = false;
+        this.isAssistantSpeaking = false;
+        this.isAiThinking = false;
+        this.streamContent = '';
+        this.agentStatus = null;
+        this.isDelegationActive = false;
+        stopThinkingAudio();
+        this.onStateChanged();
+    }
+    /**
+     * The assistant transcript completes before its audio finishes playing.
+     * Hold the speaking flag until playback truly ends (audioDone + playback
+     * monitor) so the stop button stays visible while the user can still hear
+     * the assistant.
+     */
+    async trackAssistantPlayback() {
+        await this.connectionManager?.waitForResponseComplete();
+        this.isAssistantSpeaking = false;
+        this.onStateChanged();
+    }
+    resetState() {
+        this.isUserSpeaking = false;
+        this.isAiThinking = false;
+        this.isMicrophoneReady = true;
+        this.isTranscribing = false;
+        this.transcribingContent = '';
+        this.isStreaming = false;
+        this.isAssistantSpeaking = false;
+        this.streamContent = '';
+        this.agentStatus = null;
+        this.audioLevel = 0;
+        this.audioLevelLastTime = 0;
+        this.isDelegationActive = false;
+        stopThinkingAudio();
+    }
+    /**
+     * Asymmetric envelope follower over a dBFS-mapped input. Two stages:
+     *
+     * 1. dBFS perceptual map — raw RMS (0..1) lives mostly under 0.1 so a
+     *    linear halo would barely move at normal volumes. Mapping through
+     *    20·log10(level) and clamping to roughly -50..0 dBFS spreads the
+     *    normal-speech range across the visual.
+     * 2. One-pole IIR with split time constants — 10 ms attack so syllables
+     *    pop, 150 ms release so the avatar doesn't twitch off between words.
+     *
+     * Sweet spots picked from common voice-visualisation guidance (attack
+     * 5–50 ms, release 100–300 ms, dBFS floor -60 .. -50, ceiling -10 .. 0).
+     */
+    smoothAudioLevel(level) {
+        const now = performance.now();
+        const dt = this.audioLevelLastTime ? Math.max(0, (now - this.audioLevelLastTime) / 1000) : 0.016;
+        this.audioLevelLastTime = now;
+        const db = 20 * Math.log10(Math.max(level, 0.001));
+        const perceptual = Math.max(0, Math.min(1, (db + 50) / 50));
+        const tau = perceptual > this.audioLevel ? 0.01 : 0.15;
+        const alpha = 1 - Math.exp(-dt / tau);
+        return this.audioLevel + alpha * (perceptual - this.audioLevel);
+    }
+    prepareForConnection() {
+        this.isMicrophoneBlocked = false;
+        this.isMuted = false;
+        this.isMicrophoneReady = false;
+    }
+    setupSpeechHandlers({ connectionManager, chatManager, persistence }) {
+        connectionManager.on('usageUpdated', (usage) => {
+            this.accumulateUsage(usage);
+        });
+        connectionManager.on('userSpeechStart', () => {
+            this.isUserSpeaking = true;
+            this.isAiThinking = false;
+            this.isAssistantSpeaking = false;
+            this.isTranscribing = false;
+            this.transcribingContent = '';
+            this.onStateChanged();
+        });
+        connectionManager.on('userSpeechStop', () => {
+            this.isUserSpeaking = false;
+            this.isAiThinking = true;
+            this.thinkingStartedAt = Date.now();
+            this.onStateChanged();
+        });
+        connectionManager.on('userSpeechEnd', () => {
+            this.isUserSpeaking = false;
+            this.onStateChanged();
+        });
+        connectionManager.on('audioLevel', (level) => {
+            this.audioLevel = this.smoothAudioLevel(level);
+            this.onStateChanged();
+        });
+        connectionManager.on('transcriptDelta', ({ isUser, delta }) => {
+            if (isUser) {
+                this.isTranscribing = true;
+                this.transcribingContent += delta;
+            }
+            else {
+                if (!this.isDelegationActive) {
+                    this.agentStatus = null;
+                }
+                this.isStreaming = true;
+                this.isAiThinking = false;
+                this.streamContent += delta;
+                if (!this.isAssistantSpeaking) {
+                    this.isAssistantSpeaking = true;
+                    void this.trackAssistantPlayback();
+                }
+            }
+            this.scheduleStreamingUpdate();
+        });
+        connectionManager.on('transcriptReceived', ({ isUser, text }) => {
+            if (isUser) {
+                this.isTranscribing = false;
+                this.transcribingContent = '';
+                chatManager.addUserMessage(text);
+                persistence.persistMessage('User', text, this.language, undefined, undefined, undefined, 'Spoken');
+                this.onStateChanged();
+                return;
+            }
+            this.isAiThinking = false;
+            this.isStreaming = false;
+            this.streamContent = '';
+            chatManager.addAssistantMessage(text);
+            const usage = this.latestUsage;
+            this.latestUsage = null;
+            this.accumulatedUsage = null;
+            const toolCalls = this.pendingToolCalls.length > 0 ? [...this.pendingToolCalls] : undefined;
+            this.pendingToolCalls = [];
+            const responseLatencyMs = this.thinkingStartedAt
+                ? Math.round(Date.now() - this.thinkingStartedAt)
+                : undefined;
+            this.thinkingStartedAt = null;
+            persistence.persistMessage('Assistant', text, this.language, usage ?? undefined, toolCalls, responseLatencyMs);
+        });
+    }
+    setupConnectionHandlers({ connectionManager, logPrefix }) {
+        connectionManager.on('connectionStateChanged', () => this.onStateChanged());
+        connectionManager.on('error', (err) => {
+            logger.error(`${logPrefix} Connection error:`, err);
+            this.onStateChanged();
+        });
+        connectionManager.on('microphoneBlocked', () => {
+            logger.warn(`${logPrefix} Microphone permission blocked by browser`);
+            this.isMicrophoneBlocked = true;
+            this.onStateChanged();
+        });
+        connectionManager.on('microphoneReady', () => {
+            this.isMicrophoneReady = true;
+            this.onStateChanged();
+        });
+    }
+    setupActionHandlers({ connectionManager, navigationManager, onStopConversation }) {
+        connectionManager.on('functionCall', (call) => this.handleFunctionCall(call));
+        connectionManager.on('toolCall', ({ toolName, status }) => this.handleToolCall(toolName, status));
+        connectionManager.on('stopConversation', () => this.handleStopConversation(onStopConversation));
+        connectionManager.on('openUrl', (url) => navigationManager.openUrl(url));
+        connectionManager.on('openFile', (url) => navigationManager.openUrl(url));
+    }
+    handleToolCall(toolName, status) {
+        logger.debug(`${this.logPrefix} Tool call:`, toolName, status);
+        const previouslyActive = this.isDelegationActive;
+        const state = resolveToolCallState(toolName, status, this.isDelegationActive, this.language);
+        this.agentStatus = state.agentStatus;
+        this.isDelegationActive = state.isDelegationActive;
+        if (state.isDelegationActive && !previouslyActive) {
+            startThinkingAudio();
+        }
+        else if (!state.isDelegationActive && previouslyActive) {
+            stopThinkingAudio();
+        }
+        this.onStateChanged();
+    }
+    async handleFunctionCall(call) {
+        if (!this.connectionManager || !this.apiClient)
+            return;
+        logger.debug(`${this.logPrefix} Executing tool:`, call.name);
+        this.agentStatus = getToolStatusLabel(call.name, this.language);
+        this.isAiThinking = true;
+        this.onStateChanged();
+        startThinkingAudio();
+        const controller = new AbortController();
+        this.toolAbortController = controller;
+        const startTime = Date.now();
+        let succeeded = true;
+        try {
+            const result = await this.apiClient.executeTool(call.name, call.arguments, controller.signal);
+            await this.connectionManager.sendFunctionResult(call.callId, call.name, result);
+            logger.debug(`${this.logPrefix} Tool executed successfully:`, call.name);
+        }
+        catch (err) {
+            succeeded = false;
+            const errorMessage = err instanceof Error ? err.message : 'Tool execution failed';
+            const resultMsg = controller.signal.aborted
+                ? 'The user stopped this action. Abandon the current task and briefly ask how you can help instead.'
+                : `Error: ${errorMessage}`;
+            logger.debug(`${this.logPrefix} Tool ended:`, call.name, controller.signal.aborted ? 'aborted by user' : err);
+            try {
+                await this.connectionManager.sendFunctionResult(call.callId, call.name, resultMsg);
+            }
+            catch { /* ignore errors when sending result */ }
+        }
+        finally {
+            if (this.toolAbortController === controller) {
+                this.toolAbortController = null;
+            }
+        }
+        this.pendingToolCalls.push({
+            name: call.name,
+            durationMs: Math.round(Date.now() - startTime),
+            succeeded,
+        });
+        stopThinkingAudio();
+        this.agentStatus = getSynthesizingStatus(this.language);
+        this.onStateChanged();
+    }
+    async handleStopConversation(onDisconnect) {
+        if (!this.connectionManager)
+            return;
+        logger.debug(`${this.logPrefix} StopConversation received, muting and waiting for AI to finish...`);
+        this.connectionManager.setMuted(true);
+        await this.connectionManager.waitForResponseComplete(STOP_CONVERSATION_TIMEOUT_MS);
+        logger.debug(`${this.logPrefix} AI finished speaking, disconnecting`);
+        await onDisconnect();
+    }
+    accumulateUsage(usage) {
+        if (this.accumulatedUsage) {
+            this.accumulatedUsage = {
+                provider: usage.provider,
+                model: usage.model,
+                inputTokens: this.accumulatedUsage.inputTokens + usage.inputTokens,
+                outputTokens: this.accumulatedUsage.outputTokens + usage.outputTokens,
+                totalTokens: this.accumulatedUsage.totalTokens + usage.totalTokens,
+                cachedTokens: (this.accumulatedUsage.cachedTokens ?? 0) + (usage.cachedTokens ?? 0) || undefined,
+                textCachedInputTokens: (this.accumulatedUsage.textCachedInputTokens ?? 0) + (usage.textCachedInputTokens ?? 0) || undefined,
+                audioCachedInputTokens: (this.accumulatedUsage.audioCachedInputTokens ?? 0) + (usage.audioCachedInputTokens ?? 0) || undefined,
+                audioInputTokens: (this.accumulatedUsage.audioInputTokens ?? 0) + (usage.audioInputTokens ?? 0) || undefined,
+                audioOutputTokens: (this.accumulatedUsage.audioOutputTokens ?? 0) + (usage.audioOutputTokens ?? 0) || undefined,
+                textInputTokens: (this.accumulatedUsage.textInputTokens ?? 0) + (usage.textInputTokens ?? 0) || undefined,
+                textOutputTokens: (this.accumulatedUsage.textOutputTokens ?? 0) + (usage.textOutputTokens ?? 0) || undefined,
+            };
+        }
+        else {
+            this.accumulatedUsage = usage;
+        }
+        this.latestUsage = this.accumulatedUsage;
+    }
+    scheduleStreamingUpdate() {
+        if (this.pendingStreamingUpdate)
+            return;
+        this.pendingStreamingUpdate = true;
+        requestAnimationFrame(() => {
+            this.pendingStreamingUpdate = false;
+            this.onStateChanged();
+        });
+    }
+}

package/dist/core/widget-controller.d.ts

@@ -0,0 +1,58 @@
+import { EventEmitter } from './event-emitter';
+import type { WidgetState, WidgetControllerOptions } from './types';
+import type { ConnectOptions } from '../types';
+interface WidgetControllerEvents {
+    stateChanged: (state: WidgetState) => void;
+}
+export declare class WidgetController extends EventEmitter<WidgetControllerEvents> {
+    private readonly core;
+    private readonly voiceConnectionManager;
+    private readonly voiceState;
+    private readonly textMode;
+    private readonly retryHandler;
+    private readonly sessionBeacon;
+    private currentMode;
+    private pendingConnectMode;
+    private isReconnecting;
+    private showReconnecting;
+    private voiceRecoveryAttempts;
+    private greetingOverride;
+    private get uiSoundsEnabled();
+    constructor(options: WidgetControllerOptions);
+    initialize(): Promise<void>;
+    connect(options?: ConnectOptions): Promise<void>;
+    private greetingEnabled;
+    private shouldSkipVoiceGreeting;
+    connectVoice(): Promise<void>;
+    connectText(skipGreeting?: boolean, waitForGreeting?: boolean): Promise<void>;
+    disconnect(trimFarewell?: boolean): Promise<void>;
+    endCall(): Promise<void>;
+    resetConversation(): Promise<void>;
+    submitSessionFeedback(rating: number): void;
+    submitFeedback(messageId: string, rating: number | undefined): void;
+    sendMessage(text: string): Promise<void>;
+    toggleMute(): void;
+    stopVoiceResponse(): void;
+    transcribe(audio: Blob, signal?: AbortSignal): Promise<string>;
+    toggleMinimized(): void;
+    setMinimized(minimized: boolean): void;
+    acceptTerms(): void;
+    declineTerms(): void;
+    dismissBlockedUrl(): void;
+    openBlockedUrl(): void;
+    dismissPopover(): void;
+    getState(): WidgetState;
+    dispose(): void;
+    private notifyStateChanged;
+    private getActiveConnectionState;
+    private handleRetryResult;
+    private handleSessionResumed;
+    private disconnectCurrentMode;
+    private handleUnexpectedDisconnect;
+    private handleVoiceUnhealthy;
+    private handleVoiceStopConversation;
+    private handleTextStopConversation;
+    private updateDocumentProviders;
+    private setupExtensionBridge;
+}
+export {};