@acorex-platform/extension-ai-voice-shell-client 0.0.38 → 0.0.46

package/types/acorex-platform-extension-ai-voice-shell-client-contracts.d.ts

@@ -145,6 +145,70 @@ interface AXPVoiceShellTranscriptTurn {
     text: string;
 }
 
+/** Live media path chosen by VoiceSession:Start. */
+type AXPVoiceLiveMediaMode = 'openrouter' | 'rtc' | 'vendor';
+interface AXPVoiceLiveRtcIceServer {
+    urls: string | string[];
+    username?: string;
+    credential?: string;
+}
+interface AXPVoiceLiveRtcJoin {
+    roomName: string;
+    participantIdentity: string;
+    token: string;
+    serverUrl: string;
+    providerId: string;
+    iceServers?: AXPVoiceLiveRtcIceServer[];
+    forceRelay?: boolean;
+}
+interface AXPVoiceLiveVendorJoin {
+    /** Ephemeral client secret for OpenAI Realtime WebRTC (never the catalog API key). */
+    clientSecret: string;
+    model: string;
+    realtimeUrl: string;
+    transport: 'openai' | 'gemini';
+}
+interface AXPVoiceSessionStartResult {
+    sessionId: string;
+    conversationId: string;
+    assistId: string;
+    mode: AXPVoiceLiveMediaMode;
+    rtc?: AXPVoiceLiveRtcJoin;
+    vendor?: AXPVoiceLiveVendorJoin;
+}
+interface AXPVoiceSessionPushAudioInput {
+    sessionId: string;
+    /** PCM16LE little-endian, standard base64 (not URL-safe). */
+    pcmBase64: string;
+    sampleRate: number;
+}
+interface AXPVoiceOpenRouterAudioPayload {
+    sessionId: string;
+    pcmBase64: string;
+    sampleRate: number;
+}
+interface AXPVoiceSessionStopInput {
+    sessionId?: string;
+}
+interface AXPVoiceSessionInterruptInput {
+    sessionId?: string;
+}
+interface AXPVoiceSessionCommitTurnInput {
+    sessionId: string;
+    role: 'user' | 'assistant';
+    text: string;
+}
+interface AXPVoiceSessionExecuteToolInput {
+    sessionId: string;
+    name: string;
+    argumentsJson?: string;
+}
+interface AXPVoiceSessionExecuteToolResult {
+    output: string;
+}
+declare function axeAiVoiceLiveRtcRoomName(sessionId: string): string;
+declare function axeAiVoiceLiveUserParticipantIdentity(userId: string, sessionId?: string): string;
+
 /** Speech-to-text result from the interaction transport. */
 interface AXPVoiceSttResult {
     text: string;
@@ -172,6 +236,24 @@ interface AXPVoiceInteractionTransport {
 
 /** Runtime command that opens the central voice void dialog. */
 declare const AXE_AI_VOICE_SHELL_OPEN_COMMAND: "AiVoiceShell:Open";
+/** Backend: start a live voice session (Conversation room + RTC or vendor fallback). */
+declare const AXE_AI_VOICE_SHELL_VOICE_SESSION_START_COMMAND: "AiVoiceShell:VoiceSession:Start";
+/** Backend: hang up / stop the live voice session. */
+declare const AXE_AI_VOICE_SHELL_VOICE_SESSION_STOP_COMMAND: "AiVoiceShell:VoiceSession:Stop";
+/** Backend: browser OpenRouter path — push one complete user utterance (PCM16). */
+declare const AXE_AI_VOICE_SHELL_VOICE_SESSION_PUSH_AUDIO_COMMAND: "AiVoiceShell:VoiceSession:PushAudio";
+/** OpenRouter TTS PCM to the browser (not control-plane `ai.voice.session`). */
+declare const AXE_AI_VOICE_OPENROUTER_AUDIO_TOPIC: "ai.voice.openrouter";
+/** Backend: interrupt current assistant speech (barge-in). */
+declare const AXE_AI_VOICE_SHELL_VOICE_SESSION_INTERRUPT_COMMAND: "AiVoiceShell:VoiceSession:Interrupt";
+/** Backend: persist a transcript turn when the client holds the vendor media path. */
+declare const AXE_AI_VOICE_SHELL_VOICE_SESSION_COMMIT_TURN_COMMAND: "AiVoiceShell:VoiceSession:CommitTurn";
+/** Backend: run an Assist tool for a vendor-path live function call. */
+declare const AXE_AI_VOICE_SHELL_VOICE_SESSION_EXECUTE_TOOL_COMMAND: "AiVoiceShell:VoiceSession:ExecuteTool";
+/** Conversation.Message metadata.source for live-voice transcripts (skip text Assist). */
+declare const AXE_AI_VOICE_LIVE_MESSAGE_SOURCE: "voice-live";
+declare const AXE_AI_VOICE_LIVE_RTC_ROOM_PREFIX: "ai-voice-";
+declare const AXE_AI_VOICE_LIVE_AI_PARTICIPANT_IDENTITY: "ai-voice-agent";
 /** Assessment interview capability id (registered by assessment-management). */
 declare const AXE_ASSESSMENT_QUESTIONNAIRE_VOICE_CAPABILITY_ID: "assessment-questionnaire-interview";
 declare const ASSESSMENT_FIND_CASE_FOR_VOICE_FILL_COMMAND: "Assessment:FindCaseForVoiceFill";
@@ -182,5 +264,5 @@ declare const ASSESSMENT_SAVE_QUESTIONNAIRE_DRAFT_COMMAND: "Assessment:SaveQuest
 declare const ASSESSMENT_SUBMIT_QUESTIONNAIRE_COMMAND: "Assessment:SubmitQuestionnaire";
 declare const ASSESSMENT_CANCEL_QUESTIONNAIRE_COMMAND: "Assessment:CancelQuestionnaire";
 
-export { ASSESSMENT_APPLY_QUESTIONNAIRE_ANSWER_COMMAND, ASSESSMENT_CANCEL_QUESTIONNAIRE_COMMAND, ASSESSMENT_FIND_CASE_FOR_VOICE_FILL_COMMAND, ASSESSMENT_NEXT_QUESTIONNAIRE_QUESTION_COMMAND, ASSESSMENT_SAVE_QUESTIONNAIRE_DRAFT_COMMAND, ASSESSMENT_START_HEADLESS_QUESTIONNAIRE_COMMAND, ASSESSMENT_SUBMIT_QUESTIONNAIRE_COMMAND, AXE_AI_VOICE_SHELL_OPEN_COMMAND, AXE_ASSESSMENT_QUESTIONNAIRE_VOICE_CAPABILITY_ID, isAiCompatibleCatalogRow, isAiCompatibleOperationMeta, parseAiCompatibleOperationMeta };
-export type { AXPAiCompatibleExecutionMode, AXPAiCompatibleOperationMeta, AXPAiCompatibleSafetyKind, AXPAiCompatibleSafetyMeta, AXPAiCompatibleVoiceMeta, AXPVoiceApplyResult, AXPVoiceCapabilityIntent, AXPVoiceCapabilityProvider, AXPVoiceInteractionTransport, AXPVoiceNextPromptResult, AXPVoicePrompt, AXPVoiceSaveDraftResult, AXPVoiceShellConfirmRequest, AXPVoiceShellMode, AXPVoiceShellOpenOptions, AXPVoiceShellTranscriptTurn, AXPVoiceShellUiState, AXPVoiceSttResult, AXPVoiceSubmitResult, AXPVoiceTtsResult };
+export { ASSESSMENT_APPLY_QUESTIONNAIRE_ANSWER_COMMAND, ASSESSMENT_CANCEL_QUESTIONNAIRE_COMMAND, ASSESSMENT_FIND_CASE_FOR_VOICE_FILL_COMMAND, ASSESSMENT_NEXT_QUESTIONNAIRE_QUESTION_COMMAND, ASSESSMENT_SAVE_QUESTIONNAIRE_DRAFT_COMMAND, ASSESSMENT_START_HEADLESS_QUESTIONNAIRE_COMMAND, ASSESSMENT_SUBMIT_QUESTIONNAIRE_COMMAND, AXE_AI_VOICE_LIVE_AI_PARTICIPANT_IDENTITY, AXE_AI_VOICE_LIVE_MESSAGE_SOURCE, AXE_AI_VOICE_LIVE_RTC_ROOM_PREFIX, AXE_AI_VOICE_OPENROUTER_AUDIO_TOPIC, AXE_AI_VOICE_SHELL_OPEN_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_COMMIT_TURN_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_EXECUTE_TOOL_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_INTERRUPT_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_PUSH_AUDIO_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_START_COMMAND, AXE_AI_VOICE_SHELL_VOICE_SESSION_STOP_COMMAND, AXE_ASSESSMENT_QUESTIONNAIRE_VOICE_CAPABILITY_ID, axeAiVoiceLiveRtcRoomName, axeAiVoiceLiveUserParticipantIdentity, isAiCompatibleCatalogRow, isAiCompatibleOperationMeta, parseAiCompatibleOperationMeta };
+export type { AXPAiCompatibleExecutionMode, AXPAiCompatibleOperationMeta, AXPAiCompatibleSafetyKind, AXPAiCompatibleSafetyMeta, AXPAiCompatibleVoiceMeta, AXPVoiceApplyResult, AXPVoiceCapabilityIntent, AXPVoiceCapabilityProvider, AXPVoiceInteractionTransport, AXPVoiceLiveMediaMode, AXPVoiceLiveRtcIceServer, AXPVoiceLiveRtcJoin, AXPVoiceLiveVendorJoin, AXPVoiceNextPromptResult, AXPVoiceOpenRouterAudioPayload, AXPVoicePrompt, AXPVoiceSaveDraftResult, AXPVoiceSessionCommitTurnInput, AXPVoiceSessionExecuteToolInput, AXPVoiceSessionExecuteToolResult, AXPVoiceSessionInterruptInput, AXPVoiceSessionPushAudioInput, AXPVoiceSessionStartResult, AXPVoiceSessionStopInput, AXPVoiceShellConfirmRequest, AXPVoiceShellMode, AXPVoiceShellOpenOptions, AXPVoiceShellTranscriptTurn, AXPVoiceShellUiState, AXPVoiceSttResult, AXPVoiceSubmitResult, AXPVoiceTtsResult };

package/types/acorex-platform-extension-ai-voice-shell-client.d.ts

@@ -2,15 +2,15 @@ import * as _angular_core from '@angular/core';
 import { InjectionToken } from '@angular/core';
 import * as i1 from '@acorex-platform/module-ai-management-client';
 import * as i2 from '@acorex-platform/framework-client/core';
-import { AXPVoiceCapabilityProvider, AXPVoiceCapabilityIntent, AXPVoiceShellUiState, AXPVoiceShellMode, AXPVoiceShellConfirmRequest, AXPVoiceShellTranscriptTurn, AXPVoiceShellOpenOptions, AXPVoiceInteractionTransport, AXPVoiceSttResult, AXPVoiceTtsResult } from '@acorex-platform/extension-ai-voice-shell-client/contracts';
+import { AXPVoiceCapabilityProvider, AXPVoiceCapabilityIntent, AXPVoiceShellUiState, AXPVoiceShellOpenOptions, AXPVoiceOpenRouterAudioPayload, AXPVoiceInteractionTransport, AXPVoiceSttResult, AXPVoiceTtsResult } from '@acorex-platform/extension-ai-voice-shell-client/contracts';
+import { AXPRealtimeAiVoiceSessionPayload } from '@acorex-platform/framework-client/realtime';
 import { AXPCommand, AXPCommandExecuteResultPromise } from '@acorex-platform/framework-shared/runtime';
 import { AXBasePageComponent } from '@acorex/components/page';
 import * as _ngrx_signals from '@ngrx/signals';
 import * as _acorex_platform_framework_client_themes_shared from '@acorex-platform/framework-client/themes/shared';
 
 /**
- * Central voice AI command shell (void dialog + capability registry + HTTP/SSE transport).
- * Register next to {@link AXMAiManagementModule}. Domain modules register {@link AXP_VOICE_CAPABILITY_PROVIDER}.
+ * Central live voice AI shell (void dialog + Conversation room + OpenRouter cascade).
  */
 declare class AXEAiVoiceShellExtensionModule {
     static ɵfac: _angular_core.ɵɵFactoryDeclaration<AXEAiVoiceShellExtensionModule, never>;
@@ -34,58 +34,37 @@ declare class AXPVoiceCapabilityRegistry {
     static ɵprov: _angular_core.ɵɵInjectableDeclaration<AXPVoiceCapabilityRegistry>;
 }
 
-/**
- * Session coordinator for the central voice void dialog.
- * Does not import assessment internals — only the capability registry and transport.
- */
 declare class AXPVoiceCommandShellService {
     private readonly popupService;
     private readonly translation;
-    private readonly registry;
-    private readonly transport;
+    private readonly commandService;
+    private readonly media;
     readonly uiState: _angular_core.WritableSignal<AXPVoiceShellUiState>;
-    readonly mode: _angular_core.WritableSignal<AXPVoiceShellMode>;
     readonly statusText: _angular_core.WritableSignal<string>;
     readonly lastTranscript: _angular_core.WritableSignal<string>;
     readonly lastSpoken: _angular_core.WritableSignal<string>;
     readonly errorMessage: _angular_core.WritableSignal<string>;
-    readonly confirmRequest: _angular_core.WritableSignal<AXPVoiceShellConfirmRequest | null>;
-    readonly turns: _angular_core.WritableSignal<AXPVoiceShellTranscriptTurn[]>;
-    readonly interviewProgress: _angular_core.WritableSignal<{
-        answered: number;
-        totalVisible: number;
-    } | null>;
-    private activeCapability;
-    private activeSessionId;
-    private pendingSpokenConfirm;
-    private currentAudio;
-    private currentObjectUrl;
+    readonly conversationId: _angular_core.WritableSignal<string>;
+    readonly sessionId: _angular_core.WritableSignal<string>;
     private openPromise;
-    open(options?: AXPVoiceShellOpenOptions): Promise<void>;
-    enterListening(): Promise<void>;
-    cancelListening(): Promise<void>;
-    reportError(key: string, detail?: string): Promise<void>;
-    submitRecording(audio: Blob): Promise<void>;
-    handleUserText(text: string): Promise<void>;
-    confirmYes(): Promise<void>;
-    confirmNo(): Promise<void>;
-    saveInterviewDraft(): Promise<void>;
-    cancelInterview(): Promise<void>;
-    stopPlayback(): void;
-    private beginCapabilitySession;
-    private speakNextPrompt;
-    private handleNextPrompt;
-    private applyInterviewUtterance;
-    private enterSubmitConfirm;
-    private handleConfirmUtterance;
-    private submitInterview;
-    private runCommandAssist;
-    private speak;
-    private finishSpeaking;
-    private playObjectUrl;
+    private mediaMode;
+    private lastBargeInAt;
+    private speakingStartedAt;
+    open(_options?: AXPVoiceShellOpenOptions): Promise<void>;
+    ensurePlayback(): void;
+    hangUp(_closePopup?: boolean): Promise<void>;
+    interrupt(): Promise<void>;
+    applyRealtime(payload: AXPRealtimeAiVoiceSessionPayload | undefined): void;
+    applyOpenRouterAudio(payload: AXPVoiceOpenRouterAudioPayload | undefined): void;
+    private startLiveSession;
+    private maybeBargeIn;
+    private pushOpenRouterUtterance;
+    private runVendorTool;
+    private applyState;
     private setError;
-    private appendTurn;
-    private resetSession;
+    /** Another tab/profile started a session for this user, or the server stopped us. */
+    private releaseLocalSession;
+    private resetUi;
     static ɵfac: _angular_core.ɵɵFactoryDeclaration<AXPVoiceCommandShellService, never>;
     static ɵprov: _angular_core.ɵɵInjectableDeclaration<AXPVoiceCommandShellService>;
 }
@@ -132,28 +111,13 @@ declare class AXEAiVoiceShellOpenCommand implements AXPCommand<AXEAiVoiceShellOp
 
 declare class AXEAiVoiceVoidDialogComponent extends AXBasePageComponent {
     readonly shell: AXPVoiceCommandShellService;
-    private readonly platformId;
-    private readonly destroyRef;
-    private mediaRecorder?;
-    private audioChunks;
-    private activeStream?;
     readonly orbClass: _angular_core.Signal<string>;
     readonly orbWrapClass: _angular_core.Signal<string>;
-    readonly canListen: _angular_core.Signal<boolean>;
     readonly isListening: _angular_core.Signal<boolean>;
     readonly isSpeaking: _angular_core.Signal<boolean>;
-    readonly showConfirm: _angular_core.Signal<boolean>;
-    readonly progressLabel: _angular_core.Signal<string>;
-    constructor();
-    onMicClick(): Promise<void>;
-    onCancelListen(): Promise<void>;
-    onConfirmYes(): Promise<void>;
-    onConfirmNo(): Promise<void>;
-    onSaveDraft(): Promise<void>;
-    onCancelInterview(): Promise<void>;
-    private startRecording;
-    private stopAndSubmit;
-    private stopTracks;
+    readonly isBusy: _angular_core.Signal<boolean>;
+    onHangUp(): Promise<void>;
+    onOrbClick(): Promise<void>;
     static ɵfac: _angular_core.ɵɵFactoryDeclaration<AXEAiVoiceVoidDialogComponent, never>;
     static ɵcmp: _angular_core.ɵɵComponentDeclaration<AXEAiVoiceVoidDialogComponent, "axe-ai-voice-void-dialog", never, {}, {}, never, never, true, never>;
 }
@@ -167,6 +131,7 @@ declare class AXEAiVoiceShellHeaderSlotComponent {
     private readonly commandExecutor;
     private readonly shortcutRegistry;
     private readonly destroyRef;
+    private readonly media;
     protected readonly themeStore: {
         currentMode: _angular_core.Signal<_acorex_platform_framework_client_themes_shared.AXPThemeMode>;
         currentPalette: _angular_core.Signal<string>;
@@ -184,6 +149,7 @@ declare class AXEAiVoiceShellHeaderSlotComponent {
         sideMenuWidth: _angular_core.Signal<number>;
         sideMenuCompactWidth: _angular_core.Signal<number>;
         menuVerticalMode: _angular_core.Signal<_acorex_platform_framework_client_themes_shared.AXPMenuVerticalMode>;
+        menuToggleCyclesStyle: _angular_core.Signal<boolean>;
         menuOrientation: _angular_core.Signal<_acorex_platform_framework_client_themes_shared.AXPMenuOrientation>;
         menuAppearance: _angular_core.Signal<_acorex_platform_framework_client_themes_shared.AXPMenuAppearance>;
         headerAppearance: _angular_core.Signal<_acorex_platform_framework_client_themes_shared.AXPHeaderAppearance>;
@@ -197,6 +163,7 @@ declare class AXEAiVoiceShellHeaderSlotComponent {
         overlayTaskCount: _angular_core.Signal<number>;
         isNavigationLoading: _angular_core.Signal<boolean>;
         isMenuHorizontal: _angular_core.Signal<boolean>;
+        menuToggleCyclesStyleEnabled: _angular_core.Signal<boolean>;
         effectiveMenuVerticalMode: _angular_core.Signal<_acorex_platform_framework_client_themes_shared.AXPMenuVerticalMode>;
         sideMenuDrawerWidth: _angular_core.Signal<number>;
         menuDarkEnabled: _angular_core.Signal<boolean>;
@@ -211,6 +178,7 @@ declare class AXEAiVoiceShellHeaderSlotComponent {
         setSideMenuWidth: (width: number) => void;
         setSideMenuCompactWidth: (width: number) => void;
         setSideMenuDrawerWidth: (width: number) => void;
+        setMenuVerticalMode: (mode: _acorex_platform_framework_client_themes_shared.AXPMenuVerticalMode) => void;
         toggleSideMenu: () => void;
         setNavigationLoading: (value: boolean, delay?: number) => void;
         setOverlayLoading: (value: boolean, delay?: number) => void;
@@ -233,6 +201,7 @@ declare class AXEAiVoiceShellHeaderSlotComponent {
         sideMenuWidth: number;
         sideMenuCompactWidth: number;
         menuVerticalMode: _acorex_platform_framework_client_themes_shared.AXPMenuVerticalMode;
+        menuToggleCyclesStyle: boolean;
         menuOrientation: _acorex_platform_framework_client_themes_shared.AXPMenuOrientation;
         menuAppearance: _acorex_platform_framework_client_themes_shared.AXPMenuAppearance;
         headerAppearance: _acorex_platform_framework_client_themes_shared.AXPHeaderAppearance;