@nextpa/chat-core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +53 -0
  3. package/dist/constants/defaults.d.ts +6 -0
  4. package/dist/constants/defaults.js +69 -0
  5. package/dist/core/chat-manager.d.ts +17 -0
  6. package/dist/core/chat-manager.js +39 -0
  7. package/dist/core/config-manager.d.ts +14 -0
  8. package/dist/core/config-manager.js +75 -0
  9. package/dist/core/connection-manager.d.ts +68 -0
  10. package/dist/core/connection-manager.js +273 -0
  11. package/dist/core/connection-retry-handler.d.ts +11 -0
  12. package/dist/core/connection-retry-handler.js +33 -0
  13. package/dist/core/consent-manager.d.ts +15 -0
  14. package/dist/core/consent-manager.js +40 -0
  15. package/dist/core/conversation-persistence.d.ts +23 -0
  16. package/dist/core/conversation-persistence.js +109 -0
  17. package/dist/core/event-emitter.d.ts +7 -0
  18. package/dist/core/event-emitter.js +27 -0
  19. package/dist/core/greeting-gate.d.ts +16 -0
  20. package/dist/core/greeting-gate.js +53 -0
  21. package/dist/core/index.d.ts +9 -0
  22. package/dist/core/index.js +7 -0
  23. package/dist/core/link-target.d.ts +15 -0
  24. package/dist/core/link-target.js +64 -0
  25. package/dist/core/local-storage-persistence.d.ts +11 -0
  26. package/dist/core/local-storage-persistence.js +111 -0
  27. package/dist/core/navigation-manager.d.ts +24 -0
  28. package/dist/core/navigation-manager.js +132 -0
  29. package/dist/core/popover-manager.d.ts +14 -0
  30. package/dist/core/popover-manager.js +36 -0
  31. package/dist/core/session-beacon.d.ts +11 -0
  32. package/dist/core/session-beacon.js +60 -0
  33. package/dist/core/text-connection-manager.d.ts +58 -0
  34. package/dist/core/text-connection-manager.js +200 -0
  35. package/dist/core/text-mode-manager.d.ts +50 -0
  36. package/dist/core/text-mode-manager.js +154 -0
  37. package/dist/core/text-types.d.ts +20 -0
  38. package/dist/core/text-types.js +1 -0
  39. package/dist/core/text-widget-controller.d.ts +47 -0
  40. package/dist/core/text-widget-controller.js +278 -0
  41. package/dist/core/theme-manager.d.ts +10 -0
  42. package/dist/core/theme-manager.js +25 -0
  43. package/dist/core/types.d.ts +50 -0
  44. package/dist/core/types.js +1 -0
  45. package/dist/core/voice-state-manager.d.ts +85 -0
  46. package/dist/core/voice-state-manager.js +317 -0
  47. package/dist/core/widget-controller.d.ts +58 -0
  48. package/dist/core/widget-controller.js +533 -0
  49. package/dist/core/widget-core.d.ts +51 -0
  50. package/dist/core/widget-core.js +160 -0
  51. package/dist/google.d.ts +1 -0
  52. package/dist/google.js +12 -0
  53. package/dist/index.d.ts +37 -0
  54. package/dist/index.js +40 -0
  55. package/dist/openai.d.ts +1 -0
  56. package/dist/openai.js +7 -0
  57. package/dist/services/api.d.ts +114 -0
  58. package/dist/services/api.js +336 -0
  59. package/dist/services/index.d.ts +3 -0
  60. package/dist/services/index.js +3 -0
  61. package/dist/services/page-interaction/constants.d.ts +1 -0
  62. package/dist/services/page-interaction/constants.js +1 -0
  63. package/dist/services/page-interaction/content-extractor.d.ts +2 -0
  64. package/dist/services/page-interaction/content-extractor.js +104 -0
  65. package/dist/services/page-interaction/deep-query.d.ts +8 -0
  66. package/dist/services/page-interaction/deep-query.js +44 -0
  67. package/dist/services/page-interaction/dom-executor.d.ts +7 -0
  68. package/dist/services/page-interaction/dom-executor.js +172 -0
  69. package/dist/services/page-interaction/dom-extractor.d.ts +8 -0
  70. package/dist/services/page-interaction/dom-extractor.js +252 -0
  71. package/dist/services/page-interaction/extension-bridge.d.ts +10 -0
  72. package/dist/services/page-interaction/extension-bridge.js +86 -0
  73. package/dist/services/page-interaction/iframe-navigation-manager.d.ts +28 -0
  74. package/dist/services/page-interaction/iframe-navigation-manager.js +154 -0
  75. package/dist/services/page-interaction/index.d.ts +5 -0
  76. package/dist/services/page-interaction/index.js +4 -0
  77. package/dist/services/page-interaction/page-interaction-handler.d.ts +21 -0
  78. package/dist/services/page-interaction/page-interaction-handler.js +129 -0
  79. package/dist/services/page-interaction/selector-generator.d.ts +1 -0
  80. package/dist/services/page-interaction/selector-generator.js +48 -0
  81. package/dist/services/page-interaction/types.d.ts +75 -0
  82. package/dist/services/page-interaction/types.js +1 -0
  83. package/dist/services/realtime/google/google-audio.d.ts +44 -0
  84. package/dist/services/realtime/google/google-audio.js +410 -0
  85. package/dist/services/realtime/google/google-provider.d.ts +47 -0
  86. package/dist/services/realtime/google/google-provider.js +465 -0
  87. package/dist/services/realtime/google/google-tool-mapper.d.ts +3 -0
  88. package/dist/services/realtime/google/google-tool-mapper.js +70 -0
  89. package/dist/services/realtime/google/index.d.ts +1 -0
  90. package/dist/services/realtime/google/index.js +1 -0
  91. package/dist/services/realtime/openai/index.d.ts +1 -0
  92. package/dist/services/realtime/openai/index.js +1 -0
  93. package/dist/services/realtime/openai/openai-audio-monitor.d.ts +1 -0
  94. package/dist/services/realtime/openai/openai-audio-monitor.js +55 -0
  95. package/dist/services/realtime/openai/openai-provider.d.ts +50 -0
  96. package/dist/services/realtime/openai/openai-provider.js +579 -0
  97. package/dist/services/realtime/openai/openai-tool-mapper.d.ts +15 -0
  98. package/dist/services/realtime/openai/openai-tool-mapper.js +14 -0
  99. package/dist/services/realtime/provider-factory.d.ts +4 -0
  100. package/dist/services/realtime/provider-factory.js +12 -0
  101. package/dist/services/realtime/types.d.ts +78 -0
  102. package/dist/services/realtime/types.js +1 -0
  103. package/dist/services/realtime/vertexai/index.d.ts +1 -0
  104. package/dist/services/realtime/vertexai/index.js +1 -0
  105. package/dist/services/realtime/vertexai/vertexai-provider.d.ts +53 -0
  106. package/dist/services/realtime/vertexai/vertexai-provider.js +561 -0
  107. package/dist/services/realtime/voice-session-tracker.d.ts +10 -0
  108. package/dist/services/realtime/voice-session-tracker.js +62 -0
  109. package/dist/services/signalr.d.ts +37 -0
  110. package/dist/services/signalr.js +145 -0
  111. package/dist/services/thinking-audio-pulse.d.ts +2 -0
  112. package/dist/services/thinking-audio-pulse.js +72 -0
  113. package/dist/services/thinking-audio.d.ts +2 -0
  114. package/dist/services/thinking-audio.js +105 -0
  115. package/dist/services/voice-health-watchdog.d.ts +42 -0
  116. package/dist/services/voice-health-watchdog.js +90 -0
  117. package/dist/services/webrtc-event-handler.d.ts +4 -0
  118. package/dist/services/webrtc-event-handler.js +42 -0
  119. package/dist/services/webrtc-greeting.d.ts +3 -0
  120. package/dist/services/webrtc-greeting.js +62 -0
  121. package/dist/services/webrtc-microphone.d.ts +15 -0
  122. package/dist/services/webrtc-microphone.js +64 -0
  123. package/dist/services/webrtc.d.ts +44 -0
  124. package/dist/services/webrtc.js +196 -0
  125. package/dist/types/index.d.ts +151 -0
  126. package/dist/types/index.js +1 -0
  127. package/dist/utils/audio-context.d.ts +11 -0
  128. package/dist/utils/audio-context.js +33 -0
  129. package/dist/utils/connect-chime.d.ts +11 -0
  130. package/dist/utils/connect-chime.js +230 -0
  131. package/dist/utils/consent.d.ts +4 -0
  132. package/dist/utils/consent.js +30 -0
  133. package/dist/utils/language-detection.d.ts +1 -0
  134. package/dist/utils/language-detection.js +17 -0
  135. package/dist/utils/logger.d.ts +13 -0
  136. package/dist/utils/logger.js +40 -0
  137. package/dist/utils/mic-level-analyser.d.ts +13 -0
  138. package/dist/utils/mic-level-analyser.js +50 -0
  139. package/dist/utils/proactive-events.d.ts +5 -0
  140. package/dist/utils/proactive-events.js +1 -0
  141. package/dist/utils/styles.d.ts +4 -0
  142. package/dist/utils/styles.js +35 -0
  143. package/dist/utils/theme-detection.d.ts +3 -0
  144. package/dist/utils/theme-detection.js +121 -0
  145. package/dist/utils/tool-status.d.ts +10 -0
  146. package/dist/utils/tool-status.js +61 -0
  147. package/dist/utils/ui-sounds.d.ts +3 -0
  148. package/dist/utils/ui-sounds.js +57 -0
  149. package/package.json +56 -0
@@ -0,0 +1,44 @@
1
+ import type { ConnectionState, ConversationEvents, ToolDefinition } from '../types';
2
+ import type { RealtimeProviderType } from './realtime/types';
3
+ export type TranscriptionMode = 'oob' | 'native';
4
+ export interface WebRtcConfig {
5
+ token: string;
6
+ model: string;
7
+ voice?: string;
8
+ instructions?: string;
9
+ tools?: ToolDefinition[];
10
+ speed?: number;
11
+ language?: string;
12
+ transcribeModel?: string;
13
+ transcriptionMode?: TranscriptionMode;
14
+ transcriptionPrompt?: string;
15
+ provider?: RealtimeProviderType;
16
+ proxyUrl?: string;
17
+ proxyBearerToken?: string;
18
+ skipGreeting?: boolean;
19
+ }
20
+ export declare class WebRtcService {
21
+ private client;
22
+ private events;
23
+ private _connectionState;
24
+ private watchdog;
25
+ private readonly microphoneManager;
26
+ private providerType;
27
+ private model;
28
+ get connectionState(): ConnectionState;
29
+ get isConnected(): boolean;
30
+ setEventHandlers(events: ConversationEvents): void;
31
+ requestMicrophonePermission(): Promise<'granted' | 'denied' | 'prompt'>;
32
+ connect(config: WebRtcConfig): Promise<void>;
33
+ disconnect(): Promise<void>;
34
+ private readonly resumeAudioOnVisible;
35
+ private attachVisibilityResume;
36
+ private detachVisibilityResume;
37
+ sendText(text: string): Promise<void>;
38
+ setMuted(muted: boolean): void;
39
+ interrupt(): void;
40
+ waitForResponseComplete(timeoutMs?: number): Promise<void>;
41
+ sendFunctionResult(callId: string, name: string, result: string): Promise<void>;
42
+ private handleUsageUpdated;
43
+ private setConnectionState;
44
+ }
@@ -0,0 +1,196 @@
1
+ import { createRealtimeProvider } from './realtime/provider-factory';
2
+ import { logger } from '../utils/logger';
3
+ import { MicrophoneManager } from './webrtc-microphone';
4
+ import { setupEventHandlers } from './webrtc-event-handler';
5
+ import { startGreetingFlow } from './webrtc-greeting';
6
+ import { VoiceHealthWatchdog } from './voice-health-watchdog';
7
+ export class WebRtcService {
8
+ client = null;
9
+ events = {};
10
+ _connectionState = 'disconnected';
11
+ watchdog = null;
12
+ microphoneManager = new MicrophoneManager();
13
+ providerType = 'openai';
14
+ model = '';
15
+ get connectionState() {
16
+ return this._connectionState;
17
+ }
18
+ get isConnected() {
19
+ return this._connectionState === 'connected';
20
+ }
21
+ setEventHandlers(events) {
22
+ this.events = events;
23
+ }
24
+ async requestMicrophonePermission() {
25
+ return this.microphoneManager.requestPermission();
26
+ }
27
+ async connect(config) {
28
+ if (this.client) {
29
+ await this.disconnect();
30
+ }
31
+ this.setConnectionState('connecting');
32
+ try {
33
+ this.providerType = config.provider ?? 'openai';
34
+ this.model = config.model;
35
+ logger.debug('[WebRTC] Connecting with provider:', this.providerType, 'model:', this.model);
36
+ this.client = await createRealtimeProvider(this.providerType);
37
+ setupEventHandlers(this.client, this.events, this.microphoneManager, (state) => this.setConnectionState(state), (usage) => this.handleUsageUpdated(usage));
38
+ // Start the health watchdog before connecting so it never misses an
39
+ // early sessionReady. It detects a "connected but dead" session
40
+ // (no setup / greeting never responds) and asks the owner to recover.
41
+ this.watchdog = new VoiceHealthWatchdog(this.client, !(config.skipGreeting ?? false), (reason) => this.events.onVoiceUnhealthy?.(reason), () => this.events.onVoiceHealthy?.());
42
+ this.watchdog.start();
43
+ await this.client.connect({
44
+ apiKey: config.token,
45
+ model: config.model,
46
+ instructions: config.instructions || '',
47
+ voice: config.voice || 'ash',
48
+ tools: config.tools || [],
49
+ toolChoice: 'auto',
50
+ speed: config.speed,
51
+ transcription: config.language
52
+ ? {
53
+ mode: config.transcriptionMode ?? 'native',
54
+ model: config.transcribeModel,
55
+ language: config.language,
56
+ prompt: config.transcriptionPrompt,
57
+ }
58
+ : undefined,
59
+ proxyUrl: config.proxyUrl,
60
+ proxyBearerToken: config.proxyBearerToken,
61
+ });
62
+ logger.debug('[WebRTC] Connected successfully');
63
+ this.attachVisibilityResume();
64
+ this.microphoneManager.startDeviceChangeDetection();
65
+ this.client.mute(true);
66
+ this.microphoneManager.initPendingUnmute();
67
+ logger.debug('[WebRTC] Microphone initially muted, pending unmute after greeting');
68
+ startGreetingFlow(this.client, this.microphoneManager, this.events.onMicrophoneReady, config.skipGreeting ?? false);
69
+ }
70
+ catch (error) {
71
+ this.watchdog?.stop();
72
+ this.watchdog = null;
73
+ logger.error('[WebRTC] Connection failed:', error);
74
+ // Drop the half-built client so the retry creates a fresh peer
75
+ // connection instead of trying to drive the closed one (which
76
+ // throws InvalidStateError 'signalingState is closed').
77
+ if (this.client) {
78
+ try {
79
+ this.client.close();
80
+ }
81
+ catch {
82
+ // best-effort — the SDK may already have torn it down
83
+ }
84
+ this.client = null;
85
+ }
86
+ this.setConnectionState('error');
87
+ this.events.onError?.(error instanceof Error ? error.message : 'Connection failed');
88
+ throw error;
89
+ }
90
+ }
91
+ async disconnect() {
92
+ this.detachVisibilityResume();
93
+ this.watchdog?.stop();
94
+ this.watchdog = null;
95
+ this.microphoneManager.reset();
96
+ if (this.client) {
97
+ this.client.close();
98
+ this.client = null;
99
+ }
100
+ this.setConnectionState('disconnected');
101
+ logger.debug('[WebRTC] Disconnected');
102
+ }
103
+ // iOS suspends audio when an installed PWA is backgrounded; nudge the active
104
+ // provider to re-arm playback + the mic-level analyser when we're visible again.
105
+ resumeAudioOnVisible = () => {
106
+ if (typeof document !== 'undefined' && document.visibilityState !== 'visible')
107
+ return;
108
+ this.client?.resumeAudio?.();
109
+ };
110
+ attachVisibilityResume() {
111
+ if (typeof document === 'undefined')
112
+ return;
113
+ document.addEventListener('visibilitychange', this.resumeAudioOnVisible);
114
+ if (globalThis.window !== undefined) {
115
+ globalThis.window.addEventListener('focus', this.resumeAudioOnVisible);
116
+ }
117
+ }
118
+ detachVisibilityResume() {
119
+ if (typeof document === 'undefined')
120
+ return;
121
+ document.removeEventListener('visibilitychange', this.resumeAudioOnVisible);
122
+ if (globalThis.window !== undefined) {
123
+ globalThis.window.removeEventListener('focus', this.resumeAudioOnVisible);
124
+ }
125
+ }
126
+ async sendText(text) {
127
+ if (this.client?.status !== 'connected') {
128
+ throw new Error('Not connected');
129
+ }
130
+ // The caller (WidgetController.sendMessage) already adds the user
131
+ // message and persists it as 'Typed'. Re-emitting it as a transcript
132
+ // would make the voice-state-manager add it a second time, so the
133
+ // history shows the typed line twice. Spoken transcripts still arrive
134
+ // through the provider client's native transcript events.
135
+ this.client.sendText(text);
136
+ }
137
+ setMuted(muted) {
138
+ if (!this.client)
139
+ return;
140
+ this.microphoneManager.setMuted(muted, this.client);
141
+ }
142
+ interrupt() {
143
+ if (this.client?.status === 'connected') {
144
+ this.client.interrupt();
145
+ }
146
+ }
147
+ waitForResponseComplete(timeoutMs = 30000) {
148
+ return new Promise((resolve) => {
149
+ if (this.client?.status !== 'connected') {
150
+ resolve();
151
+ return;
152
+ }
153
+ const client = this.client;
154
+ let resolved = false;
155
+ let timeoutHandle = null;
156
+ const finish = (reason) => {
157
+ if (resolved)
158
+ return;
159
+ resolved = true;
160
+ if (timeoutHandle !== null) {
161
+ clearTimeout(timeoutHandle);
162
+ timeoutHandle = null;
163
+ }
164
+ unsubscribeAudioDone();
165
+ logger.debug(`[WebRTC] ${reason}`);
166
+ resolve();
167
+ };
168
+ const unsubscribeAudioDone = client.on('audioDone', () => {
169
+ logger.debug('[WebRTC] audioDone received, monitoring audio playback...');
170
+ client.waitForAudioPlaybackEnd()
171
+ .then((silenceReason) => finish(`Audio playback complete (${silenceReason})`))
172
+ .catch(() => finish('Audio monitoring unavailable, disconnecting'));
173
+ });
174
+ timeoutHandle = setTimeout(() => {
175
+ finish('Response wait timeout');
176
+ }, timeoutMs);
177
+ });
178
+ }
179
+ async sendFunctionResult(callId, name, result) {
180
+ if (this.client?.status !== 'connected') {
181
+ throw new Error('Not connected');
182
+ }
183
+ this.client.sendToolResult(callId, name, result);
184
+ }
185
+ handleUsageUpdated(usage) {
186
+ this.events.onUsageUpdated?.({
187
+ provider: this.providerType,
188
+ model: this.model,
189
+ ...usage,
190
+ });
191
+ }
192
+ setConnectionState(state) {
193
+ this._connectionState = state;
194
+ this.events.onConnectionStateChanged?.(state);
195
+ }
196
+ }
@@ -0,0 +1,151 @@
1
+ export interface WidgetDesignTokens {
2
+ primaryColor: string;
3
+ primaryColorHover: string;
4
+ textOnPrimary: string;
5
+ secondaryColor: string;
6
+ secondaryColorHover: string;
7
+ textOnSecondary: string;
8
+ }
9
+ export type AssistantMode = 'voice' | 'text';
10
+ export interface WidgetConfig {
11
+ assistantId: string;
12
+ tenantId: string;
13
+ name: string;
14
+ welcomeMessage: string;
15
+ logoUrl: string;
16
+ theme: WidgetTheme;
17
+ size: WidgetSize;
18
+ placement: WidgetPlacement;
19
+ voice?: string;
20
+ largeTeaserText: string;
21
+ largeCtaText: string;
22
+ inputPlaceholder: string;
23
+ startButtonText: string;
24
+ connectingText: string;
25
+ listeningText: string;
26
+ preparingText: string;
27
+ sendButtonText: string;
28
+ disconnectText: string;
29
+ mutedText: string;
30
+ unmuteText: string;
31
+ minimizeText: string;
32
+ resetText: string;
33
+ resetConfirmText: string;
34
+ resetConfirmButtonText: string;
35
+ resetCancelButtonText: string;
36
+ errorText: string;
37
+ microphoneBlockedText: string;
38
+ microphoneBlockedHintText: string;
39
+ designTokens: WidgetDesignTokens;
40
+ darkDesignTokens?: WidgetDesignTokens;
41
+ lightDesignTokens?: WidgetDesignTokens;
42
+ showTermsAndConditions?: boolean;
43
+ termsAndConditions?: string;
44
+ termsAcceptButtonText?: string;
45
+ termsDeclineButtonText?: string;
46
+ speechSpeed?: number;
47
+ popoverEnabled?: boolean;
48
+ greetingEnabled?: boolean;
49
+ popoverAutoCloseSeconds?: number;
50
+ popoverText?: string;
51
+ chatTitle?: string;
52
+ assistantMode?: AssistantMode;
53
+ language?: string;
54
+ textChatFallbackText?: string;
55
+ transcribeModel?: string;
56
+ transcriptionMode?: 'oob' | 'native';
57
+ transcriptionPrompt?: string;
58
+ voiceButtonText?: string;
59
+ textButtonText?: string;
60
+ voiceButtonHint?: string;
61
+ textButtonHint?: string;
62
+ copyText?: string;
63
+ copiedText?: string;
64
+ hideBranding?: boolean;
65
+ }
66
+ export interface ConnectOptions {
67
+ /**
68
+ * Hold the first user message (display and send) until the greeting has finished streaming.
69
+ * Default `true`. Set `false` to send the first message immediately, even mid-greeting.
70
+ */
71
+ waitForGreeting?: boolean;
72
+ /**
73
+ * Override the per-assistant greeting setting (`config.greetingEnabled`) for this connect.
74
+ * `undefined` uses the server config; `true` forces a greeting; `false` suppresses it.
75
+ * Applies to both text and voice. Never re-greets a resumed conversation.
76
+ */
77
+ greeting?: boolean;
78
+ }
79
+ export type WidgetTheme = 'light' | 'dark' | 'auto';
80
+ export type WidgetSize = 'small' | 'compact' | 'large';
81
+ export type WidgetPlacement = 'top-left' | 'top-center' | 'top-right' | 'bottom-left' | 'bottom-center' | 'bottom-right';
82
+ export interface ChatMessage {
83
+ id: string;
84
+ role: 'user' | 'assistant';
85
+ content: string;
86
+ timestamp: Date;
87
+ feedback?: FeedbackRating;
88
+ }
89
+ export type FeedbackRating = -1 | 1 | 2;
90
+ export type ConnectionState = 'disconnected' | 'connecting' | 'connected' | 'reconnecting' | 'error';
91
+ export interface ToolDefinition {
92
+ type: 'function';
93
+ name: string;
94
+ description: string;
95
+ strict?: boolean;
96
+ parameters: ToolParameters;
97
+ }
98
+ export interface ToolParameters {
99
+ type: 'object';
100
+ properties: Record<string, ToolProperty>;
101
+ required?: string[];
102
+ additionalProperties?: boolean;
103
+ }
104
+ export interface ToolProperty {
105
+ type: string;
106
+ description?: string;
107
+ format?: string;
108
+ items?: ToolProperty;
109
+ properties?: Record<string, ToolProperty>;
110
+ required?: string[];
111
+ enum?: string[];
112
+ }
113
+ export interface FunctionCall {
114
+ callId: string;
115
+ name: string;
116
+ arguments: string;
117
+ }
118
+ export interface VoiceUsage {
119
+ readonly provider: string;
120
+ readonly model: string;
121
+ readonly inputTokens: number;
122
+ readonly outputTokens: number;
123
+ readonly totalTokens: number;
124
+ readonly cachedTokens?: number;
125
+ readonly textCachedInputTokens?: number;
126
+ readonly audioCachedInputTokens?: number;
127
+ readonly audioInputTokens?: number;
128
+ readonly audioOutputTokens?: number;
129
+ readonly textInputTokens?: number;
130
+ readonly textOutputTokens?: number;
131
+ }
132
+ export interface ConversationEvents {
133
+ onUserSpeechStart?: () => void;
134
+ onUserSpeechStop?: () => void;
135
+ onUserSpeechEnd?: () => void;
136
+ onTranscriptReceived?: (isUser: boolean, text: string) => void;
137
+ onTranscriptDelta?: (isUser: boolean, delta: string) => void;
138
+ onFunctionCall?: (call: FunctionCall) => void;
139
+ onUsageUpdated?: (usage: VoiceUsage) => void;
140
+ onError?: (error: string) => void;
141
+ onConnectionStateChanged?: (state: ConnectionState) => void;
142
+ onMicrophoneReady?: () => void;
143
+ onAudioLevel?: (level: number) => void;
144
+ onVoiceUnhealthy?: (reason: string) => void;
145
+ onVoiceHealthy?: () => void;
146
+ }
147
+ export interface ApiError {
148
+ message: string;
149
+ code?: string;
150
+ details?: Record<string, unknown>;
151
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,11 @@
1
+ export declare function getSharedAudioContext(): AudioContext | null;
2
+ export declare function withResumedContext(ctx: AudioContext, play: () => void, onError: (err: unknown) => void): void;
3
+ /**
4
+ * Unlock audio inside a user gesture's activation window. iOS/Safari only let an
5
+ * AudioContext run once it has been resumed during a gesture; the voice playback
6
+ * context is created later (after getUserMedia + token await) and would stay
7
+ * suspended. Resuming the shared context on the start-voice click unlocks audio
8
+ * for the page session so the later playback context runs. Call synchronously
9
+ * before any await; idempotent and provider-agnostic.
10
+ */
11
+ export declare function primeAudioOnGesture(): void;
@@ -0,0 +1,33 @@
1
+ let cachedContext = null;
2
+ export function getSharedAudioContext() {
3
+ if (cachedContext)
4
+ return cachedContext;
5
+ const Ctor = globalThis.AudioContext;
6
+ if (!Ctor)
7
+ return null;
8
+ try {
9
+ cachedContext = new Ctor();
10
+ return cachedContext;
11
+ }
12
+ catch {
13
+ return null;
14
+ }
15
+ }
16
+ export function withResumedContext(ctx, play, onError) {
17
+ const resume = ctx.state === 'suspended' ? ctx.resume() : Promise.resolve();
18
+ void resume.then(play).catch(onError);
19
+ }
20
+ /**
21
+ * Unlock audio inside a user gesture's activation window. iOS/Safari only let an
22
+ * AudioContext run once it has been resumed during a gesture; the voice playback
23
+ * context is created later (after getUserMedia + token await) and would stay
24
+ * suspended. Resuming the shared context on the start-voice click unlocks audio
25
+ * for the page session so the later playback context runs. Call synchronously
26
+ * before any await; idempotent and provider-agnostic.
27
+ */
28
+ export function primeAudioOnGesture() {
29
+ const ctx = getSharedAudioContext();
30
+ if (!ctx)
31
+ return;
32
+ withResumedContext(ctx, () => undefined, () => undefined);
33
+ }
@@ -0,0 +1,11 @@
1
+ export type ConnectChimeVariantId = 'plagal' | 'arpeggio' | 'bell';
2
+ export interface ConnectChimeVariant {
3
+ readonly id: ConnectChimeVariantId;
4
+ readonly label: string;
5
+ readonly description: string;
6
+ }
7
+ export declare const CONNECT_CHIME_VARIANTS: readonly ConnectChimeVariant[];
8
+ export declare function getActiveConnectChimeVariant(): ConnectChimeVariantId;
9
+ export declare function setActiveConnectChimeVariant(variant: ConnectChimeVariantId): void;
10
+ export declare function playConnectChimeVariant(variant: ConnectChimeVariantId): void;
11
+ export declare function playConnectChime(): void;
@@ -0,0 +1,230 @@
1
+ import { getSharedAudioContext, withResumedContext } from './audio-context';
2
+ import { logger } from './logger';
3
+ const MASTER_FILTER_Q = 0.5;
4
+ const REVERB_DECAY_POWER = 2.2;
5
+ const cachedImpulses = new Map();
6
+ function getImpulseResponse(ctx, durationSec, decayPower) {
7
+ const key = Math.round(durationSec * 100) + decayPower / 10;
8
+ const existing = cachedImpulses.get(key);
9
+ if (existing)
10
+ return existing;
11
+ const length = Math.floor(durationSec * ctx.sampleRate);
12
+ const buffer = ctx.createBuffer(2, length, ctx.sampleRate);
13
+ for (let channel = 0; channel < 2; channel++) {
14
+ const data = buffer.getChannelData(channel);
15
+ for (let i = 0; i < length; i++) {
16
+ const t = i / length;
17
+ data[i] = (Math.random() * 2 - 1) * Math.pow(1 - t, decayPower);
18
+ }
19
+ }
20
+ cachedImpulses.set(key, buffer);
21
+ return buffer;
22
+ }
23
+ function buildBus(ctx, filterFreq, reverbDurationSec, reverbWet) {
24
+ const filter = ctx.createBiquadFilter();
25
+ filter.type = 'lowpass';
26
+ filter.frequency.value = filterFreq;
27
+ filter.Q.value = MASTER_FILTER_Q;
28
+ const dryGain = ctx.createGain();
29
+ dryGain.gain.value = 1 - reverbWet;
30
+ dryGain.connect(ctx.destination);
31
+ const convolver = ctx.createConvolver();
32
+ convolver.buffer = getImpulseResponse(ctx, reverbDurationSec, REVERB_DECAY_POWER);
33
+ const wetGain = ctx.createGain();
34
+ wetGain.gain.value = reverbWet;
35
+ convolver.connect(wetGain).connect(ctx.destination);
36
+ filter.connect(dryGain);
37
+ filter.connect(convolver);
38
+ return filter;
39
+ }
40
+ function scheduleSineChord(params) {
41
+ const { ctx, voices, startAt, attackSec, holdSec, releaseSec, peakGain, destination } = params;
42
+ const envelope = ctx.createGain();
43
+ const peakAt = startAt + attackSec;
44
+ const holdEnd = peakAt + holdSec;
45
+ const releaseAt = holdEnd + releaseSec;
46
+ envelope.gain.setValueAtTime(0.0001, startAt);
47
+ envelope.gain.exponentialRampToValueAtTime(peakGain, peakAt);
48
+ envelope.gain.setValueAtTime(peakGain, holdEnd);
49
+ envelope.gain.exponentialRampToValueAtTime(0.0001, releaseAt);
50
+ envelope.connect(destination);
51
+ for (const voice of voices) {
52
+ const osc = ctx.createOscillator();
53
+ osc.type = 'sine';
54
+ osc.frequency.value = voice.frequency;
55
+ const voiceGain = ctx.createGain();
56
+ voiceGain.gain.value = voice.gain;
57
+ osc.connect(voiceGain).connect(envelope);
58
+ osc.start(startAt);
59
+ osc.stop(releaseAt + 0.05);
60
+ }
61
+ }
62
+ const PLAGAL_SUBDOMINANT = [
63
+ { frequency: 196, gain: 0.85 },
64
+ { frequency: 246.94, gain: 0.65 },
65
+ { frequency: 293.66, gain: 0.55 },
66
+ { frequency: 392, gain: 0.6 },
67
+ { frequency: 493.88, gain: 0.42 },
68
+ { frequency: 587.33, gain: 0.35 },
69
+ ];
70
+ const PLAGAL_TONIC = [
71
+ { frequency: 146.83, gain: 0.7 },
72
+ { frequency: 220, gain: 0.55 },
73
+ { frequency: 293.66, gain: 0.9 },
74
+ { frequency: 369.99, gain: 0.7 },
75
+ { frequency: 440, gain: 0.6 },
76
+ { frequency: 587.33, gain: 0.55 },
77
+ { frequency: 739.99, gain: 0.4 },
78
+ ];
79
+ function playPlagalChime(ctx) {
80
+ const bus = buildBus(ctx, 4800, 2, 0.3);
81
+ const startAt = ctx.currentTime + 0.03;
82
+ scheduleSineChord({
83
+ ctx, voices: PLAGAL_SUBDOMINANT, startAt,
84
+ attackSec: 0.04, holdSec: 0.18, releaseSec: 0.9, peakGain: 0.26, destination: bus,
85
+ });
86
+ scheduleSineChord({
87
+ ctx, voices: PLAGAL_TONIC, startAt: startAt + 0.42,
88
+ attackSec: 0.08, holdSec: 0.35, releaseSec: 1.8, peakGain: 0.32, destination: bus,
89
+ });
90
+ }
91
+ const ARPEGGIO_NOTES = [293.66, 369.99, 440];
92
+ const ARPEGGIO_STEP_SEC = 0.14;
93
+ const ARPEGGIO_HARMONICS = [
94
+ { ratio: 1, gain: 0.9 },
95
+ { ratio: 2, gain: 0.4 },
96
+ { ratio: 3, gain: 0.18 },
97
+ { ratio: 4, gain: 0.1 },
98
+ ];
99
+ const ARPEGGIO_FINAL_CHORD = [
100
+ { frequency: 146.83, gain: 0.55 },
101
+ { frequency: 220, gain: 0.5 },
102
+ { frequency: 293.66, gain: 0.85 },
103
+ { frequency: 369.99, gain: 0.7 },
104
+ { frequency: 440, gain: 0.65 },
105
+ { frequency: 587.33, gain: 0.55 },
106
+ { frequency: 739.99, gain: 0.4 },
107
+ ];
108
+ function scheduleBellPluck(ctx, fundamental, startAt, destination) {
109
+ const envelope = ctx.createGain();
110
+ const peakAt = startAt + 0.005;
111
+ const holdEnd = peakAt + 0.04;
112
+ const releaseAt = holdEnd + 0.6;
113
+ envelope.gain.setValueAtTime(0.0001, startAt);
114
+ envelope.gain.exponentialRampToValueAtTime(0.36, peakAt);
115
+ envelope.gain.setValueAtTime(0.36, holdEnd);
116
+ envelope.gain.exponentialRampToValueAtTime(0.0001, releaseAt);
117
+ envelope.connect(destination);
118
+ for (const harmonic of ARPEGGIO_HARMONICS) {
119
+ const osc = ctx.createOscillator();
120
+ osc.type = 'sine';
121
+ osc.frequency.value = fundamental * harmonic.ratio;
122
+ const layerGain = ctx.createGain();
123
+ layerGain.gain.value = harmonic.gain;
124
+ osc.connect(layerGain).connect(envelope);
125
+ osc.start(startAt);
126
+ osc.stop(releaseAt + 0.05);
127
+ }
128
+ }
129
+ function playArpeggioChime(ctx) {
130
+ const bus = buildBus(ctx, 5400, 2, 0.3);
131
+ const startAt = ctx.currentTime + 0.03;
132
+ ARPEGGIO_NOTES.forEach((freq, index) => {
133
+ scheduleBellPluck(ctx, freq, startAt + index * ARPEGGIO_STEP_SEC, bus);
134
+ });
135
+ scheduleSineChord({
136
+ ctx, voices: ARPEGGIO_FINAL_CHORD,
137
+ startAt: startAt + ARPEGGIO_STEP_SEC * 2,
138
+ attackSec: 0.06, holdSec: 0.35, releaseSec: 1.7, peakGain: 0.28, destination: bus,
139
+ });
140
+ }
141
+ const MODERN_BELL_VOICES = [
142
+ { frequency: 146.83, gain: 0.55, decaySec: 1.6 },
143
+ { frequency: 293.66, gain: 0.85, decaySec: 2.4 },
144
+ { frequency: 369.99, gain: 0.7, decaySec: 2.2 },
145
+ { frequency: 440, gain: 0.7, decaySec: 2.2 },
146
+ { frequency: 554.37, gain: 0.55, decaySec: 2 },
147
+ { frequency: 587.33, gain: 0.6, decaySec: 2 },
148
+ { frequency: 739.99, gain: 0.42, decaySec: 1.8 },
149
+ { frequency: 880, gain: 0.36, decaySec: 1.6 },
150
+ { frequency: 1174.66, gain: 0.22, decaySec: 1.3 },
151
+ ];
152
+ function playModernBellChime(ctx) {
153
+ const bus = buildBus(ctx, 6800, 2.6, 0.34);
154
+ const startAt = ctx.currentTime + 0.03;
155
+ const master = ctx.createGain();
156
+ master.gain.setValueAtTime(0.34, startAt);
157
+ master.connect(bus);
158
+ for (const voice of MODERN_BELL_VOICES) {
159
+ const envelope = ctx.createGain();
160
+ const peakAt = startAt + 0.004;
161
+ const holdEnd = peakAt + 0.04;
162
+ const releaseAt = holdEnd + voice.decaySec;
163
+ envelope.gain.setValueAtTime(0.0001, startAt);
164
+ envelope.gain.exponentialRampToValueAtTime(voice.gain, peakAt);
165
+ envelope.gain.setValueAtTime(voice.gain, holdEnd);
166
+ envelope.gain.exponentialRampToValueAtTime(0.0001, releaseAt);
167
+ envelope.connect(master);
168
+ const osc = ctx.createOscillator();
169
+ osc.type = 'sine';
170
+ osc.frequency.value = voice.frequency;
171
+ osc.connect(envelope);
172
+ osc.start(startAt);
173
+ osc.stop(releaseAt + 0.05);
174
+ }
175
+ }
176
+ export const CONNECT_CHIME_VARIANTS = [
177
+ { id: 'plagal', label: 'Plagal-Cadence', description: 'warm, geborgen (G-Dur → D-Dur)' },
178
+ { id: 'arpeggio', label: 'Arpeggio-Up', description: 'perlend, hell (D–F♯–A aufsteigend)' },
179
+ { id: 'bell', label: 'Modern Bell', description: 'modern, Dmaj7 als Glockenanschlag' },
180
+ ];
181
+ const DEFAULT_VARIANT = 'arpeggio';
182
+ const STORAGE_KEY = 'nextpa.chat-app.connect-chime-variant';
183
+ function isValidVariantId(value) {
184
+ return value === 'plagal' || value === 'arpeggio' || value === 'bell';
185
+ }
186
+ export function getActiveConnectChimeVariant() {
187
+ try {
188
+ const stored = globalThis.localStorage?.getItem(STORAGE_KEY);
189
+ if (isValidVariantId(stored))
190
+ return stored;
191
+ }
192
+ catch {
193
+ // localStorage may be unavailable (SSR, privacy mode); fall through.
194
+ }
195
+ return DEFAULT_VARIANT;
196
+ }
197
+ export function setActiveConnectChimeVariant(variant) {
198
+ try {
199
+ globalThis.localStorage?.setItem(STORAGE_KEY, variant);
200
+ }
201
+ catch {
202
+ // best-effort persistence
203
+ }
204
+ }
205
+ function playVariant(ctx, variant) {
206
+ switch (variant) {
207
+ case 'plagal':
208
+ playPlagalChime(ctx);
209
+ return;
210
+ case 'arpeggio':
211
+ playArpeggioChime(ctx);
212
+ return;
213
+ case 'bell':
214
+ playModernBellChime(ctx);
215
+ return;
216
+ }
217
+ }
218
+ export function playConnectChimeVariant(variant) {
219
+ const ctx = getSharedAudioContext();
220
+ if (!ctx)
221
+ return;
222
+ withResumedContext(ctx, () => playVariant(ctx, variant), (err) => logger.debug('[ConnectChime] failed to play', err));
223
+ }
224
+ export function playConnectChime() {
225
+ // The live voice-connect cue is fixed to 'arpeggio' (Norbert's pick).
226
+ // getActiveConnectChimeVariant + the localStorage key remain available
227
+ // for the hidden ChimeTestSelector, but a stale persisted choice from
228
+ // earlier testing must not override the live cue.
229
+ playConnectChimeVariant('arpeggio');
230
+ }
@@ -0,0 +1,4 @@
1
+ export declare function getConsentCookieName(assistantId: string): string;
2
+ export declare function hasUserConsented(assistantId: string): boolean;
3
+ export declare function setUserConsent(assistantId: string, consented: boolean): void;
4
+ export declare function clearUserConsent(assistantId: string): void;