@bytexbyte/nxtlinq-ai-agent-ui-react-native-development 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/NxtlinqAgentAssistant.d.ts +4 -4
- package/dist/NxtlinqAgentAssistant.d.ts.map +1 -1
- package/dist/NxtlinqAgentAssistant.js +5 -6
- package/dist/components/AgentAssistantShell.d.ts +1 -3
- package/dist/components/AgentAssistantShell.d.ts.map +1 -1
- package/dist/components/AgentAssistantShell.js +3 -7
- package/dist/components/AgentMessageList.d.ts.map +1 -1
- package/dist/components/AgentMessageList.js +7 -9
- package/dist/components/AgentVoiceBar.d.ts.map +1 -1
- package/dist/components/AgentVoiceBar.js +14 -34
- package/dist/components/MessageAttachmentPreview.d.ts +10 -0
- package/dist/components/MessageAttachmentPreview.d.ts.map +1 -0
- package/dist/components/MessageAttachmentPreview.js +15 -0
- package/dist/components/VoiceAddMediaModal.d.ts +12 -0
- package/dist/components/VoiceAddMediaModal.d.ts.map +1 -0
- package/dist/components/VoiceAddMediaModal.js +31 -0
- package/dist/components/VoiceAttachmentButton.d.ts +3 -0
- package/dist/components/VoiceAttachmentButton.d.ts.map +1 -0
- package/dist/components/VoiceAttachmentButton.js +58 -0
- package/dist/components/VoiceIcons.d.ts +1 -0
- package/dist/components/VoiceIcons.d.ts.map +1 -1
- package/dist/components/VoiceIcons.js +3 -0
- package/dist/components/VoiceWaveform.d.ts +2 -2
- package/dist/components/VoiceWaveform.d.ts.map +1 -1
- package/dist/components/VoiceWaveform.js +16 -5
- package/dist/components/useMessageListAutoScroll.d.ts +12 -0
- package/dist/components/useMessageListAutoScroll.d.ts.map +1 -0
- package/dist/components/useMessageListAutoScroll.js +42 -0
- package/dist/context/AgentAssistantContext.d.ts +3 -3
- package/dist/context/AgentAssistantContext.d.ts.map +1 -1
- package/dist/context/AgentAssistantContext.js +76 -29
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/types.d.ts +3 -8
- package/dist/types.d.ts.map +1 -1
- package/dist/voice/float32ToPcm16.d.ts +2 -0
- package/dist/voice/float32ToPcm16.d.ts.map +1 -0
- package/dist/voice/float32ToPcm16.js +8 -0
- package/dist/voice/loadImageCropPicker.d.ts +11 -0
- package/dist/voice/loadImageCropPicker.d.ts.map +1 -0
- package/dist/voice/loadImageCropPicker.js +12 -0
- package/dist/voice/sendVoiceImageAttachment.d.ts +15 -0
- package/dist/voice/sendVoiceImageAttachment.d.ts.map +1 -0
- package/dist/voice/sendVoiceImageAttachment.js +29 -0
- package/dist/voice/useVoiceImagePicker.d.ts +11 -0
- package/dist/voice/useVoiceImagePicker.d.ts.map +1 -0
- package/dist/voice/useVoiceImagePicker.js +38 -0
- package/dist/voice/useVoiceMicState.d.ts +4 -0
- package/dist/voice/useVoiceMicState.d.ts.map +1 -1
- package/dist/voice/useVoiceMicState.js +32 -3
- package/dist/voice/useVoiceSilenceCommit.d.ts +10 -0
- package/dist/voice/useVoiceSilenceCommit.d.ts.map +1 -0
- package/dist/voice/useVoiceSilenceCommit.js +76 -0
- package/dist/voice/useVoiceTranscriptMessages.d.ts +16 -0
- package/dist/voice/useVoiceTranscriptMessages.d.ts.map +1 -0
- package/dist/voice/useVoiceTranscriptMessages.js +133 -0
- package/dist/voice/useWsRealtimeAudio.d.ts +17 -0
- package/dist/voice/useWsRealtimeAudio.d.ts.map +1 -0
- package/dist/voice/useWsRealtimeAudio.js +165 -0
- package/dist/voice/voiceImagePickerOptions.d.ts +11 -0
- package/dist/voice/voiceImagePickerOptions.d.ts.map +1 -0
- package/dist/voice/voiceImagePickerOptions.js +10 -0
- package/dist/voice/voiceSilenceConstants.d.ts +8 -0
- package/dist/voice/voiceSilenceConstants.d.ts.map +1 -0
- package/dist/voice/voiceSilenceConstants.js +7 -0
- package/dist/voice/wsPcmPlayer.d.ts +24 -0
- package/dist/voice/wsPcmPlayer.d.ts.map +1 -0
- package/dist/voice/wsPcmPlayer.js +146 -0
- package/dist/voice/wsPcmRecorder.d.ts +26 -0
- package/dist/voice/wsPcmRecorder.d.ts.map +1 -0
- package/dist/voice/wsPcmRecorder.js +145 -0
- package/dist/voice/wsRealtimeConstants.d.ts +2 -0
- package/dist/voice/wsRealtimeConstants.d.ts.map +1 -0
- package/dist/voice/wsRealtimeConstants.js +1 -0
- package/package.json +8 -5
- package/src/NxtlinqAgentAssistant.tsx +3 -12
- package/src/components/AgentAssistantShell.tsx +2 -18
- package/src/components/AgentMessageList.tsx +18 -15
- package/src/components/AgentVoiceBar.tsx +35 -70
- package/src/components/MessageAttachmentPreview.tsx +43 -0
- package/src/components/VoiceAddMediaModal.tsx +69 -0
- package/src/components/VoiceAttachmentButton.tsx +100 -0
- package/src/components/VoiceIcons.tsx +4 -0
- package/src/components/VoiceWaveform.tsx +15 -5
- package/src/components/useMessageListAutoScroll.ts +57 -0
- package/src/context/AgentAssistantContext.tsx +100 -32
- package/src/index.ts +2 -2
- package/src/react-native.d.ts +18 -1
- package/src/types.ts +3 -8
- package/src/voice/float32ToPcm16.ts +8 -0
- package/src/voice/loadImageCropPicker.ts +18 -0
- package/src/voice/sendVoiceImageAttachment.ts +49 -0
- package/src/voice/useVoiceImagePicker.ts +54 -0
- package/src/voice/useVoiceMicState.ts +38 -3
- package/src/voice/useVoiceSilenceCommit.ts +94 -0
- package/src/voice/useVoiceTranscriptMessages.ts +176 -0
- package/src/voice/useWsRealtimeAudio.ts +200 -0
- package/src/voice/voiceImagePickerOptions.ts +10 -0
- package/src/voice/voiceSilenceConstants.ts +10 -0
- package/src/voice/wsPcmPlayer.ts +166 -0
- package/src/voice/wsPcmRecorder.ts +152 -0
- package/src/voice/wsRealtimeConstants.ts +1 -0
- package/src/components/AgentRemoteAudio.tsx +0 -105
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Message, VoiceDoneEvent, VoiceTranscriptEvent } from '@bytexbyte/nxtlinq-ai-agent-core-development';
|
|
2
|
+
import type { InteractionMode } from '../context/AgentAssistantContext';
|
|
3
|
+
type VoiceTranscriptAgentApi = {
|
|
4
|
+
getMessages: () => Message[];
|
|
5
|
+
setMessages: (messages: Message[]) => void;
|
|
6
|
+
syncVoiceTurnHistory: (options?: {
|
|
7
|
+
last?: number;
|
|
8
|
+
}) => Promise<void>;
|
|
9
|
+
};
|
|
10
|
+
export declare function useVoiceTranscriptMessages(api: VoiceTranscriptAgentApi, interactionMode: InteractionMode, voiceSessionId: string | null): {
|
|
11
|
+
handleTranscript: (event: VoiceTranscriptEvent) => void;
|
|
12
|
+
handleDone: (event: VoiceDoneEvent) => void;
|
|
13
|
+
clearVoiceStream: () => void;
|
|
14
|
+
};
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=useVoiceTranscriptMessages.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useVoiceTranscriptMessages.d.ts","sourceRoot":"","sources":["../../src/voice/useVoiceTranscriptMessages.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,cAAc,EACd,oBAAoB,EACrB,MAAM,8CAA8C,CAAC;AAGtD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAExE,KAAK,uBAAuB,GAAG;IAC7B,WAAW,EAAE,MAAM,OAAO,EAAE,CAAC;IAC7B,WAAW,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAC3C,oBAAoB,EAAE,CAAC,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACtE,CAAC;AAWF,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,uBAAuB,EAC5B,eAAe,EAAE,eAAe,EAChC,cAAc,EAAE,MAAM,GAAG,IAAI;8BA6FnB,oBAAoB;wBA+BpB,cAAc;;EAwBzB"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { mergeStreamingTranscript } from '@bytexbyte/nxtlinq-ai-agent-core-development';
|
|
2
|
+
import { useCallback, useRef } from 'react';
|
|
3
|
+
const STREAM_PREFIX = 'voice-stream-';
|
|
4
|
+
function voiceMeta(sessionId) {
|
|
5
|
+
return {
|
|
6
|
+
voiceRealtime: true,
|
|
7
|
+
voiceSessionId: sessionId ?? undefined,
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
export function useVoiceTranscriptMessages(api, interactionMode, voiceSessionId) {
|
|
11
|
+
const streamIdRef = useRef(null);
|
|
12
|
+
const sessionIdRef = useRef(voiceSessionId);
|
|
13
|
+
sessionIdRef.current = voiceSessionId;
|
|
14
|
+
const isVoiceUiActive = useCallback(() => interactionMode === 'voice' && sessionIdRef.current != null, [interactionMode]);
|
|
15
|
+
const upsertStreaming = useCallback((text) => {
|
|
16
|
+
const messages = api.getMessages();
|
|
17
|
+
let streamId = streamIdRef.current;
|
|
18
|
+
if (!streamId) {
|
|
19
|
+
streamId = `${STREAM_PREFIX}${Date.now()}`;
|
|
20
|
+
streamIdRef.current = streamId;
|
|
21
|
+
}
|
|
22
|
+
const idx = messages.findIndex((m) => m.id === streamId);
|
|
23
|
+
const partialContent = idx >= 0
|
|
24
|
+
? mergeStreamingTranscript(messages[idx]?.partialContent ?? '', text)
|
|
25
|
+
: text;
|
|
26
|
+
const meta = voiceMeta(sessionIdRef.current);
|
|
27
|
+
if (idx >= 0) {
|
|
28
|
+
api.setMessages(messages.map((m, i) => i === idx
|
|
29
|
+
? { ...m, partialContent, isStreaming: true, metadata: { ...m.metadata, ...meta } }
|
|
30
|
+
: m));
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
api.setMessages([
|
|
34
|
+
...messages,
|
|
35
|
+
{
|
|
36
|
+
id: streamId,
|
|
37
|
+
role: 'assistant',
|
|
38
|
+
content: '',
|
|
39
|
+
partialContent,
|
|
40
|
+
isStreaming: true,
|
|
41
|
+
timestamp: new Date().toISOString(),
|
|
42
|
+
metadata: meta,
|
|
43
|
+
},
|
|
44
|
+
]);
|
|
45
|
+
}, [api]);
|
|
46
|
+
const finalizeAssistant = useCallback((text, messageId) => {
|
|
47
|
+
const trimmed = text.trim();
|
|
48
|
+
streamIdRef.current = null;
|
|
49
|
+
if (!trimmed)
|
|
50
|
+
return;
|
|
51
|
+
const messages = api.getMessages();
|
|
52
|
+
const streamIdx = messages.findIndex((m) => m.isStreaming && m.role === 'assistant');
|
|
53
|
+
if (streamIdx >= 0) {
|
|
54
|
+
api.setMessages(messages.map((m, i) => i === streamIdx
|
|
55
|
+
? {
|
|
56
|
+
...m,
|
|
57
|
+
id: messageId ?? m.id,
|
|
58
|
+
content: trimmed,
|
|
59
|
+
partialContent: undefined,
|
|
60
|
+
isStreaming: false,
|
|
61
|
+
metadata: { ...m.metadata, ...voiceMeta(sessionIdRef.current) },
|
|
62
|
+
}
|
|
63
|
+
: m));
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
const last = messages[messages.length - 1];
|
|
67
|
+
if (last?.role === 'assistant' && last.content === trimmed)
|
|
68
|
+
return;
|
|
69
|
+
api.setMessages([
|
|
70
|
+
...messages,
|
|
71
|
+
{
|
|
72
|
+
id: messageId ?? `voice-asst-${Date.now()}`,
|
|
73
|
+
role: 'assistant',
|
|
74
|
+
content: trimmed,
|
|
75
|
+
timestamp: new Date().toISOString(),
|
|
76
|
+
metadata: voiceMeta(sessionIdRef.current),
|
|
77
|
+
},
|
|
78
|
+
]);
|
|
79
|
+
}, [api]);
|
|
80
|
+
const handleTranscript = useCallback((event) => {
|
|
81
|
+
if (!isVoiceUiActive())
|
|
82
|
+
return;
|
|
83
|
+
const text = event.text?.trim() ?? '';
|
|
84
|
+
if (event.role === 'assistant') {
|
|
85
|
+
if (event.interim) {
|
|
86
|
+
if (text)
|
|
87
|
+
upsertStreaming(text);
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
if (text)
|
|
91
|
+
finalizeAssistant(text);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
if (event.role === 'user' && !event.interim && text) {
|
|
95
|
+
const messages = api.getMessages();
|
|
96
|
+
const last = messages[messages.length - 1];
|
|
97
|
+
if (last?.role === 'user' && last.content === text)
|
|
98
|
+
return;
|
|
99
|
+
api.setMessages([
|
|
100
|
+
...messages,
|
|
101
|
+
{
|
|
102
|
+
id: `voice-user-${Date.now()}`,
|
|
103
|
+
role: 'user',
|
|
104
|
+
content: text,
|
|
105
|
+
timestamp: new Date().toISOString(),
|
|
106
|
+
metadata: voiceMeta(sessionIdRef.current),
|
|
107
|
+
},
|
|
108
|
+
]);
|
|
109
|
+
}
|
|
110
|
+
}, [api, finalizeAssistant, isVoiceUiActive, upsertStreaming]);
|
|
111
|
+
const handleDone = useCallback((event) => {
|
|
112
|
+
if (!isVoiceUiActive())
|
|
113
|
+
return;
|
|
114
|
+
if (event.guardrailsBlocked || event.billingBlocked || event.error) {
|
|
115
|
+
streamIdRef.current = null;
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
const reply = event.replyText?.trim() ?? '';
|
|
119
|
+
if (reply) {
|
|
120
|
+
finalizeAssistant(reply, event.assistantMessageId ?? undefined);
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
streamIdRef.current = null;
|
|
124
|
+
}
|
|
125
|
+
void api.syncVoiceTurnHistory({ last: 20 }).catch((err) => {
|
|
126
|
+
console.warn('[nxtlinq] syncVoiceTurnHistory after voice turn failed', err);
|
|
127
|
+
});
|
|
128
|
+
}, [api, finalizeAssistant, isVoiceUiActive]);
|
|
129
|
+
const clearVoiceStream = useCallback(() => {
|
|
130
|
+
streamIdRef.current = null;
|
|
131
|
+
}, []);
|
|
132
|
+
return { handleTranscript, handleDone, clearVoiceStream };
|
|
133
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { VoiceSession, VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
|
|
2
|
+
import type { UseNxtlinqVoiceOptions } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
|
|
3
|
+
type WsVoiceCallbacks = Pick<UseNxtlinqVoiceOptions, 'onOpen' | 'onAudioDelta' | 'onClose' | 'onError'>;
|
|
4
|
+
export type UseWsRealtimeAudioOptions = {
|
|
5
|
+
voiceStatus: VoiceStatus;
|
|
6
|
+
muteAfterSilenceCommit: () => void;
|
|
7
|
+
};
|
|
8
|
+
export declare function useWsRealtimeAudio(isCaptureActive: boolean, isVoiceActive: boolean, options: UseWsRealtimeAudioOptions): {
|
|
9
|
+
buildCallbacks: (overrides?: Partial<WsVoiceCallbacks>) => WsVoiceCallbacks;
|
|
10
|
+
bindSession: (session: VoiceSession | null, captureWhenUnmuted?: boolean) => void;
|
|
11
|
+
cleanup: () => void;
|
|
12
|
+
beginCapture: () => Promise<void>;
|
|
13
|
+
endCapture: (commit: boolean) => void;
|
|
14
|
+
getOutputAudioLevel: () => number;
|
|
15
|
+
};
|
|
16
|
+
export {};
|
|
17
|
+
//# sourceMappingURL=useWsRealtimeAudio.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useWsRealtimeAudio.d.ts","sourceRoot":"","sources":["../../src/voice/useWsRealtimeAudio.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,8CAA8C,CAAC;AAC9F,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,sDAAsD,CAAC;AAOnG,KAAK,gBAAgB,GAAG,IAAI,CAC1B,sBAAsB,EACtB,QAAQ,GAAG,cAAc,GAAG,SAAS,GAAG,SAAS,CAClD,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,WAAW,EAAE,WAAW,CAAC;IACzB,sBAAsB,EAAE,MAAM,IAAI,CAAC;CACpC,CAAC;AAYF,wBAAgB,kBAAkB,CAChC,eAAe,EAAE,OAAO,EACxB,aAAa,EAAE,OAAO,EACtB,OAAO,EAAE,yBAAyB;iCAsGnB,OAAO,CAAC,gBAAgB,CAAC,KAAG,gBAAgB;2BAX/C,YAAY,GAAG,IAAI;;;yBA7DU,OAAO;;EA0IjD"}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { waitForIOSAudioSessionReady } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
|
|
2
|
+
import { useCallback, useEffect, useRef } from 'react';
|
|
3
|
+
import { useVoiceSilenceCommit } from './useVoiceSilenceCommit';
|
|
4
|
+
import { WsPcmPlayer } from './wsPcmPlayer';
|
|
5
|
+
import { WsPcmRecorder } from './wsPcmRecorder';
|
|
6
|
+
function isIOS() {
|
|
7
|
+
try {
|
|
8
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
9
|
+
const { Platform } = require('react-native');
|
|
10
|
+
return Platform.OS === 'ios';
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return false;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
export function useWsRealtimeAudio(isCaptureActive, isVoiceActive, options) {
|
|
17
|
+
const playerRef = useRef(null);
|
|
18
|
+
const recorderRef = useRef(null);
|
|
19
|
+
const sessionRef = useRef(null);
|
|
20
|
+
const isCaptureActiveRef = useRef(isCaptureActive);
|
|
21
|
+
isCaptureActiveRef.current = isCaptureActive;
|
|
22
|
+
const prevCaptureActiveRef = useRef(isCaptureActive);
|
|
23
|
+
const iosSessionPrimedRef = useRef(false);
|
|
24
|
+
const getSession = useCallback(() => sessionRef.current, []);
|
|
25
|
+
const muteAfterSilenceCommitRef = useRef(options.muteAfterSilenceCommit);
|
|
26
|
+
muteAfterSilenceCommitRef.current = options.muteAfterSilenceCommit;
|
|
27
|
+
const silence = useVoiceSilenceCommit(getSession, () => muteAfterSilenceCommitRef.current(), options.voiceStatus);
|
|
28
|
+
const silenceRef = useRef(silence);
|
|
29
|
+
silenceRef.current = silence;
|
|
30
|
+
const ensurePlayer = useCallback(async () => {
|
|
31
|
+
if (!playerRef.current) {
|
|
32
|
+
playerRef.current = new WsPcmPlayer();
|
|
33
|
+
playerRef.current.prewarm();
|
|
34
|
+
}
|
|
35
|
+
await playerRef.current.ensureRunning();
|
|
36
|
+
}, []);
|
|
37
|
+
const stopCapture = useCallback((commit) => {
|
|
38
|
+
const s = silenceRef.current;
|
|
39
|
+
s.clearPoll();
|
|
40
|
+
recorderRef.current?.stop();
|
|
41
|
+
if (!commit)
|
|
42
|
+
return;
|
|
43
|
+
if (s.consumeSkipCommitOnMute())
|
|
44
|
+
return;
|
|
45
|
+
s.tryCommit('manual');
|
|
46
|
+
}, []);
|
|
47
|
+
const startCapture = useCallback(async () => {
|
|
48
|
+
const session = sessionRef.current;
|
|
49
|
+
if (!session) {
|
|
50
|
+
console.warn('[nxtlinq] startCapture skipped: voice session not bound');
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
if (isIOS() && !iosSessionPrimedRef.current) {
|
|
54
|
+
try {
|
|
55
|
+
await waitForIOSAudioSessionReady();
|
|
56
|
+
iosSessionPrimedRef.current = true;
|
|
57
|
+
}
|
|
58
|
+
catch (err) {
|
|
59
|
+
console.warn('[nxtlinq] waitForIOSAudioSessionReady failed', err);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const s = silenceRef.current;
|
|
63
|
+
if (!recorderRef.current) {
|
|
64
|
+
recorderRef.current = new WsPcmRecorder();
|
|
65
|
+
}
|
|
66
|
+
playerRef.current?.clearQueue();
|
|
67
|
+
recorderRef.current.bindSession(session);
|
|
68
|
+
recorderRef.current.setOnRms(s.onSpeechRms);
|
|
69
|
+
s.resetTurn();
|
|
70
|
+
try {
|
|
71
|
+
await recorderRef.current.initialize();
|
|
72
|
+
await recorderRef.current.start();
|
|
73
|
+
s.startPoll();
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
s.clearPoll();
|
|
77
|
+
console.error('[nxtlinq] mic capture start failed', err);
|
|
78
|
+
}
|
|
79
|
+
}, []);
|
|
80
|
+
const cleanup = useCallback(() => {
|
|
81
|
+
const s = silenceRef.current;
|
|
82
|
+
s.clearPoll();
|
|
83
|
+
iosSessionPrimedRef.current = false;
|
|
84
|
+
recorderRef.current?.stop();
|
|
85
|
+
recorderRef.current?.cleanup();
|
|
86
|
+
recorderRef.current = null;
|
|
87
|
+
playerRef.current?.cleanup();
|
|
88
|
+
playerRef.current = null;
|
|
89
|
+
sessionRef.current = null;
|
|
90
|
+
}, []);
|
|
91
|
+
const stopCaptureRef = useRef(stopCapture);
|
|
92
|
+
const startCaptureRef = useRef(startCapture);
|
|
93
|
+
const cleanupRef = useRef(cleanup);
|
|
94
|
+
stopCaptureRef.current = stopCapture;
|
|
95
|
+
startCaptureRef.current = startCapture;
|
|
96
|
+
cleanupRef.current = cleanup;
|
|
97
|
+
const bindSession = useCallback((session, captureWhenUnmuted = false) => {
|
|
98
|
+
sessionRef.current = session;
|
|
99
|
+
recorderRef.current?.bindSession(session);
|
|
100
|
+
if (session && captureWhenUnmuted && isCaptureActiveRef.current) {
|
|
101
|
+
void startCaptureRef.current();
|
|
102
|
+
}
|
|
103
|
+
}, []);
|
|
104
|
+
const buildCallbacks = useCallback((overrides) => ({
|
|
105
|
+
onOpen: () => {
|
|
106
|
+
void ensurePlayer();
|
|
107
|
+
overrides?.onOpen?.();
|
|
108
|
+
},
|
|
109
|
+
onAudioDelta: (pcm16) => {
|
|
110
|
+
void ensurePlayer().then(() => playerRef.current?.addAudio(pcm16));
|
|
111
|
+
overrides?.onAudioDelta?.(pcm16);
|
|
112
|
+
},
|
|
113
|
+
onClose: (reason) => {
|
|
114
|
+
cleanupRef.current();
|
|
115
|
+
overrides?.onClose?.(reason);
|
|
116
|
+
},
|
|
117
|
+
onError: (err) => {
|
|
118
|
+
cleanupRef.current();
|
|
119
|
+
overrides?.onError?.(err);
|
|
120
|
+
},
|
|
121
|
+
}), [ensurePlayer]);
|
|
122
|
+
useEffect(() => {
|
|
123
|
+
if (!isVoiceActive) {
|
|
124
|
+
prevCaptureActiveRef.current = false;
|
|
125
|
+
cleanupRef.current();
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
const prev = prevCaptureActiveRef.current;
|
|
129
|
+
prevCaptureActiveRef.current = isCaptureActive;
|
|
130
|
+
if (isCaptureActive && !prev) {
|
|
131
|
+
void startCaptureRef.current().catch((err) => {
|
|
132
|
+
console.error('[nxtlinq] mic capture start failed', err);
|
|
133
|
+
});
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
if (!isCaptureActive && prev) {
|
|
137
|
+
stopCaptureRef.current(true);
|
|
138
|
+
}
|
|
139
|
+
}, [isCaptureActive, isVoiceActive]);
|
|
140
|
+
useEffect(() => {
|
|
141
|
+
if (!isVoiceActive)
|
|
142
|
+
return;
|
|
143
|
+
if (!recorderRef.current) {
|
|
144
|
+
recorderRef.current = new WsPcmRecorder();
|
|
145
|
+
}
|
|
146
|
+
void recorderRef.current.initialize().catch((err) => {
|
|
147
|
+
console.warn('[nxtlinq] WsPcmRecorder prewarm failed', err);
|
|
148
|
+
});
|
|
149
|
+
}, [isVoiceActive]);
|
|
150
|
+
useEffect(() => () => cleanupRef.current(), []);
|
|
151
|
+
const getOutputAudioLevel = useCallback(() => {
|
|
152
|
+
const fromPlayer = playerRef.current?.getAudioLevel() ?? 0;
|
|
153
|
+
if (fromPlayer > 0)
|
|
154
|
+
return fromPlayer;
|
|
155
|
+
return sessionRef.current?.getOutputAudioLevel() ?? 0;
|
|
156
|
+
}, []);
|
|
157
|
+
return {
|
|
158
|
+
buildCallbacks,
|
|
159
|
+
bindSession,
|
|
160
|
+
cleanup,
|
|
161
|
+
beginCapture: startCapture,
|
|
162
|
+
endCapture: stopCapture,
|
|
163
|
+
getOutputAudioLevel,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Smaller payload for WS / SCTP voice channels (Berify-aligned). */
|
|
2
|
+
export declare const VOICE_IMAGE_PICKER_OPTIONS: {
|
|
3
|
+
mediaType: "photo";
|
|
4
|
+
multiple: boolean;
|
|
5
|
+
compressImageMaxWidth: number;
|
|
6
|
+
compressImageMaxHeight: number;
|
|
7
|
+
compressImageQuality: number;
|
|
8
|
+
/** iOS albums are often HEIC; force JPEG for OpenAI-compatible MIME. */
|
|
9
|
+
forceJpg: boolean;
|
|
10
|
+
};
|
|
11
|
+
//# sourceMappingURL=voiceImagePickerOptions.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voiceImagePickerOptions.d.ts","sourceRoot":"","sources":["../../src/voice/voiceImagePickerOptions.ts"],"names":[],"mappings":"AAAA,qEAAqE;AACrE,eAAO,MAAM,0BAA0B;;;;;;IAMrC,wEAAwE;;CAEzE,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** Smaller payload for WS / SCTP voice channels (Berify-aligned). */
|
|
2
|
+
export const VOICE_IMAGE_PICKER_OPTIONS = {
|
|
3
|
+
mediaType: 'photo',
|
|
4
|
+
multiple: false,
|
|
5
|
+
compressImageMaxWidth: 384,
|
|
6
|
+
compressImageMaxHeight: 384,
|
|
7
|
+
compressImageQuality: 0.55,
|
|
8
|
+
/** iOS albums are often HEIC; force JPEG for OpenAI-compatible MIME. */
|
|
9
|
+
forceJpg: true,
|
|
10
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** Min RMS to treat as user speech (~-42 dBFS). Berify-aligned. */
|
|
2
|
+
export declare const MIC_SPEECH_RMS_THRESHOLD = 0.008;
|
|
3
|
+
/** Louder threshold while assistant speaks — reduces echo false triggers. */
|
|
4
|
+
export declare const MIC_BARGE_IN_RMS_THRESHOLD = 0.045;
|
|
5
|
+
/** Commit turn after this much silence while mic is open. */
|
|
6
|
+
export declare const MIC_SILENCE_COMMIT_MS = 400;
|
|
7
|
+
export declare const MIC_SILENCE_POLL_MS = 100;
|
|
8
|
+
//# sourceMappingURL=voiceSilenceConstants.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voiceSilenceConstants.d.ts","sourceRoot":"","sources":["../../src/voice/voiceSilenceConstants.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAE9C,6EAA6E;AAC7E,eAAO,MAAM,0BAA0B,QAAQ,CAAC;AAEhD,6DAA6D;AAC7D,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,eAAO,MAAM,mBAAmB,MAAM,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** Min RMS to treat as user speech (~-42 dBFS). Berify-aligned. */
|
|
2
|
+
export const MIC_SPEECH_RMS_THRESHOLD = 0.008;
|
|
3
|
+
/** Louder threshold while assistant speaks — reduces echo false triggers. */
|
|
4
|
+
export const MIC_BARGE_IN_RMS_THRESHOLD = 0.045;
|
|
5
|
+
/** Commit turn after this much silence while mic is open. */
|
|
6
|
+
export const MIC_SILENCE_COMMIT_MS = 400;
|
|
7
|
+
export const MIC_SILENCE_POLL_MS = 100;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export declare class WsPcmPlayer {
|
|
2
|
+
private readonly sampleRate;
|
|
3
|
+
private readonly fadeSamples;
|
|
4
|
+
private audioContext;
|
|
5
|
+
private gainNode;
|
|
6
|
+
private analyserNode;
|
|
7
|
+
private analyserBuffer;
|
|
8
|
+
private readonly activeSources;
|
|
9
|
+
private queue;
|
|
10
|
+
private isPlaying;
|
|
11
|
+
private playHead;
|
|
12
|
+
private lastChunkRms;
|
|
13
|
+
private lastChunkAt;
|
|
14
|
+
ensureRunning(): Promise<void>;
|
|
15
|
+
prewarm(): void;
|
|
16
|
+
addAudio(pcm16: Int16Array | ArrayBuffer): void;
|
|
17
|
+
getAudioLevel(): number;
|
|
18
|
+
clearQueue(): void;
|
|
19
|
+
cleanup(): void;
|
|
20
|
+
private ensureContext;
|
|
21
|
+
/** Berify WavStreamPlayer: schedule chunks with lookahead + crossfade, no gaps. */
|
|
22
|
+
private pipelineQueue;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=wsPcmPlayer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wsPcmPlayer.d.ts","sourceRoot":"","sources":["../../src/voice/wsPcmPlayer.ts"],"names":[],"mappings":"AAsBA,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA2B;IACtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmD;IAC/E,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAyB;IACzC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoC;IAClE,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,SAAS,CAAQ;IACzB,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,WAAW,CAAK;IAElB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAQpC,OAAO,IAAI,IAAI;IAIf,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,GAAG,IAAI;IAuB/C,aAAa,IAAI,MAAM;IAkBvB,UAAU,IAAI,IAAI;IAelB,OAAO,IAAI,IAAI;IASf,OAAO,CAAC,aAAa;IAcrB,mFAAmF;IACnF,OAAO,CAAC,aAAa;CAqCtB"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { AudioContext, } from 'react-native-audio-api';
|
|
2
|
+
import { WS_REALTIME_SAMPLE_RATE } from './wsRealtimeConstants';
|
|
3
|
+
const LEVEL_SCALE = 3.2;
|
|
4
|
+
const LOOKAHEAD_SECONDS = 0.02;
|
|
5
|
+
const CROSSFADE_SECONDS = 0.04;
|
|
6
|
+
function applyFade(buffer, fadeSamples) {
|
|
7
|
+
const fade = Math.min(fadeSamples, Math.floor(buffer.length / 2));
|
|
8
|
+
if (fade <= 0)
|
|
9
|
+
return;
|
|
10
|
+
for (let i = 0; i < fade; i += 1) {
|
|
11
|
+
buffer[i] *= i / fade;
|
|
12
|
+
buffer[buffer.length - 1 - i] *= (fade - i) / fade;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
export class WsPcmPlayer {
|
|
16
|
+
constructor() {
|
|
17
|
+
this.sampleRate = WS_REALTIME_SAMPLE_RATE;
|
|
18
|
+
this.fadeSamples = Math.max(1, Math.floor(this.sampleRate * 0.01));
|
|
19
|
+
this.audioContext = null;
|
|
20
|
+
this.gainNode = null;
|
|
21
|
+
this.analyserNode = null;
|
|
22
|
+
this.analyserBuffer = null;
|
|
23
|
+
this.activeSources = new Set();
|
|
24
|
+
this.queue = [];
|
|
25
|
+
this.isPlaying = true;
|
|
26
|
+
this.playHead = 0;
|
|
27
|
+
this.lastChunkRms = 0;
|
|
28
|
+
this.lastChunkAt = 0;
|
|
29
|
+
}
|
|
30
|
+
async ensureRunning() {
|
|
31
|
+
const ctx = this.ensureContext();
|
|
32
|
+
if (ctx.state === 'suspended') {
|
|
33
|
+
await ctx.resume();
|
|
34
|
+
}
|
|
35
|
+
this.isPlaying = true;
|
|
36
|
+
}
|
|
37
|
+
prewarm() {
|
|
38
|
+
this.ensureContext();
|
|
39
|
+
}
|
|
40
|
+
addAudio(pcm16) {
|
|
41
|
+
const ctx = this.ensureContext();
|
|
42
|
+
const int16 = pcm16 instanceof ArrayBuffer ? new Int16Array(pcm16) : pcm16;
|
|
43
|
+
if (int16.length === 0)
|
|
44
|
+
return;
|
|
45
|
+
const floats = new Float32Array(int16.length);
|
|
46
|
+
let sumSq = 0;
|
|
47
|
+
for (let i = 0; i < int16.length; i += 1) {
|
|
48
|
+
const sample = int16[i] / 32768;
|
|
49
|
+
floats[i] = Math.max(-1, Math.min(1, sample));
|
|
50
|
+
sumSq += sample * sample;
|
|
51
|
+
}
|
|
52
|
+
applyFade(floats, this.fadeSamples);
|
|
53
|
+
this.lastChunkRms = Math.min(1, Math.sqrt(sumSq / int16.length) * LEVEL_SCALE);
|
|
54
|
+
this.lastChunkAt = Date.now();
|
|
55
|
+
const buffer = ctx.createBuffer(1, floats.length, this.sampleRate);
|
|
56
|
+
buffer.getChannelData(0).set(floats);
|
|
57
|
+
this.queue.push(buffer);
|
|
58
|
+
this.isPlaying = true;
|
|
59
|
+
this.pipelineQueue();
|
|
60
|
+
}
|
|
61
|
+
getAudioLevel() {
|
|
62
|
+
const analyser = this.analyserNode;
|
|
63
|
+
const buf = this.analyserBuffer;
|
|
64
|
+
if (analyser && buf && this.activeSources.size > 0) {
|
|
65
|
+
analyser.getFloatTimeDomainData(buf);
|
|
66
|
+
let sumSq = 0;
|
|
67
|
+
for (let i = 0; i < buf.length; i += 1) {
|
|
68
|
+
const v = buf[i];
|
|
69
|
+
sumSq += v * v;
|
|
70
|
+
}
|
|
71
|
+
return Math.min(1, Math.sqrt(sumSq / buf.length) * LEVEL_SCALE);
|
|
72
|
+
}
|
|
73
|
+
if (Date.now() - this.lastChunkAt < 500) {
|
|
74
|
+
return this.lastChunkRms;
|
|
75
|
+
}
|
|
76
|
+
return 0;
|
|
77
|
+
}
|
|
78
|
+
clearQueue() {
|
|
79
|
+
for (const source of this.activeSources) {
|
|
80
|
+
try {
|
|
81
|
+
source.stop();
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
/* already stopped */
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
this.activeSources.clear();
|
|
88
|
+
this.queue = [];
|
|
89
|
+
this.playHead = this.audioContext?.currentTime ?? 0;
|
|
90
|
+
this.lastChunkRms = 0;
|
|
91
|
+
this.lastChunkAt = 0;
|
|
92
|
+
}
|
|
93
|
+
cleanup() {
|
|
94
|
+
this.clearQueue();
|
|
95
|
+
this.isPlaying = false;
|
|
96
|
+
this.analyserNode = null;
|
|
97
|
+
this.analyserBuffer = null;
|
|
98
|
+
this.gainNode = null;
|
|
99
|
+
this.audioContext = null;
|
|
100
|
+
}
|
|
101
|
+
ensureContext() {
|
|
102
|
+
if (!this.audioContext) {
|
|
103
|
+
this.audioContext = new AudioContext({ sampleRate: this.sampleRate });
|
|
104
|
+
this.gainNode = this.audioContext.createGain();
|
|
105
|
+
this.analyserNode = this.audioContext.createAnalyser();
|
|
106
|
+
this.analyserNode.fftSize = 512;
|
|
107
|
+
this.analyserBuffer = new Float32Array(this.analyserNode.fftSize);
|
|
108
|
+
this.gainNode.connect(this.analyserNode);
|
|
109
|
+
this.analyserNode.connect(this.audioContext.destination);
|
|
110
|
+
this.playHead = this.audioContext.currentTime;
|
|
111
|
+
}
|
|
112
|
+
return this.audioContext;
|
|
113
|
+
}
|
|
114
|
+
/** Berify WavStreamPlayer: schedule chunks with lookahead + crossfade, no gaps. */
|
|
115
|
+
pipelineQueue() {
|
|
116
|
+
const ctx = this.audioContext;
|
|
117
|
+
const gain = this.gainNode;
|
|
118
|
+
if (!ctx || !gain || !this.isPlaying)
|
|
119
|
+
return;
|
|
120
|
+
while (this.queue.length > 0) {
|
|
121
|
+
const audioBuffer = this.queue.shift();
|
|
122
|
+
if (!audioBuffer)
|
|
123
|
+
return;
|
|
124
|
+
const source = ctx.createBufferSource();
|
|
125
|
+
source.buffer = audioBuffer;
|
|
126
|
+
const perSourceGain = ctx.createGain();
|
|
127
|
+
source.connect(perSourceGain);
|
|
128
|
+
perSourceGain.connect(gain);
|
|
129
|
+
const now = ctx.currentTime;
|
|
130
|
+
const startAt = Math.max((this.playHead || now) - CROSSFADE_SECONDS * 0.75, now + LOOKAHEAD_SECONDS);
|
|
131
|
+
const duration = audioBuffer.duration;
|
|
132
|
+
const endAt = startAt + duration;
|
|
133
|
+
const fade = Math.min(CROSSFADE_SECONDS, duration / 2);
|
|
134
|
+
perSourceGain.gain.setValueAtTime(0, startAt);
|
|
135
|
+
perSourceGain.gain.linearRampToValueAtTime(1, startAt + fade);
|
|
136
|
+
perSourceGain.gain.setValueAtTime(1, endAt - fade);
|
|
137
|
+
perSourceGain.gain.linearRampToValueAtTime(0, endAt);
|
|
138
|
+
this.playHead = endAt;
|
|
139
|
+
this.activeSources.add(source);
|
|
140
|
+
source.start(startAt);
|
|
141
|
+
source.onEnded = () => {
|
|
142
|
+
this.activeSources.delete(source);
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { VoiceSession } from '@bytexbyte/nxtlinq-ai-agent-core-development';
|
|
2
|
+
/**
|
|
3
|
+
* Berify {@link WavRecorder}-aligned lifecycle: reuse one native AudioRecorder,
|
|
4
|
+
* configure playAndRecord on start, reset to playback on stop.
|
|
5
|
+
*/
|
|
6
|
+
export declare class WsPcmRecorder {
|
|
7
|
+
private recorder;
|
|
8
|
+
private recording;
|
|
9
|
+
private audioReadyAttached;
|
|
10
|
+
private session;
|
|
11
|
+
private onRms?;
|
|
12
|
+
initialize(options?: {
|
|
13
|
+
force?: boolean;
|
|
14
|
+
}): Promise<void>;
|
|
15
|
+
bindSession(session: VoiceSession | null): void;
|
|
16
|
+
setOnRms(handler: (rms: number) => void): void;
|
|
17
|
+
private disposeRecorderInternal;
|
|
18
|
+
start(): Promise<void>;
|
|
19
|
+
stop(): void;
|
|
20
|
+
cleanup(): void;
|
|
21
|
+
get isRecording(): boolean;
|
|
22
|
+
private configureSession;
|
|
23
|
+
private resetSession;
|
|
24
|
+
private handleAudioReady;
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=wsPcmRecorder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wsPcmRecorder.d.ts","sourceRoot":"","sources":["../../src/voice/wsPcmRecorder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,8CAA8C,CAAC;AASjF;;;GAGG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,KAAK,CAAC,CAAwB;IAEhC,UAAU,CAAC,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAY9D,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,GAAG,IAAI;IAI/C,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;IAI9C,OAAO,CAAC,uBAAuB;IAazB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgC5B,IAAI,IAAI,IAAI;IAWZ,OAAO,IAAI,IAAI;IAmBf,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,gBAAgB;CAYzB"}
|