@bytexbyte/nxtlinq-ai-agent-ui-react-native-development 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/NxtlinqAgentAssistant.d.ts +4 -4
- package/dist/NxtlinqAgentAssistant.d.ts.map +1 -1
- package/dist/NxtlinqAgentAssistant.js +5 -6
- package/dist/components/AgentAssistantShell.d.ts +1 -3
- package/dist/components/AgentAssistantShell.d.ts.map +1 -1
- package/dist/components/AgentAssistantShell.js +3 -7
- package/dist/components/AgentMessageList.d.ts.map +1 -1
- package/dist/components/AgentMessageList.js +7 -9
- package/dist/components/AgentVoiceBar.d.ts.map +1 -1
- package/dist/components/AgentVoiceBar.js +14 -34
- package/dist/components/MessageAttachmentPreview.d.ts +10 -0
- package/dist/components/MessageAttachmentPreview.d.ts.map +1 -0
- package/dist/components/MessageAttachmentPreview.js +15 -0
- package/dist/components/VoiceAddMediaModal.d.ts +12 -0
- package/dist/components/VoiceAddMediaModal.d.ts.map +1 -0
- package/dist/components/VoiceAddMediaModal.js +31 -0
- package/dist/components/VoiceAttachmentButton.d.ts +3 -0
- package/dist/components/VoiceAttachmentButton.d.ts.map +1 -0
- package/dist/components/VoiceAttachmentButton.js +58 -0
- package/dist/components/VoiceIcons.d.ts +1 -0
- package/dist/components/VoiceIcons.d.ts.map +1 -1
- package/dist/components/VoiceIcons.js +3 -0
- package/dist/components/VoiceWaveform.d.ts +2 -2
- package/dist/components/VoiceWaveform.d.ts.map +1 -1
- package/dist/components/VoiceWaveform.js +16 -5
- package/dist/components/useMessageListAutoScroll.d.ts +12 -0
- package/dist/components/useMessageListAutoScroll.d.ts.map +1 -0
- package/dist/components/useMessageListAutoScroll.js +42 -0
- package/dist/context/AgentAssistantContext.d.ts +3 -3
- package/dist/context/AgentAssistantContext.d.ts.map +1 -1
- package/dist/context/AgentAssistantContext.js +76 -29
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/types.d.ts +3 -8
- package/dist/types.d.ts.map +1 -1
- package/dist/voice/float32ToPcm16.d.ts +2 -0
- package/dist/voice/float32ToPcm16.d.ts.map +1 -0
- package/dist/voice/float32ToPcm16.js +8 -0
- package/dist/voice/loadImageCropPicker.d.ts +11 -0
- package/dist/voice/loadImageCropPicker.d.ts.map +1 -0
- package/dist/voice/loadImageCropPicker.js +12 -0
- package/dist/voice/sendVoiceImageAttachment.d.ts +15 -0
- package/dist/voice/sendVoiceImageAttachment.d.ts.map +1 -0
- package/dist/voice/sendVoiceImageAttachment.js +29 -0
- package/dist/voice/useVoiceImagePicker.d.ts +11 -0
- package/dist/voice/useVoiceImagePicker.d.ts.map +1 -0
- package/dist/voice/useVoiceImagePicker.js +38 -0
- package/dist/voice/useVoiceMicState.d.ts +4 -0
- package/dist/voice/useVoiceMicState.d.ts.map +1 -1
- package/dist/voice/useVoiceMicState.js +32 -3
- package/dist/voice/useVoiceSilenceCommit.d.ts +10 -0
- package/dist/voice/useVoiceSilenceCommit.d.ts.map +1 -0
- package/dist/voice/useVoiceSilenceCommit.js +76 -0
- package/dist/voice/useVoiceTranscriptMessages.d.ts +16 -0
- package/dist/voice/useVoiceTranscriptMessages.d.ts.map +1 -0
- package/dist/voice/useVoiceTranscriptMessages.js +133 -0
- package/dist/voice/useWsRealtimeAudio.d.ts +17 -0
- package/dist/voice/useWsRealtimeAudio.d.ts.map +1 -0
- package/dist/voice/useWsRealtimeAudio.js +165 -0
- package/dist/voice/voiceImagePickerOptions.d.ts +11 -0
- package/dist/voice/voiceImagePickerOptions.d.ts.map +1 -0
- package/dist/voice/voiceImagePickerOptions.js +10 -0
- package/dist/voice/voiceSilenceConstants.d.ts +8 -0
- package/dist/voice/voiceSilenceConstants.d.ts.map +1 -0
- package/dist/voice/voiceSilenceConstants.js +7 -0
- package/dist/voice/wsPcmPlayer.d.ts +24 -0
- package/dist/voice/wsPcmPlayer.d.ts.map +1 -0
- package/dist/voice/wsPcmPlayer.js +146 -0
- package/dist/voice/wsPcmRecorder.d.ts +26 -0
- package/dist/voice/wsPcmRecorder.d.ts.map +1 -0
- package/dist/voice/wsPcmRecorder.js +145 -0
- package/dist/voice/wsRealtimeConstants.d.ts +2 -0
- package/dist/voice/wsRealtimeConstants.d.ts.map +1 -0
- package/dist/voice/wsRealtimeConstants.js +1 -0
- package/package.json +8 -5
- package/src/NxtlinqAgentAssistant.tsx +3 -12
- package/src/components/AgentAssistantShell.tsx +2 -18
- package/src/components/AgentMessageList.tsx +18 -15
- package/src/components/AgentVoiceBar.tsx +35 -70
- package/src/components/MessageAttachmentPreview.tsx +43 -0
- package/src/components/VoiceAddMediaModal.tsx +69 -0
- package/src/components/VoiceAttachmentButton.tsx +100 -0
- package/src/components/VoiceIcons.tsx +4 -0
- package/src/components/VoiceWaveform.tsx +15 -5
- package/src/components/useMessageListAutoScroll.ts +57 -0
- package/src/context/AgentAssistantContext.tsx +100 -32
- package/src/index.ts +2 -2
- package/src/react-native.d.ts +18 -1
- package/src/types.ts +3 -8
- package/src/voice/float32ToPcm16.ts +8 -0
- package/src/voice/loadImageCropPicker.ts +18 -0
- package/src/voice/sendVoiceImageAttachment.ts +49 -0
- package/src/voice/useVoiceImagePicker.ts +54 -0
- package/src/voice/useVoiceMicState.ts +38 -3
- package/src/voice/useVoiceSilenceCommit.ts +94 -0
- package/src/voice/useVoiceTranscriptMessages.ts +176 -0
- package/src/voice/useWsRealtimeAudio.ts +200 -0
- package/src/voice/voiceImagePickerOptions.ts +10 -0
- package/src/voice/voiceSilenceConstants.ts +10 -0
- package/src/voice/wsPcmPlayer.ts +166 -0
- package/src/voice/wsPcmRecorder.ts +152 -0
- package/src/voice/wsRealtimeConstants.ts +1 -0
- package/src/components/AgentRemoteAudio.tsx +0 -105
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import React, { useCallback, useState } from 'react';
|
|
2
|
+
import { ActivityIndicator, Pressable, StyleSheet } from 'react-native';
|
|
3
|
+
import { useAgentAssistant } from '../context/AgentAssistantContext';
|
|
4
|
+
import { sendVoiceImageAttachment } from '../voice/sendVoiceImageAttachment';
|
|
5
|
+
import { useVoiceImagePicker } from '../voice/useVoiceImagePicker';
|
|
6
|
+
import { AddIcon } from './VoiceIcons';
|
|
7
|
+
import { VoiceAddMediaModal } from './VoiceAddMediaModal';
|
|
8
|
+
|
|
9
|
+
export function VoiceAttachmentButton(): React.ReactElement {
|
|
10
|
+
const {
|
|
11
|
+
theme,
|
|
12
|
+
messages,
|
|
13
|
+
setMessages,
|
|
14
|
+
sendVoiceUserInput,
|
|
15
|
+
voiceSessionId,
|
|
16
|
+
isVoiceChannelReady,
|
|
17
|
+
isVoiceConnecting,
|
|
18
|
+
} = useAgentAssistant();
|
|
19
|
+
const [busy, setBusy] = useState(false);
|
|
20
|
+
|
|
21
|
+
const handlePicked = useCallback(
|
|
22
|
+
async (fileUri: string) => {
|
|
23
|
+
if (!isVoiceChannelReady || busy) return;
|
|
24
|
+
setBusy(true);
|
|
25
|
+
try {
|
|
26
|
+
await sendVoiceImageAttachment({
|
|
27
|
+
fileUri,
|
|
28
|
+
messages,
|
|
29
|
+
voiceSessionId,
|
|
30
|
+
setMessages,
|
|
31
|
+
sendVoiceUserInput,
|
|
32
|
+
});
|
|
33
|
+
} catch (e) {
|
|
34
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
35
|
+
console.warn('[nxtlinq] sendVoiceImageAttachment failed:', message);
|
|
36
|
+
} finally {
|
|
37
|
+
setBusy(false);
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
[
|
|
41
|
+
busy,
|
|
42
|
+
isVoiceChannelReady,
|
|
43
|
+
messages,
|
|
44
|
+
sendVoiceUserInput,
|
|
45
|
+
setMessages,
|
|
46
|
+
voiceSessionId,
|
|
47
|
+
],
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
const {
|
|
51
|
+
modalVisible,
|
|
52
|
+
openModal,
|
|
53
|
+
closeModal,
|
|
54
|
+
pickFromLibrary,
|
|
55
|
+
takePhoto,
|
|
56
|
+
pickerMissing,
|
|
57
|
+
} = useVoiceImagePicker(handlePicked);
|
|
58
|
+
|
|
59
|
+
const disabled = busy || !isVoiceChannelReady || isVoiceConnecting || pickerMissing;
|
|
60
|
+
|
|
61
|
+
return (
|
|
62
|
+
<>
|
|
63
|
+
<Pressable
|
|
64
|
+
onPress={openModal}
|
|
65
|
+
disabled={disabled}
|
|
66
|
+
accessibilityLabel="Add attachment"
|
|
67
|
+
style={({ pressed }) => [
|
|
68
|
+
styles.button,
|
|
69
|
+
{
|
|
70
|
+
backgroundColor: theme.colors.assistantBubble,
|
|
71
|
+
opacity: pressed || disabled ? 0.45 : 1,
|
|
72
|
+
},
|
|
73
|
+
]}
|
|
74
|
+
>
|
|
75
|
+
{busy ? (
|
|
76
|
+
<ActivityIndicator color={theme.colors.assistantText} size="small" />
|
|
77
|
+
) : (
|
|
78
|
+
<AddIcon size={22} color={theme.colors.assistantText} />
|
|
79
|
+
)}
|
|
80
|
+
</Pressable>
|
|
81
|
+
<VoiceAddMediaModal
|
|
82
|
+
visible={modalVisible}
|
|
83
|
+
theme={theme}
|
|
84
|
+
onClose={closeModal}
|
|
85
|
+
onPickPhotos={() => void pickFromLibrary()}
|
|
86
|
+
onTakePhoto={() => void takePhoto()}
|
|
87
|
+
/>
|
|
88
|
+
</>
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const styles = StyleSheet.create({
|
|
93
|
+
button: {
|
|
94
|
+
width: 40,
|
|
95
|
+
height: 40,
|
|
96
|
+
borderRadius: 20,
|
|
97
|
+
alignItems: 'center',
|
|
98
|
+
justifyContent: 'center',
|
|
99
|
+
},
|
|
100
|
+
});
|
|
@@ -30,3 +30,7 @@ export function SpeakerIcon({
|
|
|
30
30
|
}: IconProps): React.ReactElement {
|
|
31
31
|
return <MaterialIcon name="volume-up" size={size} color={color} />;
|
|
32
32
|
}
|
|
33
|
+
|
|
34
|
+
export function AddIcon({ size = 24, color = '#4b5563' }: IconProps): React.ReactElement {
|
|
35
|
+
return <MaterialIcon name="add" size={size} color={color} />;
|
|
36
|
+
}
|
|
@@ -10,30 +10,40 @@ import {
|
|
|
10
10
|
const BAR_COUNT = 12;
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
|
-
* Polls
|
|
14
|
-
*
|
|
13
|
+
* Polls local player RMS (Berify JeannieVoiceWaveform pattern).
|
|
14
|
+
* Animates while assistant audio is audible; voice status extends hold at turn start.
|
|
15
15
|
*/
|
|
16
16
|
export function VoiceWaveform(): React.ReactElement | null {
|
|
17
17
|
const { theme, interactionMode, voiceStatus, getOutputAudioLevel } = useAgentAssistant();
|
|
18
18
|
const [levels, setLevels] = useState<number[]>(() => Array(BAR_COUNT).fill(0));
|
|
19
19
|
const lastAudibleAtRef = useRef(0);
|
|
20
|
+
const smoothedRef = useRef(0);
|
|
20
21
|
|
|
21
22
|
useEffect(() => {
|
|
22
23
|
if (interactionMode !== 'voice') {
|
|
23
24
|
setLevels(Array(BAR_COUNT).fill(0));
|
|
25
|
+
smoothedRef.current = 0;
|
|
24
26
|
return;
|
|
25
27
|
}
|
|
26
28
|
|
|
27
29
|
const id = setInterval(() => {
|
|
28
30
|
const now = Date.now();
|
|
29
|
-
const statusActive = WAVEFORM_ACTIVE_STATUSES.has(voiceStatus);
|
|
30
31
|
const rawLevel = getOutputAudioLevel();
|
|
31
32
|
if (rawLevel > WAVEFORM_AUDIBLE_THRESHOLD) {
|
|
32
33
|
lastAudibleAtRef.current = now;
|
|
33
34
|
}
|
|
35
|
+
if (WAVEFORM_ACTIVE_STATUSES.has(voiceStatus)) {
|
|
36
|
+
lastAudibleAtRef.current = now;
|
|
37
|
+
}
|
|
34
38
|
const audible = now - lastAudibleAtRef.current < WAVEFORM_AUDIBLE_HOLD_MS;
|
|
35
|
-
|
|
36
|
-
|
|
39
|
+
if (rawLevel > smoothedRef.current) {
|
|
40
|
+
smoothedRef.current = rawLevel;
|
|
41
|
+
} else {
|
|
42
|
+
smoothedRef.current *= 0.9;
|
|
43
|
+
}
|
|
44
|
+
const scaled = audible
|
|
45
|
+
? Math.min(1, Math.max(0.12, smoothedRef.current))
|
|
46
|
+
: 0;
|
|
37
47
|
setLevels((prev) => [...prev.slice(1), scaled]);
|
|
38
48
|
}, 100);
|
|
39
49
|
return () => clearInterval(id);
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { Message } from '@bytexbyte/nxtlinq-ai-agent-core-development';
|
|
2
|
+
import { useCallback, useEffect, useMemo, useRef, type RefObject } from 'react';
|
|
3
|
+
|
|
4
|
+
type ScrollableList = {
|
|
5
|
+
scrollToEnd: (opts: { animated: boolean }) => void;
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
function latestMessageScrollKey(messages: Message[]): string {
|
|
9
|
+
const last = messages[messages.length - 1];
|
|
10
|
+
if (!last) return '0';
|
|
11
|
+
return [
|
|
12
|
+
messages.length,
|
|
13
|
+
last.id,
|
|
14
|
+
last.content,
|
|
15
|
+
last.partialContent ?? '',
|
|
16
|
+
last.attachments?.length ?? 0,
|
|
17
|
+
last.isStreaming ? '1' : '0',
|
|
18
|
+
].join('|');
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function useMessageListAutoScroll(
|
|
22
|
+
messages: Message[],
|
|
23
|
+
listRef: RefObject<ScrollableList | null>,
|
|
24
|
+
): { onContentSizeChange: () => void } {
|
|
25
|
+
const scrollKey = useMemo(() => latestMessageScrollKey(messages), [messages]);
|
|
26
|
+
const pendingFrameRef = useRef<number | null>(null);
|
|
27
|
+
|
|
28
|
+
const scrollToLatest = useCallback(
|
|
29
|
+
(animated: boolean) => {
|
|
30
|
+
if (messages.length === 0) return;
|
|
31
|
+
if (pendingFrameRef.current != null) {
|
|
32
|
+
cancelAnimationFrame(pendingFrameRef.current);
|
|
33
|
+
}
|
|
34
|
+
pendingFrameRef.current = requestAnimationFrame(() => {
|
|
35
|
+
pendingFrameRef.current = null;
|
|
36
|
+
listRef.current?.scrollToEnd({ animated });
|
|
37
|
+
});
|
|
38
|
+
},
|
|
39
|
+
[listRef, messages.length],
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
useEffect(() => {
|
|
43
|
+
scrollToLatest(true);
|
|
44
|
+
return () => {
|
|
45
|
+
if (pendingFrameRef.current != null) {
|
|
46
|
+
cancelAnimationFrame(pendingFrameRef.current);
|
|
47
|
+
pendingFrameRef.current = null;
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
}, [scrollKey, scrollToLatest]);
|
|
51
|
+
|
|
52
|
+
const onContentSizeChange = useCallback(() => {
|
|
53
|
+
scrollToLatest(false);
|
|
54
|
+
}, [scrollToLatest]);
|
|
55
|
+
|
|
56
|
+
return { onContentSizeChange };
|
|
57
|
+
}
|
|
@@ -17,6 +17,8 @@ import React, {
|
|
|
17
17
|
import { defaultAgentAssistantTheme } from '../theme/defaultTheme';
|
|
18
18
|
import type { AgentAssistantTheme, NxtlinqAgentAssistantProps, PresetMessage } from '../types';
|
|
19
19
|
import { useVoiceMicState } from '../voice/useVoiceMicState';
|
|
20
|
+
import { useVoiceTranscriptMessages } from '../voice/useVoiceTranscriptMessages';
|
|
21
|
+
import { useWsRealtimeAudio } from '../voice/useWsRealtimeAudio';
|
|
20
22
|
import { waitForIOSVoiceCaptureRelease } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
|
|
21
23
|
import { isTextTtsPlayerSupported, TextTtsPlayer, type TextTtsPlayerHandle } from '../voice/TextTtsPlayer';
|
|
22
24
|
|
|
@@ -41,6 +43,8 @@ export type AgentAssistantContextValue = UseNxtlinqAgentResult &
|
|
|
41
43
|
selectPreset: (preset: PresetMessage) => Promise<void>;
|
|
42
44
|
/** True when voice data channel is open and ready for user_input. */
|
|
43
45
|
isVoiceChannelReady: boolean;
|
|
46
|
+
/** Show "+" image picker in voice bar (SDK-2). */
|
|
47
|
+
showVoiceImageInput: boolean;
|
|
44
48
|
postTextTts: (text: string) => Promise<{ audio: ArrayBuffer; mimeType: string }>;
|
|
45
49
|
buildTextTtsPlaybackUri: (result: { audio: ArrayBuffer; mimeType: string }) => string;
|
|
46
50
|
playMessageTts: (messageId: string, text: string) => Promise<void>;
|
|
@@ -62,10 +66,8 @@ export type AgentAssistantProviderProps = {
|
|
|
62
66
|
| 'startWithMicMuted'
|
|
63
67
|
| 'holdMicDuringAssistant'
|
|
64
68
|
| 'textTtsVolume'
|
|
65
|
-
| '
|
|
66
|
-
|
|
67
|
-
webrtcEnabled: boolean;
|
|
68
|
-
};
|
|
69
|
+
| 'showVoiceImageInput'
|
|
70
|
+
>;
|
|
69
71
|
};
|
|
70
72
|
|
|
71
73
|
export function AgentAssistantProvider({
|
|
@@ -98,34 +100,71 @@ export function AgentAssistantProvider({
|
|
|
98
100
|
const ttsRequestIdRef = React.useRef(0);
|
|
99
101
|
const voiceConnectChainRef = React.useRef<Promise<unknown>>(Promise.resolve());
|
|
100
102
|
|
|
101
|
-
const isVoiceAvailable =
|
|
103
|
+
const isVoiceAvailable = ui.enableVoice !== false;
|
|
102
104
|
const micStartsMuted = ui.startWithMicMuted ?? true;
|
|
103
105
|
|
|
106
|
+
const handleVoiceBargeIn = useCallback(() => {
|
|
107
|
+
voice.interrupt();
|
|
108
|
+
}, [voice]);
|
|
109
|
+
|
|
104
110
|
const {
|
|
105
111
|
isMicMuted,
|
|
112
|
+
isCaptureActive,
|
|
106
113
|
isMicHeldForAssistant,
|
|
107
114
|
toggleVoiceMicMute,
|
|
115
|
+
muteAfterSilenceCommit,
|
|
108
116
|
prepareForVoiceConnect,
|
|
109
117
|
resetMicState,
|
|
110
118
|
clearAssistantMicHold,
|
|
111
119
|
} = useVoiceMicState(voice, isVoiceConnecting, {
|
|
112
120
|
startWithMicMuted: micStartsMuted,
|
|
113
|
-
// Decoupled: open-mic demo can start unmuted yet still mute during assistant TTS.
|
|
114
121
|
holdMicDuringAssistant: ui.holdMicDuringAssistant ?? true,
|
|
122
|
+
onBargeIn: handleVoiceBargeIn,
|
|
115
123
|
});
|
|
116
124
|
|
|
125
|
+
const isVoiceActive = voice.voiceSessionId != null;
|
|
126
|
+
const {
|
|
127
|
+
buildCallbacks: buildWsAudioCallbacks,
|
|
128
|
+
bindSession: bindWsAudioSession,
|
|
129
|
+
getOutputAudioLevel: getWsOutputAudioLevel,
|
|
130
|
+
} = useWsRealtimeAudio(isCaptureActive, isVoiceActive, {
|
|
131
|
+
voiceStatus: voice.voiceStatus,
|
|
132
|
+
muteAfterSilenceCommit,
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
const getOutputAudioLevel = useCallback(
|
|
136
|
+
() => getWsOutputAudioLevel(),
|
|
137
|
+
[getWsOutputAudioLevel],
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
const voiceTranscriptApi = useMemo(
|
|
141
|
+
() => ({
|
|
142
|
+
getMessages: () => agent.agent.getSnapshot().messages,
|
|
143
|
+
setMessages: agent.setMessages,
|
|
144
|
+
syncVoiceTurnHistory: agent.syncVoiceTurnHistory,
|
|
145
|
+
}),
|
|
146
|
+
[agent.agent, agent.setMessages, agent.syncVoiceTurnHistory],
|
|
147
|
+
);
|
|
148
|
+
|
|
149
|
+
const { handleTranscript, handleDone, clearVoiceStream } = useVoiceTranscriptMessages(
|
|
150
|
+
voiceTranscriptApi,
|
|
151
|
+
interactionMode,
|
|
152
|
+
voice.voiceSessionId,
|
|
153
|
+
);
|
|
154
|
+
|
|
117
155
|
const setInteractionMode = useCallback(
|
|
118
156
|
(mode: InteractionMode) => {
|
|
119
157
|
if (mode === 'text' && interactionMode === 'voice') {
|
|
120
158
|
void (async () => {
|
|
121
159
|
await voice.stopVoice('switch_to_text');
|
|
122
160
|
await waitForIOSVoiceCaptureRelease();
|
|
161
|
+
clearVoiceStream();
|
|
123
162
|
resetMicState();
|
|
124
163
|
})();
|
|
125
164
|
}
|
|
126
165
|
setInteractionModeState(mode);
|
|
127
166
|
},
|
|
128
|
-
[interactionMode, voice, resetMicState],
|
|
167
|
+
[interactionMode, voice, resetMicState, clearVoiceStream],
|
|
129
168
|
);
|
|
130
169
|
|
|
131
170
|
useEffect(() => {
|
|
@@ -133,10 +172,11 @@ export function AgentAssistantProvider({
|
|
|
133
172
|
void (async () => {
|
|
134
173
|
await voice.stopVoice('mode_text_cleanup');
|
|
135
174
|
await waitForIOSVoiceCaptureRelease();
|
|
175
|
+
clearVoiceStream();
|
|
136
176
|
resetMicState();
|
|
137
177
|
})();
|
|
138
178
|
}
|
|
139
|
-
}, [interactionMode, voice.voiceSessionId, voice, resetMicState]);
|
|
179
|
+
}, [interactionMode, voice.voiceSessionId, voice, resetMicState, clearVoiceStream]);
|
|
140
180
|
|
|
141
181
|
const sendText = useCallback(async () => {
|
|
142
182
|
const text = inputText.trim();
|
|
@@ -169,32 +209,46 @@ export function AgentAssistantProvider({
|
|
|
169
209
|
try {
|
|
170
210
|
const session = await voice.startVoice({
|
|
171
211
|
startWithMicMuted: micStartsMuted,
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
212
|
+
keepMicCaptureActive: true,
|
|
213
|
+
...buildWsAudioCallbacks({
|
|
214
|
+
...options,
|
|
215
|
+
onOpen: () => {
|
|
216
|
+
if (!micStartsMuted) {
|
|
217
|
+
voice.muteMic(false);
|
|
218
|
+
}
|
|
219
|
+
options?.onOpen?.();
|
|
220
|
+
},
|
|
221
|
+
onClose: (reason) => {
|
|
222
|
+
clearVoiceStream();
|
|
223
|
+
resetMicState();
|
|
224
|
+
const userInitiated =
|
|
225
|
+
reason === 'switch_to_text' ||
|
|
226
|
+
reason === 'client_stop' ||
|
|
227
|
+
reason === 'mode_text_cleanup';
|
|
228
|
+
if (userInitiated) {
|
|
229
|
+
setInteractionModeState('text');
|
|
230
|
+
} else {
|
|
231
|
+
console.warn('[nxtlinq] voice session closed:', reason);
|
|
232
|
+
}
|
|
233
|
+
options?.onClose?.(reason);
|
|
234
|
+
},
|
|
235
|
+
onError: (err) => {
|
|
236
|
+
clearVoiceStream();
|
|
237
|
+
resetMicState();
|
|
238
|
+
console.warn('[nxtlinq] voice session error:', err.message);
|
|
239
|
+
options?.onError?.(err);
|
|
240
|
+
},
|
|
241
|
+
}),
|
|
242
|
+
onTranscript: (event) => {
|
|
243
|
+
handleTranscript(event);
|
|
244
|
+
options?.onTranscript?.(event);
|
|
178
245
|
},
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
reason === 'switch_to_text' ||
|
|
183
|
-
reason === 'client_stop' ||
|
|
184
|
-
reason === 'mode_text_cleanup';
|
|
185
|
-
if (userInitiated) {
|
|
186
|
-
setInteractionModeState('text');
|
|
187
|
-
} else {
|
|
188
|
-
console.warn('[nxtlinq] voice session closed:', reason);
|
|
189
|
-
}
|
|
190
|
-
options?.onClose?.(reason);
|
|
191
|
-
},
|
|
192
|
-
onError: (err) => {
|
|
193
|
-
resetMicState();
|
|
194
|
-
console.warn('[nxtlinq] voice session error:', err.message);
|
|
195
|
-
options?.onError?.(err);
|
|
246
|
+
onDone: (event) => {
|
|
247
|
+
handleDone(event);
|
|
248
|
+
options?.onDone?.(event);
|
|
196
249
|
},
|
|
197
250
|
});
|
|
251
|
+
bindWsAudioSession(session, !micStartsMuted);
|
|
198
252
|
return session;
|
|
199
253
|
} catch (err) {
|
|
200
254
|
setInteractionModeState('text');
|
|
@@ -208,7 +262,17 @@ export function AgentAssistantProvider({
|
|
|
208
262
|
voiceConnectChainRef.current = next.catch(() => undefined);
|
|
209
263
|
return next;
|
|
210
264
|
},
|
|
211
|
-
[
|
|
265
|
+
[
|
|
266
|
+
voice,
|
|
267
|
+
prepareForVoiceConnect,
|
|
268
|
+
resetMicState,
|
|
269
|
+
clearVoiceStream,
|
|
270
|
+
micStartsMuted,
|
|
271
|
+
buildWsAudioCallbacks,
|
|
272
|
+
bindWsAudioSession,
|
|
273
|
+
handleTranscript,
|
|
274
|
+
handleDone,
|
|
275
|
+
],
|
|
212
276
|
);
|
|
213
277
|
|
|
214
278
|
const wrappedStopVoice = useCallback(async () => {
|
|
@@ -273,6 +337,7 @@ export function AgentAssistantProvider({
|
|
|
273
337
|
() => ({
|
|
274
338
|
...agent,
|
|
275
339
|
...voice,
|
|
340
|
+
getOutputAudioLevel,
|
|
276
341
|
isVoiceActive:
|
|
277
342
|
interactionMode === 'voice' && voice.voiceSessionId != null,
|
|
278
343
|
startVoice: wrappedStartVoice,
|
|
@@ -292,6 +357,7 @@ export function AgentAssistantProvider({
|
|
|
292
357
|
isVoiceAvailable,
|
|
293
358
|
isVoiceConnecting,
|
|
294
359
|
isVoiceChannelReady,
|
|
360
|
+
showVoiceImageInput: ui.showVoiceImageInput ?? false,
|
|
295
361
|
postTextTts,
|
|
296
362
|
buildTextTtsPlaybackUri,
|
|
297
363
|
playMessageTts,
|
|
@@ -303,6 +369,7 @@ export function AgentAssistantProvider({
|
|
|
303
369
|
[
|
|
304
370
|
agent,
|
|
305
371
|
voice,
|
|
372
|
+
getOutputAudioLevel,
|
|
306
373
|
wrappedStartVoice,
|
|
307
374
|
wrappedStopVoice,
|
|
308
375
|
wrappedInterrupt,
|
|
@@ -318,6 +385,7 @@ export function AgentAssistantProvider({
|
|
|
318
385
|
isVoiceAvailable,
|
|
319
386
|
isVoiceConnecting,
|
|
320
387
|
isVoiceChannelReady,
|
|
388
|
+
ui.showVoiceImageInput,
|
|
321
389
|
postTextTts,
|
|
322
390
|
buildTextTtsPlaybackUri,
|
|
323
391
|
playMessageTts,
|
package/src/index.ts
CHANGED
|
@@ -21,11 +21,12 @@ export {
|
|
|
21
21
|
export { AgentMessageList } from './components/AgentMessageList';
|
|
22
22
|
export { AgentComposer } from './components/AgentComposer';
|
|
23
23
|
export { AgentVoiceBar } from './components/AgentVoiceBar';
|
|
24
|
-
export { AgentRemoteAudio } from './components/AgentRemoteAudio';
|
|
25
24
|
export { AudioSessionWaker } from './voice/AudioSessionWaker';
|
|
26
25
|
export { PresetMessageChips } from './components/PresetMessageChips';
|
|
27
26
|
export { AgentAssistantShell } from './components/AgentAssistantShell';
|
|
28
27
|
export { VoiceGreetTrigger } from './components/VoiceGreetTrigger';
|
|
28
|
+
export { VoiceAttachmentButton } from './components/VoiceAttachmentButton';
|
|
29
|
+
export { VoiceImageInput } from './components/VoiceImageInput';
|
|
29
30
|
|
|
30
31
|
// Headless SDK re-exports for apps that compose custom layouts
|
|
31
32
|
export {
|
|
@@ -34,7 +35,6 @@ export {
|
|
|
34
35
|
useNxtlinqVoice,
|
|
35
36
|
createNxtlinqAgentRN,
|
|
36
37
|
type NxtlinqAgentProviderProps,
|
|
37
|
-
type RNWebRTCModule,
|
|
38
38
|
} from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
|
|
39
39
|
|
|
40
40
|
export type {
|
package/src/react-native.d.ts
CHANGED
|
@@ -19,11 +19,27 @@ declare module 'react-native' {
|
|
|
19
19
|
}>;
|
|
20
20
|
export const TextInput: React.ComponentType<Record<string, unknown>>;
|
|
21
21
|
export const Pressable: React.ComponentType<{
|
|
22
|
-
onPress?: () => void;
|
|
22
|
+
onPress?: (event?: unknown) => void;
|
|
23
23
|
disabled?: boolean;
|
|
24
|
+
accessibilityLabel?: string;
|
|
24
25
|
style?: PressableStyle;
|
|
25
26
|
children?: React.ReactNode;
|
|
26
27
|
}>;
|
|
28
|
+
export const Image: React.ComponentType<{
|
|
29
|
+
source: { uri: string };
|
|
30
|
+
style?: StyleProp<ViewStyle>;
|
|
31
|
+
resizeMode?: 'cover' | 'contain' | 'stretch' | 'repeat' | 'center';
|
|
32
|
+
}>;
|
|
33
|
+
export const Modal: React.ComponentType<{
|
|
34
|
+
transparent?: boolean;
|
|
35
|
+
visible?: boolean;
|
|
36
|
+
animationType?: 'none' | 'slide' | 'fade';
|
|
37
|
+
onRequestClose?: () => void;
|
|
38
|
+
children?: React.ReactNode;
|
|
39
|
+
}>;
|
|
40
|
+
export const InteractionManager: {
|
|
41
|
+
runAfterInteractions: (task: () => void) => { cancel: () => void };
|
|
42
|
+
};
|
|
27
43
|
export const ScrollView: React.ComponentType<Record<string, unknown>>;
|
|
28
44
|
export const FlatList: <ItemT>(props: {
|
|
29
45
|
ref?: React.Ref<{ scrollToEnd: (opts: { animated: boolean }) => void }>;
|
|
@@ -31,6 +47,7 @@ declare module 'react-native' {
|
|
|
31
47
|
keyExtractor?: (item: ItemT, index: number) => string;
|
|
32
48
|
renderItem: (info: { item: ItemT; index: number }) => React.ReactElement | null;
|
|
33
49
|
contentContainerStyle?: StyleProp<ViewStyle>;
|
|
50
|
+
onContentSizeChange?: (width: number, height: number) => void;
|
|
34
51
|
ListEmptyComponent?: React.ReactElement | null;
|
|
35
52
|
ListFooterComponent?: React.ReactElement | null;
|
|
36
53
|
}) => React.ReactElement | null;
|
package/src/types.ts
CHANGED
|
@@ -48,11 +48,11 @@ export type NxtlinqAgentAssistantProps = Omit<NxtlinqAgentProviderProps, 'childr
|
|
|
48
48
|
title?: string;
|
|
49
49
|
placeholder?: string;
|
|
50
50
|
presetMessages?: PresetMessage[];
|
|
51
|
-
/** Fetch history on mount when `pseudoId
|
|
51
|
+
/** Fetch history on mount when identity is configured (`pseudoId`, and optionally `conversationId`). */
|
|
52
52
|
loadHistoryOnMount?: boolean;
|
|
53
53
|
/** Max messages to prefetch (default 50). */
|
|
54
54
|
historyLast?: number;
|
|
55
|
-
/** Show voice mode controls
|
|
55
|
+
/** Show voice mode controls. @default true */
|
|
56
56
|
enableVoice?: boolean;
|
|
57
57
|
/** Start in voice interaction mode. */
|
|
58
58
|
startInVoiceMode?: boolean;
|
|
@@ -76,14 +76,9 @@ export type NxtlinqAgentAssistantProps = Omit<NxtlinqAgentProviderProps, 'childr
|
|
|
76
76
|
voiceAutoGreet?: VoiceAutoGreetConfig | boolean;
|
|
77
77
|
/**
|
|
78
78
|
* iOS: bundled silent MP3 (`require('./assets/silent.mp3')`) to wake AVAudioSession.
|
|
79
|
-
* Strongly recommended — without it
|
|
79
|
+
* Strongly recommended — without it voice/TTS may be silent on cold start.
|
|
80
80
|
*/
|
|
81
81
|
iosSilentAudioSource?: number | { uri: string };
|
|
82
|
-
/**
|
|
83
|
-
* Voice-mode assistant playback gain (WebRTC `_setVolume`, 0–10). Default 5 to
|
|
84
|
-
* match text TTS loudness (react-native-video volume 1). Default 10.
|
|
85
|
-
*/
|
|
86
|
-
voiceRemoteAudioGain?: number;
|
|
87
82
|
/** Text-mode TTS volume (react-native-video, 0–1). @default 1 */
|
|
88
83
|
textTtsVolume?: number;
|
|
89
84
|
theme?: Partial<AgentAssistantTheme>;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export function float32ToPcm16(input: Float32Array): Int16Array {
|
|
2
|
+
const out = new Int16Array(input.length);
|
|
3
|
+
for (let i = 0; i < input.length; i += 1) {
|
|
4
|
+
const sample = Math.max(-1, Math.min(1, input[i]!));
|
|
5
|
+
out[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
|
|
6
|
+
}
|
|
7
|
+
return out;
|
|
8
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
type PickerResult = { path?: string };
|
|
2
|
+
type ImageCropPickerModule = {
|
|
3
|
+
openPicker: (options: Record<string, unknown>) => Promise<PickerResult>;
|
|
4
|
+
openCamera: (options: Record<string, unknown>) => Promise<PickerResult>;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
export function loadImageCropPicker(): ImageCropPickerModule | null {
|
|
8
|
+
try {
|
|
9
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
10
|
+
return require('react-native-image-crop-picker') as ImageCropPickerModule;
|
|
11
|
+
} catch {
|
|
12
|
+
return null;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function toFileUri(path: string): string {
|
|
17
|
+
return path.startsWith('file://') ? path : `file://${path}`;
|
|
18
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { Message } from '@bytexbyte/nxtlinq-ai-agent-core-development';
|
|
2
|
+
import { uriToVoiceImageAttachment } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
|
|
3
|
+
|
|
4
|
+
type SendVoiceImageAttachmentParams = {
|
|
5
|
+
fileUri: string;
|
|
6
|
+
messages: Message[];
|
|
7
|
+
voiceSessionId: string | null;
|
|
8
|
+
setMessages: (messages: Message[]) => void;
|
|
9
|
+
sendVoiceUserInput: (options: {
|
|
10
|
+
attachments: Awaited<ReturnType<typeof uriToVoiceImageAttachment>>[];
|
|
11
|
+
clientMessageId?: string;
|
|
12
|
+
}) => void;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export async function sendVoiceImageAttachment({
|
|
16
|
+
fileUri,
|
|
17
|
+
messages,
|
|
18
|
+
voiceSessionId,
|
|
19
|
+
setMessages,
|
|
20
|
+
sendVoiceUserInput,
|
|
21
|
+
}: SendVoiceImageAttachmentParams): Promise<void> {
|
|
22
|
+
const clientMessageId = `user_img_${Date.now()}`;
|
|
23
|
+
const previewAttachment = {
|
|
24
|
+
type: 'image' as const,
|
|
25
|
+
url: fileUri,
|
|
26
|
+
name: 'image.jpg',
|
|
27
|
+
mimeType: 'image/jpeg',
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
setMessages([
|
|
31
|
+
...messages,
|
|
32
|
+
{
|
|
33
|
+
id: clientMessageId,
|
|
34
|
+
role: 'user',
|
|
35
|
+
content: '',
|
|
36
|
+
timestamp: new Date().toISOString(),
|
|
37
|
+
attachments: [previewAttachment],
|
|
38
|
+
metadata: voiceSessionId ? { voiceRealtime: true, voiceSessionId } : undefined,
|
|
39
|
+
},
|
|
40
|
+
]);
|
|
41
|
+
|
|
42
|
+
try {
|
|
43
|
+
const attachment = await uriToVoiceImageAttachment(fileUri, 'voice-image.jpg');
|
|
44
|
+
sendVoiceUserInput({ attachments: [attachment], clientMessageId });
|
|
45
|
+
} catch (error) {
|
|
46
|
+
setMessages(messages.filter((message) => message.id !== clientMessageId));
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { useCallback, useState } from 'react';
|
|
2
|
+
import { InteractionManager } from 'react-native';
|
|
3
|
+
import { loadImageCropPicker, toFileUri } from './loadImageCropPicker';
|
|
4
|
+
import { VOICE_IMAGE_PICKER_OPTIONS } from './voiceImagePickerOptions';
|
|
5
|
+
|
|
6
|
+
type UseVoiceImagePickerResult = {
|
|
7
|
+
modalVisible: boolean;
|
|
8
|
+
openModal: () => void;
|
|
9
|
+
closeModal: () => void;
|
|
10
|
+
pickFromLibrary: () => Promise<void>;
|
|
11
|
+
takePhoto: () => Promise<void>;
|
|
12
|
+
pickerMissing: boolean;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export function useVoiceImagePicker(
|
|
16
|
+
onPicked?: (fileUri: string) => void,
|
|
17
|
+
): UseVoiceImagePickerResult {
|
|
18
|
+
const [modalVisible, setModalVisible] = useState(false);
|
|
19
|
+
const pickerMissing = loadImageCropPicker() == null;
|
|
20
|
+
|
|
21
|
+
const runPick = useCallback(
|
|
22
|
+
async (mode: 'library' | 'camera') => {
|
|
23
|
+
const picker = loadImageCropPicker();
|
|
24
|
+
if (!picker) return;
|
|
25
|
+
try {
|
|
26
|
+
const result =
|
|
27
|
+
mode === 'library'
|
|
28
|
+
? await picker.openPicker({ ...VOICE_IMAGE_PICKER_OPTIONS })
|
|
29
|
+
: await picker.openCamera({ ...VOICE_IMAGE_PICKER_OPTIONS });
|
|
30
|
+
if (!result?.path) {
|
|
31
|
+
setModalVisible(false);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
const uri = toFileUri(result.path);
|
|
35
|
+
setModalVisible(false);
|
|
36
|
+
InteractionManager.runAfterInteractions(() => {
|
|
37
|
+
onPicked?.(uri);
|
|
38
|
+
});
|
|
39
|
+
} catch {
|
|
40
|
+
setModalVisible(false);
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
[onPicked],
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
return {
|
|
47
|
+
modalVisible,
|
|
48
|
+
openModal: useCallback(() => setModalVisible(true), []),
|
|
49
|
+
closeModal: useCallback(() => setModalVisible(false), []),
|
|
50
|
+
pickFromLibrary: useCallback(() => runPick('library'), [runPick]),
|
|
51
|
+
takePhoto: useCallback(() => runPick('camera'), [runPick]),
|
|
52
|
+
pickerMissing,
|
|
53
|
+
};
|
|
54
|
+
}
|