@bytexbyte/nxtlinq-ai-agent-ui-react-native-development 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/NxtlinqAgentAssistant.d.ts +4 -4
  2. package/dist/NxtlinqAgentAssistant.d.ts.map +1 -1
  3. package/dist/NxtlinqAgentAssistant.js +5 -6
  4. package/dist/components/AgentAssistantShell.d.ts +1 -3
  5. package/dist/components/AgentAssistantShell.d.ts.map +1 -1
  6. package/dist/components/AgentAssistantShell.js +3 -7
  7. package/dist/components/AgentMessageList.d.ts.map +1 -1
  8. package/dist/components/AgentMessageList.js +7 -9
  9. package/dist/components/AgentVoiceBar.d.ts.map +1 -1
  10. package/dist/components/AgentVoiceBar.js +14 -34
  11. package/dist/components/MessageAttachmentPreview.d.ts +10 -0
  12. package/dist/components/MessageAttachmentPreview.d.ts.map +1 -0
  13. package/dist/components/MessageAttachmentPreview.js +15 -0
  14. package/dist/components/VoiceAddMediaModal.d.ts +12 -0
  15. package/dist/components/VoiceAddMediaModal.d.ts.map +1 -0
  16. package/dist/components/VoiceAddMediaModal.js +31 -0
  17. package/dist/components/VoiceAttachmentButton.d.ts +3 -0
  18. package/dist/components/VoiceAttachmentButton.d.ts.map +1 -0
  19. package/dist/components/VoiceAttachmentButton.js +58 -0
  20. package/dist/components/VoiceIcons.d.ts +1 -0
  21. package/dist/components/VoiceIcons.d.ts.map +1 -1
  22. package/dist/components/VoiceIcons.js +3 -0
  23. package/dist/components/VoiceWaveform.d.ts +2 -2
  24. package/dist/components/VoiceWaveform.d.ts.map +1 -1
  25. package/dist/components/VoiceWaveform.js +16 -5
  26. package/dist/components/useMessageListAutoScroll.d.ts +12 -0
  27. package/dist/components/useMessageListAutoScroll.d.ts.map +1 -0
  28. package/dist/components/useMessageListAutoScroll.js +42 -0
  29. package/dist/context/AgentAssistantContext.d.ts +3 -3
  30. package/dist/context/AgentAssistantContext.d.ts.map +1 -1
  31. package/dist/context/AgentAssistantContext.js +76 -29
  32. package/dist/index.d.ts +3 -2
  33. package/dist/index.d.ts.map +1 -1
  34. package/dist/index.js +2 -1
  35. package/dist/types.d.ts +3 -8
  36. package/dist/types.d.ts.map +1 -1
  37. package/dist/voice/float32ToPcm16.d.ts +2 -0
  38. package/dist/voice/float32ToPcm16.d.ts.map +1 -0
  39. package/dist/voice/float32ToPcm16.js +8 -0
  40. package/dist/voice/loadImageCropPicker.d.ts +11 -0
  41. package/dist/voice/loadImageCropPicker.d.ts.map +1 -0
  42. package/dist/voice/loadImageCropPicker.js +12 -0
  43. package/dist/voice/sendVoiceImageAttachment.d.ts +15 -0
  44. package/dist/voice/sendVoiceImageAttachment.d.ts.map +1 -0
  45. package/dist/voice/sendVoiceImageAttachment.js +29 -0
  46. package/dist/voice/useVoiceImagePicker.d.ts +11 -0
  47. package/dist/voice/useVoiceImagePicker.d.ts.map +1 -0
  48. package/dist/voice/useVoiceImagePicker.js +38 -0
  49. package/dist/voice/useVoiceMicState.d.ts +4 -0
  50. package/dist/voice/useVoiceMicState.d.ts.map +1 -1
  51. package/dist/voice/useVoiceMicState.js +32 -3
  52. package/dist/voice/useVoiceSilenceCommit.d.ts +10 -0
  53. package/dist/voice/useVoiceSilenceCommit.d.ts.map +1 -0
  54. package/dist/voice/useVoiceSilenceCommit.js +76 -0
  55. package/dist/voice/useVoiceTranscriptMessages.d.ts +16 -0
  56. package/dist/voice/useVoiceTranscriptMessages.d.ts.map +1 -0
  57. package/dist/voice/useVoiceTranscriptMessages.js +129 -0
  58. package/dist/voice/useWsRealtimeAudio.d.ts +17 -0
  59. package/dist/voice/useWsRealtimeAudio.d.ts.map +1 -0
  60. package/dist/voice/useWsRealtimeAudio.js +165 -0
  61. package/dist/voice/voiceImagePickerOptions.d.ts +11 -0
  62. package/dist/voice/voiceImagePickerOptions.d.ts.map +1 -0
  63. package/dist/voice/voiceImagePickerOptions.js +10 -0
  64. package/dist/voice/voiceSilenceConstants.d.ts +8 -0
  65. package/dist/voice/voiceSilenceConstants.d.ts.map +1 -0
  66. package/dist/voice/voiceSilenceConstants.js +7 -0
  67. package/dist/voice/wsPcmPlayer.d.ts +24 -0
  68. package/dist/voice/wsPcmPlayer.d.ts.map +1 -0
  69. package/dist/voice/wsPcmPlayer.js +146 -0
  70. package/dist/voice/wsPcmRecorder.d.ts +26 -0
  71. package/dist/voice/wsPcmRecorder.d.ts.map +1 -0
  72. package/dist/voice/wsPcmRecorder.js +145 -0
  73. package/dist/voice/wsRealtimeConstants.d.ts +2 -0
  74. package/dist/voice/wsRealtimeConstants.d.ts.map +1 -0
  75. package/dist/voice/wsRealtimeConstants.js +1 -0
  76. package/package.json +8 -5
  77. package/src/NxtlinqAgentAssistant.tsx +3 -12
  78. package/src/components/AgentAssistantShell.tsx +2 -18
  79. package/src/components/AgentMessageList.tsx +18 -15
  80. package/src/components/AgentVoiceBar.tsx +35 -70
  81. package/src/components/MessageAttachmentPreview.tsx +43 -0
  82. package/src/components/VoiceAddMediaModal.tsx +69 -0
  83. package/src/components/VoiceAttachmentButton.tsx +100 -0
  84. package/src/components/VoiceIcons.tsx +4 -0
  85. package/src/components/VoiceWaveform.tsx +15 -5
  86. package/src/components/useMessageListAutoScroll.ts +57 -0
  87. package/src/context/AgentAssistantContext.tsx +100 -32
  88. package/src/index.ts +2 -2
  89. package/src/react-native.d.ts +18 -1
  90. package/src/types.ts +3 -8
  91. package/src/voice/float32ToPcm16.ts +8 -0
  92. package/src/voice/loadImageCropPicker.ts +18 -0
  93. package/src/voice/sendVoiceImageAttachment.ts +49 -0
  94. package/src/voice/useVoiceImagePicker.ts +54 -0
  95. package/src/voice/useVoiceMicState.ts +38 -3
  96. package/src/voice/useVoiceSilenceCommit.ts +94 -0
  97. package/src/voice/useVoiceTranscriptMessages.ts +173 -0
  98. package/src/voice/useWsRealtimeAudio.ts +200 -0
  99. package/src/voice/voiceImagePickerOptions.ts +10 -0
  100. package/src/voice/voiceSilenceConstants.ts +10 -0
  101. package/src/voice/wsPcmPlayer.ts +166 -0
  102. package/src/voice/wsPcmRecorder.ts +152 -0
  103. package/src/voice/wsRealtimeConstants.ts +1 -0
  104. package/src/components/AgentRemoteAudio.tsx +0 -105
@@ -0,0 +1,100 @@
1
+ import React, { useCallback, useState } from 'react';
2
+ import { ActivityIndicator, Pressable, StyleSheet } from 'react-native';
3
+ import { useAgentAssistant } from '../context/AgentAssistantContext';
4
+ import { sendVoiceImageAttachment } from '../voice/sendVoiceImageAttachment';
5
+ import { useVoiceImagePicker } from '../voice/useVoiceImagePicker';
6
+ import { AddIcon } from './VoiceIcons';
7
+ import { VoiceAddMediaModal } from './VoiceAddMediaModal';
8
+
9
+ export function VoiceAttachmentButton(): React.ReactElement {
10
+ const {
11
+ theme,
12
+ messages,
13
+ setMessages,
14
+ sendVoiceUserInput,
15
+ voiceSessionId,
16
+ isVoiceChannelReady,
17
+ isVoiceConnecting,
18
+ } = useAgentAssistant();
19
+ const [busy, setBusy] = useState(false);
20
+
21
+ const handlePicked = useCallback(
22
+ async (fileUri: string) => {
23
+ if (!isVoiceChannelReady || busy) return;
24
+ setBusy(true);
25
+ try {
26
+ await sendVoiceImageAttachment({
27
+ fileUri,
28
+ messages,
29
+ voiceSessionId,
30
+ setMessages,
31
+ sendVoiceUserInput,
32
+ });
33
+ } catch (e) {
34
+ const message = e instanceof Error ? e.message : String(e);
35
+ console.warn('[nxtlinq] sendVoiceImageAttachment failed:', message);
36
+ } finally {
37
+ setBusy(false);
38
+ }
39
+ },
40
+ [
41
+ busy,
42
+ isVoiceChannelReady,
43
+ messages,
44
+ sendVoiceUserInput,
45
+ setMessages,
46
+ voiceSessionId,
47
+ ],
48
+ );
49
+
50
+ const {
51
+ modalVisible,
52
+ openModal,
53
+ closeModal,
54
+ pickFromLibrary,
55
+ takePhoto,
56
+ pickerMissing,
57
+ } = useVoiceImagePicker(handlePicked);
58
+
59
+ const disabled = busy || !isVoiceChannelReady || isVoiceConnecting || pickerMissing;
60
+
61
+ return (
62
+ <>
63
+ <Pressable
64
+ onPress={openModal}
65
+ disabled={disabled}
66
+ accessibilityLabel="Add attachment"
67
+ style={({ pressed }) => [
68
+ styles.button,
69
+ {
70
+ backgroundColor: theme.colors.assistantBubble,
71
+ opacity: pressed || disabled ? 0.45 : 1,
72
+ },
73
+ ]}
74
+ >
75
+ {busy ? (
76
+ <ActivityIndicator color={theme.colors.assistantText} size="small" />
77
+ ) : (
78
+ <AddIcon size={22} color={theme.colors.assistantText} />
79
+ )}
80
+ </Pressable>
81
+ <VoiceAddMediaModal
82
+ visible={modalVisible}
83
+ theme={theme}
84
+ onClose={closeModal}
85
+ onPickPhotos={() => void pickFromLibrary()}
86
+ onTakePhoto={() => void takePhoto()}
87
+ />
88
+ </>
89
+ );
90
+ }
91
+
92
+ const styles = StyleSheet.create({
93
+ button: {
94
+ width: 40,
95
+ height: 40,
96
+ borderRadius: 20,
97
+ alignItems: 'center',
98
+ justifyContent: 'center',
99
+ },
100
+ });
@@ -30,3 +30,7 @@ export function SpeakerIcon({
30
30
  }: IconProps): React.ReactElement {
31
31
  return <MaterialIcon name="volume-up" size={size} color={color} />;
32
32
  }
33
+
34
+ export function AddIcon({ size = 24, color = '#4b5563' }: IconProps): React.ReactElement {
35
+ return <MaterialIcon name="add" size={size} color={color} />;
36
+ }
@@ -10,30 +10,40 @@ import {
10
10
  const BAR_COUNT = 12;
11
11
 
12
12
  /**
13
- * Polls `getOutputAudioLevel()` while the assistant is speaking (Berify-aligned).
14
- * JeannieVoiceWaveform uses `active && audible` not during user mic / idle.
13
+ * Polls local player RMS (Berify JeannieVoiceWaveform pattern).
14
+ * Animates while assistant audio is audible; voice status extends hold at turn start.
15
15
  */
16
16
  export function VoiceWaveform(): React.ReactElement | null {
17
17
  const { theme, interactionMode, voiceStatus, getOutputAudioLevel } = useAgentAssistant();
18
18
  const [levels, setLevels] = useState<number[]>(() => Array(BAR_COUNT).fill(0));
19
19
  const lastAudibleAtRef = useRef(0);
20
+ const smoothedRef = useRef(0);
20
21
 
21
22
  useEffect(() => {
22
23
  if (interactionMode !== 'voice') {
23
24
  setLevels(Array(BAR_COUNT).fill(0));
25
+ smoothedRef.current = 0;
24
26
  return;
25
27
  }
26
28
 
27
29
  const id = setInterval(() => {
28
30
  const now = Date.now();
29
- const statusActive = WAVEFORM_ACTIVE_STATUSES.has(voiceStatus);
30
31
  const rawLevel = getOutputAudioLevel();
31
32
  if (rawLevel > WAVEFORM_AUDIBLE_THRESHOLD) {
32
33
  lastAudibleAtRef.current = now;
33
34
  }
35
+ if (WAVEFORM_ACTIVE_STATUSES.has(voiceStatus)) {
36
+ lastAudibleAtRef.current = now;
37
+ }
34
38
  const audible = now - lastAudibleAtRef.current < WAVEFORM_AUDIBLE_HOLD_MS;
35
- const effectiveActive = statusActive && audible;
36
- const scaled = effectiveActive ? rawLevel : 0;
39
+ if (rawLevel > smoothedRef.current) {
40
+ smoothedRef.current = rawLevel;
41
+ } else {
42
+ smoothedRef.current *= 0.9;
43
+ }
44
+ const scaled = audible
45
+ ? Math.min(1, Math.max(0.12, smoothedRef.current))
46
+ : 0;
37
47
  setLevels((prev) => [...prev.slice(1), scaled]);
38
48
  }, 100);
39
49
  return () => clearInterval(id);
@@ -0,0 +1,57 @@
1
+ import type { Message } from '@bytexbyte/nxtlinq-ai-agent-core-development';
2
+ import { useCallback, useEffect, useMemo, useRef, type RefObject } from 'react';
3
+
4
+ type ScrollableList = {
5
+ scrollToEnd: (opts: { animated: boolean }) => void;
6
+ };
7
+
8
+ function latestMessageScrollKey(messages: Message[]): string {
9
+ const last = messages[messages.length - 1];
10
+ if (!last) return '0';
11
+ return [
12
+ messages.length,
13
+ last.id,
14
+ last.content,
15
+ last.partialContent ?? '',
16
+ last.attachments?.length ?? 0,
17
+ last.isStreaming ? '1' : '0',
18
+ ].join('|');
19
+ }
20
+
21
+ export function useMessageListAutoScroll(
22
+ messages: Message[],
23
+ listRef: RefObject<ScrollableList | null>,
24
+ ): { onContentSizeChange: () => void } {
25
+ const scrollKey = useMemo(() => latestMessageScrollKey(messages), [messages]);
26
+ const pendingFrameRef = useRef<number | null>(null);
27
+
28
+ const scrollToLatest = useCallback(
29
+ (animated: boolean) => {
30
+ if (messages.length === 0) return;
31
+ if (pendingFrameRef.current != null) {
32
+ cancelAnimationFrame(pendingFrameRef.current);
33
+ }
34
+ pendingFrameRef.current = requestAnimationFrame(() => {
35
+ pendingFrameRef.current = null;
36
+ listRef.current?.scrollToEnd({ animated });
37
+ });
38
+ },
39
+ [listRef, messages.length],
40
+ );
41
+
42
+ useEffect(() => {
43
+ scrollToLatest(true);
44
+ return () => {
45
+ if (pendingFrameRef.current != null) {
46
+ cancelAnimationFrame(pendingFrameRef.current);
47
+ pendingFrameRef.current = null;
48
+ }
49
+ };
50
+ }, [scrollKey, scrollToLatest]);
51
+
52
+ const onContentSizeChange = useCallback(() => {
53
+ scrollToLatest(false);
54
+ }, [scrollToLatest]);
55
+
56
+ return { onContentSizeChange };
57
+ }
@@ -17,6 +17,8 @@ import React, {
17
17
  import { defaultAgentAssistantTheme } from '../theme/defaultTheme';
18
18
  import type { AgentAssistantTheme, NxtlinqAgentAssistantProps, PresetMessage } from '../types';
19
19
  import { useVoiceMicState } from '../voice/useVoiceMicState';
20
+ import { useVoiceTranscriptMessages } from '../voice/useVoiceTranscriptMessages';
21
+ import { useWsRealtimeAudio } from '../voice/useWsRealtimeAudio';
20
22
  import { waitForIOSVoiceCaptureRelease } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
21
23
  import { isTextTtsPlayerSupported, TextTtsPlayer, type TextTtsPlayerHandle } from '../voice/TextTtsPlayer';
22
24
 
@@ -41,6 +43,8 @@ export type AgentAssistantContextValue = UseNxtlinqAgentResult &
41
43
  selectPreset: (preset: PresetMessage) => Promise<void>;
42
44
  /** True when voice data channel is open and ready for user_input. */
43
45
  isVoiceChannelReady: boolean;
46
+ /** Show "+" image picker in voice bar (SDK-2). */
47
+ showVoiceImageInput: boolean;
44
48
  postTextTts: (text: string) => Promise<{ audio: ArrayBuffer; mimeType: string }>;
45
49
  buildTextTtsPlaybackUri: (result: { audio: ArrayBuffer; mimeType: string }) => string;
46
50
  playMessageTts: (messageId: string, text: string) => Promise<void>;
@@ -62,10 +66,8 @@ export type AgentAssistantProviderProps = {
62
66
  | 'startWithMicMuted'
63
67
  | 'holdMicDuringAssistant'
64
68
  | 'textTtsVolume'
65
- | 'voiceRemoteAudioGain'
66
- > & {
67
- webrtcEnabled: boolean;
68
- };
69
+ | 'showVoiceImageInput'
70
+ >;
69
71
  };
70
72
 
71
73
  export function AgentAssistantProvider({
@@ -98,34 +100,71 @@ export function AgentAssistantProvider({
98
100
  const ttsRequestIdRef = React.useRef(0);
99
101
  const voiceConnectChainRef = React.useRef<Promise<unknown>>(Promise.resolve());
100
102
 
101
- const isVoiceAvailable = Boolean(ui.webrtcEnabled && ui.enableVoice !== false);
103
+ const isVoiceAvailable = ui.enableVoice !== false;
102
104
  const micStartsMuted = ui.startWithMicMuted ?? true;
103
105
 
106
+ const handleVoiceBargeIn = useCallback(() => {
107
+ voice.interrupt();
108
+ }, [voice]);
109
+
104
110
  const {
105
111
  isMicMuted,
112
+ isCaptureActive,
106
113
  isMicHeldForAssistant,
107
114
  toggleVoiceMicMute,
115
+ muteAfterSilenceCommit,
108
116
  prepareForVoiceConnect,
109
117
  resetMicState,
110
118
  clearAssistantMicHold,
111
119
  } = useVoiceMicState(voice, isVoiceConnecting, {
112
120
  startWithMicMuted: micStartsMuted,
113
- // Decoupled: open-mic demo can start unmuted yet still mute during assistant TTS.
114
121
  holdMicDuringAssistant: ui.holdMicDuringAssistant ?? true,
122
+ onBargeIn: handleVoiceBargeIn,
115
123
  });
116
124
 
125
+ const isVoiceActive = voice.voiceSessionId != null;
126
+ const {
127
+ buildCallbacks: buildWsAudioCallbacks,
128
+ bindSession: bindWsAudioSession,
129
+ getOutputAudioLevel: getWsOutputAudioLevel,
130
+ } = useWsRealtimeAudio(isCaptureActive, isVoiceActive, {
131
+ voiceStatus: voice.voiceStatus,
132
+ muteAfterSilenceCommit,
133
+ });
134
+
135
+ const getOutputAudioLevel = useCallback(
136
+ () => getWsOutputAudioLevel(),
137
+ [getWsOutputAudioLevel],
138
+ );
139
+
140
+ const voiceTranscriptApi = useMemo(
141
+ () => ({
142
+ getMessages: () => agent.agent.getSnapshot().messages,
143
+ setMessages: agent.setMessages,
144
+ syncVoiceTurnHistory: agent.syncVoiceTurnHistory,
145
+ }),
146
+ [agent.agent, agent.setMessages, agent.syncVoiceTurnHistory],
147
+ );
148
+
149
+ const { handleTranscript, handleDone, clearVoiceStream } = useVoiceTranscriptMessages(
150
+ voiceTranscriptApi,
151
+ interactionMode,
152
+ voice.voiceSessionId,
153
+ );
154
+
117
155
  const setInteractionMode = useCallback(
118
156
  (mode: InteractionMode) => {
119
157
  if (mode === 'text' && interactionMode === 'voice') {
120
158
  void (async () => {
121
159
  await voice.stopVoice('switch_to_text');
122
160
  await waitForIOSVoiceCaptureRelease();
161
+ clearVoiceStream();
123
162
  resetMicState();
124
163
  })();
125
164
  }
126
165
  setInteractionModeState(mode);
127
166
  },
128
- [interactionMode, voice, resetMicState],
167
+ [interactionMode, voice, resetMicState, clearVoiceStream],
129
168
  );
130
169
 
131
170
  useEffect(() => {
@@ -133,10 +172,11 @@ export function AgentAssistantProvider({
133
172
  void (async () => {
134
173
  await voice.stopVoice('mode_text_cleanup');
135
174
  await waitForIOSVoiceCaptureRelease();
175
+ clearVoiceStream();
136
176
  resetMicState();
137
177
  })();
138
178
  }
139
- }, [interactionMode, voice.voiceSessionId, voice, resetMicState]);
179
+ }, [interactionMode, voice.voiceSessionId, voice, resetMicState, clearVoiceStream]);
140
180
 
141
181
  const sendText = useCallback(async () => {
142
182
  const text = inputText.trim();
@@ -169,32 +209,46 @@ export function AgentAssistantProvider({
169
209
  try {
170
210
  const session = await voice.startVoice({
171
211
  startWithMicMuted: micStartsMuted,
172
- ...options,
173
- onOpen: () => {
174
- if (!micStartsMuted) {
175
- voice.muteMic(false);
176
- }
177
- options?.onOpen?.();
212
+ keepMicCaptureActive: true,
213
+ ...buildWsAudioCallbacks({
214
+ ...options,
215
+ onOpen: () => {
216
+ if (!micStartsMuted) {
217
+ voice.muteMic(false);
218
+ }
219
+ options?.onOpen?.();
220
+ },
221
+ onClose: (reason) => {
222
+ clearVoiceStream();
223
+ resetMicState();
224
+ const userInitiated =
225
+ reason === 'switch_to_text' ||
226
+ reason === 'client_stop' ||
227
+ reason === 'mode_text_cleanup';
228
+ if (userInitiated) {
229
+ setInteractionModeState('text');
230
+ } else {
231
+ console.warn('[nxtlinq] voice session closed:', reason);
232
+ }
233
+ options?.onClose?.(reason);
234
+ },
235
+ onError: (err) => {
236
+ clearVoiceStream();
237
+ resetMicState();
238
+ console.warn('[nxtlinq] voice session error:', err.message);
239
+ options?.onError?.(err);
240
+ },
241
+ }),
242
+ onTranscript: (event) => {
243
+ handleTranscript(event);
244
+ options?.onTranscript?.(event);
178
245
  },
179
- onClose: (reason) => {
180
- resetMicState();
181
- const userInitiated =
182
- reason === 'switch_to_text' ||
183
- reason === 'client_stop' ||
184
- reason === 'mode_text_cleanup';
185
- if (userInitiated) {
186
- setInteractionModeState('text');
187
- } else {
188
- console.warn('[nxtlinq] voice session closed:', reason);
189
- }
190
- options?.onClose?.(reason);
191
- },
192
- onError: (err) => {
193
- resetMicState();
194
- console.warn('[nxtlinq] voice session error:', err.message);
195
- options?.onError?.(err);
246
+ onDone: (event) => {
247
+ handleDone(event);
248
+ options?.onDone?.(event);
196
249
  },
197
250
  });
251
+ bindWsAudioSession(session, !micStartsMuted);
198
252
  return session;
199
253
  } catch (err) {
200
254
  setInteractionModeState('text');
@@ -208,7 +262,17 @@ export function AgentAssistantProvider({
208
262
  voiceConnectChainRef.current = next.catch(() => undefined);
209
263
  return next;
210
264
  },
211
- [voice, prepareForVoiceConnect, resetMicState, micStartsMuted, ui.voiceRemoteAudioGain],
265
+ [
266
+ voice,
267
+ prepareForVoiceConnect,
268
+ resetMicState,
269
+ clearVoiceStream,
270
+ micStartsMuted,
271
+ buildWsAudioCallbacks,
272
+ bindWsAudioSession,
273
+ handleTranscript,
274
+ handleDone,
275
+ ],
212
276
  );
213
277
 
214
278
  const wrappedStopVoice = useCallback(async () => {
@@ -273,6 +337,7 @@ export function AgentAssistantProvider({
273
337
  () => ({
274
338
  ...agent,
275
339
  ...voice,
340
+ getOutputAudioLevel,
276
341
  isVoiceActive:
277
342
  interactionMode === 'voice' && voice.voiceSessionId != null,
278
343
  startVoice: wrappedStartVoice,
@@ -292,6 +357,7 @@ export function AgentAssistantProvider({
292
357
  isVoiceAvailable,
293
358
  isVoiceConnecting,
294
359
  isVoiceChannelReady,
360
+ showVoiceImageInput: ui.showVoiceImageInput ?? false,
295
361
  postTextTts,
296
362
  buildTextTtsPlaybackUri,
297
363
  playMessageTts,
@@ -303,6 +369,7 @@ export function AgentAssistantProvider({
303
369
  [
304
370
  agent,
305
371
  voice,
372
+ getOutputAudioLevel,
306
373
  wrappedStartVoice,
307
374
  wrappedStopVoice,
308
375
  wrappedInterrupt,
@@ -318,6 +385,7 @@ export function AgentAssistantProvider({
318
385
  isVoiceAvailable,
319
386
  isVoiceConnecting,
320
387
  isVoiceChannelReady,
388
+ ui.showVoiceImageInput,
321
389
  postTextTts,
322
390
  buildTextTtsPlaybackUri,
323
391
  playMessageTts,
package/src/index.ts CHANGED
@@ -21,11 +21,12 @@ export {
21
21
  export { AgentMessageList } from './components/AgentMessageList';
22
22
  export { AgentComposer } from './components/AgentComposer';
23
23
  export { AgentVoiceBar } from './components/AgentVoiceBar';
24
- export { AgentRemoteAudio } from './components/AgentRemoteAudio';
25
24
  export { AudioSessionWaker } from './voice/AudioSessionWaker';
26
25
  export { PresetMessageChips } from './components/PresetMessageChips';
27
26
  export { AgentAssistantShell } from './components/AgentAssistantShell';
28
27
  export { VoiceGreetTrigger } from './components/VoiceGreetTrigger';
28
+ export { VoiceAttachmentButton } from './components/VoiceAttachmentButton';
29
+ export { VoiceImageInput } from './components/VoiceImageInput';
29
30
 
30
31
  // Headless SDK re-exports for apps that compose custom layouts
31
32
  export {
@@ -34,7 +35,6 @@ export {
34
35
  useNxtlinqVoice,
35
36
  createNxtlinqAgentRN,
36
37
  type NxtlinqAgentProviderProps,
37
- type RNWebRTCModule,
38
38
  } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
39
39
 
40
40
  export type {
@@ -19,11 +19,27 @@ declare module 'react-native' {
19
19
  }>;
20
20
  export const TextInput: React.ComponentType<Record<string, unknown>>;
21
21
  export const Pressable: React.ComponentType<{
22
- onPress?: () => void;
22
+ onPress?: (event?: unknown) => void;
23
23
  disabled?: boolean;
24
+ accessibilityLabel?: string;
24
25
  style?: PressableStyle;
25
26
  children?: React.ReactNode;
26
27
  }>;
28
+ export const Image: React.ComponentType<{
29
+ source: { uri: string };
30
+ style?: StyleProp<ViewStyle>;
31
+ resizeMode?: 'cover' | 'contain' | 'stretch' | 'repeat' | 'center';
32
+ }>;
33
+ export const Modal: React.ComponentType<{
34
+ transparent?: boolean;
35
+ visible?: boolean;
36
+ animationType?: 'none' | 'slide' | 'fade';
37
+ onRequestClose?: () => void;
38
+ children?: React.ReactNode;
39
+ }>;
40
+ export const InteractionManager: {
41
+ runAfterInteractions: (task: () => void) => { cancel: () => void };
42
+ };
27
43
  export const ScrollView: React.ComponentType<Record<string, unknown>>;
28
44
  export const FlatList: <ItemT>(props: {
29
45
  ref?: React.Ref<{ scrollToEnd: (opts: { animated: boolean }) => void }>;
@@ -31,6 +47,7 @@ declare module 'react-native' {
31
47
  keyExtractor?: (item: ItemT, index: number) => string;
32
48
  renderItem: (info: { item: ItemT; index: number }) => React.ReactElement | null;
33
49
  contentContainerStyle?: StyleProp<ViewStyle>;
50
+ onContentSizeChange?: (width: number, height: number) => void;
34
51
  ListEmptyComponent?: React.ReactElement | null;
35
52
  ListFooterComponent?: React.ReactElement | null;
36
53
  }) => React.ReactElement | null;
package/src/types.ts CHANGED
@@ -48,11 +48,11 @@ export type NxtlinqAgentAssistantProps = Omit<NxtlinqAgentProviderProps, 'childr
48
48
  title?: string;
49
49
  placeholder?: string;
50
50
  presetMessages?: PresetMessage[];
51
- /** Fetch history on mount when `pseudoId` is set. */
51
+ /** Fetch history on mount when identity is configured (`pseudoId`, and optionally `conversationId`). */
52
52
  loadHistoryOnMount?: boolean;
53
53
  /** Max messages to prefetch (default 50). */
54
54
  historyLast?: number;
55
- /** Show voice mode controls (requires `webrtcModule`). */
55
+ /** Show voice mode controls. @default true */
56
56
  enableVoice?: boolean;
57
57
  /** Start in voice interaction mode. */
58
58
  startInVoiceMode?: boolean;
@@ -76,14 +76,9 @@ export type NxtlinqAgentAssistantProps = Omit<NxtlinqAgentProviderProps, 'childr
76
76
  voiceAutoGreet?: VoiceAutoGreetConfig | boolean;
77
77
  /**
78
78
  * iOS: bundled silent MP3 (`require('./assets/silent.mp3')`) to wake AVAudioSession.
79
- * Strongly recommended — without it WebRTC/TTS may be silent on cold start.
79
+ * Strongly recommended — without it voice/TTS may be silent on cold start.
80
80
  */
81
81
  iosSilentAudioSource?: number | { uri: string };
82
- /**
83
- * Voice-mode assistant playback gain (WebRTC `_setVolume`, 0–10). Default 5 to
84
- * match text TTS loudness (react-native-video volume 1). Default 10.
85
- */
86
- voiceRemoteAudioGain?: number;
87
82
  /** Text-mode TTS volume (react-native-video, 0–1). @default 1 */
88
83
  textTtsVolume?: number;
89
84
  theme?: Partial<AgentAssistantTheme>;
@@ -0,0 +1,8 @@
1
+ export function float32ToPcm16(input: Float32Array): Int16Array {
2
+ const out = new Int16Array(input.length);
3
+ for (let i = 0; i < input.length; i += 1) {
4
+ const sample = Math.max(-1, Math.min(1, input[i]!));
5
+ out[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
6
+ }
7
+ return out;
8
+ }
@@ -0,0 +1,18 @@
1
+ type PickerResult = { path?: string };
2
+ type ImageCropPickerModule = {
3
+ openPicker: (options: Record<string, unknown>) => Promise<PickerResult>;
4
+ openCamera: (options: Record<string, unknown>) => Promise<PickerResult>;
5
+ };
6
+
7
+ export function loadImageCropPicker(): ImageCropPickerModule | null {
8
+ try {
9
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
10
+ return require('react-native-image-crop-picker') as ImageCropPickerModule;
11
+ } catch {
12
+ return null;
13
+ }
14
+ }
15
+
16
+ export function toFileUri(path: string): string {
17
+ return path.startsWith('file://') ? path : `file://${path}`;
18
+ }
@@ -0,0 +1,49 @@
1
+ import type { Message } from '@bytexbyte/nxtlinq-ai-agent-core-development';
2
+ import { uriToVoiceImageAttachment } from '@bytexbyte/nxtlinq-ai-agent-react-native-development';
3
+
4
+ type SendVoiceImageAttachmentParams = {
5
+ fileUri: string;
6
+ messages: Message[];
7
+ voiceSessionId: string | null;
8
+ setMessages: (messages: Message[]) => void;
9
+ sendVoiceUserInput: (options: {
10
+ attachments: Awaited<ReturnType<typeof uriToVoiceImageAttachment>>[];
11
+ clientMessageId?: string;
12
+ }) => void;
13
+ };
14
+
15
+ export async function sendVoiceImageAttachment({
16
+ fileUri,
17
+ messages,
18
+ voiceSessionId,
19
+ setMessages,
20
+ sendVoiceUserInput,
21
+ }: SendVoiceImageAttachmentParams): Promise<void> {
22
+ const clientMessageId = `user_img_${Date.now()}`;
23
+ const previewAttachment = {
24
+ type: 'image' as const,
25
+ url: fileUri,
26
+ name: 'image.jpg',
27
+ mimeType: 'image/jpeg',
28
+ };
29
+
30
+ setMessages([
31
+ ...messages,
32
+ {
33
+ id: clientMessageId,
34
+ role: 'user',
35
+ content: '',
36
+ timestamp: new Date().toISOString(),
37
+ attachments: [previewAttachment],
38
+ metadata: voiceSessionId ? { voiceRealtime: true, voiceSessionId } : undefined,
39
+ },
40
+ ]);
41
+
42
+ try {
43
+ const attachment = await uriToVoiceImageAttachment(fileUri, 'voice-image.jpg');
44
+ sendVoiceUserInput({ attachments: [attachment], clientMessageId });
45
+ } catch (error) {
46
+ setMessages(messages.filter((message) => message.id !== clientMessageId));
47
+ throw error;
48
+ }
49
+ }
@@ -0,0 +1,54 @@
1
+ import { useCallback, useState } from 'react';
2
+ import { InteractionManager } from 'react-native';
3
+ import { loadImageCropPicker, toFileUri } from './loadImageCropPicker';
4
+ import { VOICE_IMAGE_PICKER_OPTIONS } from './voiceImagePickerOptions';
5
+
6
+ type UseVoiceImagePickerResult = {
7
+ modalVisible: boolean;
8
+ openModal: () => void;
9
+ closeModal: () => void;
10
+ pickFromLibrary: () => Promise<void>;
11
+ takePhoto: () => Promise<void>;
12
+ pickerMissing: boolean;
13
+ };
14
+
15
+ export function useVoiceImagePicker(
16
+ onPicked?: (fileUri: string) => void,
17
+ ): UseVoiceImagePickerResult {
18
+ const [modalVisible, setModalVisible] = useState(false);
19
+ const pickerMissing = loadImageCropPicker() == null;
20
+
21
+ const runPick = useCallback(
22
+ async (mode: 'library' | 'camera') => {
23
+ const picker = loadImageCropPicker();
24
+ if (!picker) return;
25
+ try {
26
+ const result =
27
+ mode === 'library'
28
+ ? await picker.openPicker({ ...VOICE_IMAGE_PICKER_OPTIONS })
29
+ : await picker.openCamera({ ...VOICE_IMAGE_PICKER_OPTIONS });
30
+ if (!result?.path) {
31
+ setModalVisible(false);
32
+ return;
33
+ }
34
+ const uri = toFileUri(result.path);
35
+ setModalVisible(false);
36
+ InteractionManager.runAfterInteractions(() => {
37
+ onPicked?.(uri);
38
+ });
39
+ } catch {
40
+ setModalVisible(false);
41
+ }
42
+ },
43
+ [onPicked],
44
+ );
45
+
46
+ return {
47
+ modalVisible,
48
+ openModal: useCallback(() => setModalVisible(true), []),
49
+ closeModal: useCallback(() => setModalVisible(false), []),
50
+ pickFromLibrary: useCallback(() => runPick('library'), [runPick]),
51
+ takePhoto: useCallback(() => runPick('camera'), [runPick]),
52
+ pickerMissing,
53
+ };
54
+ }