@bytexbyte/nxtlinq-ai-agent-web-development 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/dist/context/NxtlinqAgentContext.d.ts +12 -0
  2. package/dist/context/NxtlinqAgentContext.d.ts.map +1 -0
  3. package/dist/context/NxtlinqAgentContext.js +33 -0
  4. package/dist/createNxtlinqAgent.d.ts +9 -0
  5. package/dist/createNxtlinqAgent.d.ts.map +1 -0
  6. package/dist/createNxtlinqAgent.js +19 -0
  7. package/dist/hooks/useNxtlinqAgent.d.ts +18 -0
  8. package/dist/hooks/useNxtlinqAgent.d.ts.map +1 -0
  9. package/dist/hooks/useNxtlinqAgent.js +23 -0
  10. package/dist/hooks/useNxtlinqVoice.d.ts +21 -0
  11. package/dist/hooks/useNxtlinqVoice.d.ts.map +1 -0
  12. package/dist/hooks/useNxtlinqVoice.js +75 -0
  13. package/dist/index.d.ts +12 -0
  14. package/dist/index.d.ts.map +1 -0
  15. package/dist/index.js +9 -0
  16. package/dist/legacy/api/nxtlinq-api.d.ts +8 -0
  17. package/dist/legacy/api/nxtlinq-api.d.ts.map +1 -0
  18. package/dist/legacy/api/nxtlinq-api.js +13 -0
  19. package/dist/legacy/api/voice.d.ts +11 -0
  20. package/dist/legacy/api/voice.d.ts.map +1 -0
  21. package/dist/legacy/api/voice.js +26 -0
  22. package/dist/legacy/core/lib/messageHistory.d.ts +2 -0
  23. package/dist/legacy/core/lib/messageHistory.d.ts.map +1 -0
  24. package/dist/legacy/core/lib/messageHistory.js +1 -0
  25. package/dist/legacy/core/lib/textToSpeech.d.ts +14 -0
  26. package/dist/legacy/core/lib/textToSpeech.d.ts.map +1 -0
  27. package/dist/legacy/core/lib/textToSpeech.js +82 -0
  28. package/dist/legacy/core/lib/useDraggable.d.ts +15 -0
  29. package/dist/legacy/core/lib/useDraggable.d.ts.map +1 -0
  30. package/dist/legacy/core/lib/useDraggable.js +158 -0
  31. package/dist/legacy/core/lib/useLocalStorage.d.ts +11 -0
  32. package/dist/legacy/core/lib/useLocalStorage.d.ts.map +1 -0
  33. package/dist/legacy/core/lib/useLocalStorage.js +83 -0
  34. package/dist/legacy/core/lib/useResizable.d.ts +17 -0
  35. package/dist/legacy/core/lib/useResizable.d.ts.map +1 -0
  36. package/dist/legacy/core/lib/useResizable.js +203 -0
  37. package/dist/legacy/core/lib/useSessionStorage.d.ts +11 -0
  38. package/dist/legacy/core/lib/useSessionStorage.d.ts.map +1 -0
  39. package/dist/legacy/core/lib/useSessionStorage.js +37 -0
  40. package/dist/legacy/core/lib/useSpeechToTextFromMic/helper.d.ts +26 -0
  41. package/dist/legacy/core/lib/useSpeechToTextFromMic/helper.d.ts.map +1 -0
  42. package/dist/legacy/core/lib/useSpeechToTextFromMic/helper.js +102 -0
  43. package/dist/legacy/core/lib/useSpeechToTextFromMic/index.d.ts +16 -0
  44. package/dist/legacy/core/lib/useSpeechToTextFromMic/index.d.ts.map +1 -0
  45. package/dist/legacy/core/lib/useSpeechToTextFromMic/index.js +92 -0
  46. package/dist/legacy/core/lib/useVoiceMode.d.ts +32 -0
  47. package/dist/legacy/core/lib/useVoiceMode.d.ts.map +1 -0
  48. package/dist/legacy/core/lib/useVoiceMode.js +373 -0
  49. package/dist/legacy/core/metakeepClient.d.ts +4 -0
  50. package/dist/legacy/core/metakeepClient.d.ts.map +1 -0
  51. package/dist/legacy/core/metakeepClient.js +10 -0
  52. package/dist/legacy/core/utils/aitUtils.d.ts +31 -0
  53. package/dist/legacy/core/utils/aitUtils.d.ts.map +1 -0
  54. package/dist/legacy/core/utils/aitUtils.js +35 -0
  55. package/dist/legacy/core/utils/ethersUtils.d.ts +8 -0
  56. package/dist/legacy/core/utils/ethersUtils.d.ts.map +1 -0
  57. package/dist/legacy/core/utils/ethersUtils.js +19 -0
  58. package/dist/legacy/core/utils/index.d.ts +3 -0
  59. package/dist/legacy/core/utils/index.d.ts.map +1 -0
  60. package/dist/legacy/core/utils/index.js +4 -0
  61. package/dist/legacy/core/utils/notificationUtils.d.ts +29 -0
  62. package/dist/legacy/core/utils/notificationUtils.d.ts.map +1 -0
  63. package/dist/legacy/core/utils/notificationUtils.js +47 -0
  64. package/dist/legacy/core/utils/urlUtils.d.ts +25 -0
  65. package/dist/legacy/core/utils/urlUtils.d.ts.map +1 -0
  66. package/dist/legacy/core/utils/urlUtils.js +135 -0
  67. package/dist/legacy/core/utils/walletTextUtils.d.ts +14 -0
  68. package/dist/legacy/core/utils/walletTextUtils.d.ts.map +1 -0
  69. package/dist/legacy/core/utils/walletTextUtils.js +23 -0
  70. package/dist/legacy/core/utils/walletUtils.d.ts +10 -0
  71. package/dist/legacy/core/utils/walletUtils.d.ts.map +1 -0
  72. package/dist/legacy/core/utils/walletUtils.js +38 -0
  73. package/dist/legacy/index.d.ts +19 -0
  74. package/dist/legacy/index.d.ts.map +1 -0
  75. package/dist/legacy/index.js +16 -0
  76. package/dist/ports/createWebPlatformPorts.d.ts +13 -0
  77. package/dist/ports/createWebPlatformPorts.d.ts.map +1 -0
  78. package/dist/ports/createWebPlatformPorts.js +25 -0
  79. package/dist/utils/fileToAttachment.d.ts +4 -0
  80. package/dist/utils/fileToAttachment.d.ts.map +1 -0
  81. package/dist/utils/fileToAttachment.js +28 -0
  82. package/dist/voice/useVoiceSilenceCommit.d.ts +11 -0
  83. package/dist/voice/useVoiceSilenceCommit.d.ts.map +1 -0
  84. package/dist/voice/useVoiceSilenceCommit.js +68 -0
  85. package/dist/voice/useVoiceTranscriptMessages.d.ts +16 -0
  86. package/dist/voice/useVoiceTranscriptMessages.d.ts.map +1 -0
  87. package/dist/voice/useVoiceTranscriptMessages.js +134 -0
  88. package/dist/voice/useWsRealtimeAudio.d.ts +18 -0
  89. package/dist/voice/useWsRealtimeAudio.d.ts.map +1 -0
  90. package/dist/voice/useWsRealtimeAudio.js +115 -0
  91. package/dist/voice/voiceMicConstants.d.ts +4 -0
  92. package/dist/voice/voiceMicConstants.d.ts.map +1 -0
  93. package/dist/voice/voiceMicConstants.js +10 -0
  94. package/dist/voice/ws/BrowserWsPcmPlayer.d.ts +23 -0
  95. package/dist/voice/ws/BrowserWsPcmPlayer.d.ts.map +1 -0
  96. package/dist/voice/ws/BrowserWsPcmPlayer.js +138 -0
  97. package/dist/voice/ws/BrowserWsPcmRecorder.d.ts +19 -0
  98. package/dist/voice/ws/BrowserWsPcmRecorder.d.ts.map +1 -0
  99. package/dist/voice/ws/BrowserWsPcmRecorder.js +76 -0
  100. package/dist/voice/ws/float32ToPcm16.d.ts +2 -0
  101. package/dist/voice/ws/float32ToPcm16.d.ts.map +1 -0
  102. package/dist/voice/ws/float32ToPcm16.js +8 -0
  103. package/dist/voice/ws/voiceSilenceConstants.d.ts +5 -0
  104. package/dist/voice/ws/voiceSilenceConstants.d.ts.map +1 -0
  105. package/dist/voice/ws/voiceSilenceConstants.js +4 -0
  106. package/dist/voice/ws/wsRealtimeConstants.d.ts +2 -0
  107. package/dist/voice/ws/wsRealtimeConstants.d.ts.map +1 -0
  108. package/dist/voice/ws/wsRealtimeConstants.js +1 -0
  109. package/dist/webAgentDefaults.d.ts +9 -0
  110. package/dist/webAgentDefaults.d.ts.map +1 -0
  111. package/dist/webAgentDefaults.js +9 -0
  112. package/package.json +55 -0
  113. package/src/context/NxtlinqAgentContext.tsx +79 -0
  114. package/src/createNxtlinqAgent.ts +36 -0
  115. package/src/hooks/useNxtlinqAgent.ts +73 -0
  116. package/src/hooks/useNxtlinqVoice.ts +143 -0
  117. package/src/index.ts +84 -0
  118. package/src/legacy/api/nxtlinq-api.ts +32 -0
  119. package/src/legacy/api/voice.ts +72 -0
  120. package/src/legacy/core/lib/messageHistory.ts +6 -0
  121. package/src/legacy/core/lib/textToSpeech.ts +127 -0
  122. package/src/legacy/core/lib/useDraggable.ts +193 -0
  123. package/src/legacy/core/lib/useLocalStorage.ts +89 -0
  124. package/src/legacy/core/lib/useResizable.ts +256 -0
  125. package/src/legacy/core/lib/useSessionStorage.ts +43 -0
  126. package/src/legacy/core/lib/useSpeechToTextFromMic/helper.ts +132 -0
  127. package/src/legacy/core/lib/useSpeechToTextFromMic/index.ts +126 -0
  128. package/src/legacy/core/lib/useVoiceMode.ts +407 -0
  129. package/src/legacy/core/metakeepClient.ts +12 -0
  130. package/src/legacy/core/utils/aitUtils.ts +55 -0
  131. package/src/legacy/core/utils/ethersUtils.ts +24 -0
  132. package/src/legacy/core/utils/index.ts +5 -0
  133. package/src/legacy/core/utils/notificationUtils.ts +64 -0
  134. package/src/legacy/core/utils/urlUtils.ts +160 -0
  135. package/src/legacy/core/utils/walletTextUtils.ts +26 -0
  136. package/src/legacy/core/utils/walletUtils.ts +53 -0
  137. package/src/legacy/index.ts +35 -0
  138. package/src/ports/createWebPlatformPorts.ts +44 -0
  139. package/src/utils/fileToAttachment.ts +32 -0
  140. package/src/voice/useVoiceSilenceCommit.ts +84 -0
  141. package/src/voice/useVoiceTranscriptMessages.ts +184 -0
  142. package/src/voice/useWsRealtimeAudio.ts +141 -0
  143. package/src/voice/voiceMicConstants.ts +13 -0
  144. package/src/voice/ws/BrowserWsPcmPlayer.ts +139 -0
  145. package/src/voice/ws/BrowserWsPcmRecorder.ts +83 -0
  146. package/src/voice/ws/float32ToPcm16.ts +8 -0
  147. package/src/voice/ws/voiceSilenceConstants.ts +4 -0
  148. package/src/voice/ws/wsRealtimeConstants.ts +1 -0
  149. package/src/webAgentDefaults.ts +12 -0
@@ -0,0 +1,115 @@
1
+ import { useCallback, useEffect, useRef } from 'react';
2
+ import { BrowserWsPcmPlayer } from './ws/BrowserWsPcmPlayer';
3
+ import { BrowserWsPcmRecorder } from './ws/BrowserWsPcmRecorder';
4
+ import { useVoiceSilenceCommit } from './useVoiceSilenceCommit';
5
+ export function useWsRealtimeAudio(isCaptureActive, isVoiceActive, options) {
6
+ const playerRef = useRef(null);
7
+ const recorderRef = useRef(null);
8
+ const sessionRef = useRef(null);
9
+ const captureGenRef = useRef(0);
10
+ const isCaptureActiveRef = useRef(isCaptureActive);
11
+ isCaptureActiveRef.current = isCaptureActive;
12
+ const prevCaptureActiveRef = useRef(isCaptureActive);
13
+ const getSession = useCallback(() => sessionRef.current, []);
14
+ const muteAfterSilenceCommitRef = useRef(options.muteAfterSilenceCommit);
15
+ muteAfterSilenceCommitRef.current = options.muteAfterSilenceCommit;
16
+ const silence = useVoiceSilenceCommit(getSession, () => muteAfterSilenceCommitRef.current(), options.voiceStatus);
17
+ const silenceRef = useRef(silence);
18
+ silenceRef.current = silence;
19
+ const ensurePlayer = useCallback(async () => {
20
+ if (!playerRef.current) {
21
+ playerRef.current = new BrowserWsPcmPlayer();
22
+ playerRef.current.prewarm();
23
+ }
24
+ await playerRef.current.ensureRunning();
25
+ }, []);
26
+ const stopCapture = useCallback((commit) => {
27
+ captureGenRef.current += 1;
28
+ const s = silenceRef.current;
29
+ s.clearPoll();
30
+ // Commit before recorder.stop() — stop() must not clear the OpenAI input buffer first.
31
+ if (commit && !s.consumeSkipCommitOnMute()) {
32
+ s.tryCommit('manual');
33
+ }
34
+ recorderRef.current?.stop();
35
+ }, []);
36
+ const startCapture = useCallback(async () => {
37
+ const session = sessionRef.current;
38
+ if (!session)
39
+ return;
40
+ const gen = ++captureGenRef.current;
41
+ const s = silenceRef.current;
42
+ if (!recorderRef.current)
43
+ recorderRef.current = new BrowserWsPcmRecorder();
44
+ playerRef.current?.clearQueue();
45
+ recorderRef.current.bindSession(session);
46
+ recorderRef.current.setOnRms(s.onSpeechRms);
47
+ recorderRef.current.setShouldAppend(() => s.hasHadSpeech());
48
+ // Let mic-unmute state settle before getUserMedia (Berify beginMicCapture).
49
+ await new Promise((resolve) => {
50
+ requestAnimationFrame(() => requestAnimationFrame(() => resolve()));
51
+ });
52
+ if (gen !== captureGenRef.current || !isCaptureActiveRef.current)
53
+ return;
54
+ s.resetTurn();
55
+ try {
56
+ await recorderRef.current.start();
57
+ if (gen !== captureGenRef.current || !isCaptureActiveRef.current) {
58
+ recorderRef.current.stop();
59
+ return;
60
+ }
61
+ s.startPoll();
62
+ }
63
+ catch (err) {
64
+ s.clearPoll();
65
+ console.error('[nxtlinq] mic capture start failed', err);
66
+ }
67
+ }, []);
68
+ const cleanup = useCallback(() => {
69
+ silenceRef.current.clearPoll();
70
+ recorderRef.current?.cleanup();
71
+ recorderRef.current = null;
72
+ playerRef.current?.cleanup();
73
+ playerRef.current = null;
74
+ sessionRef.current = null;
75
+ }, []);
76
+ const bindSession = useCallback((session, captureWhenUnmuted = false) => {
77
+ sessionRef.current = session;
78
+ recorderRef.current?.bindSession(session);
79
+ if (session && captureWhenUnmuted && isCaptureActiveRef.current) {
80
+ void startCapture();
81
+ }
82
+ }, [startCapture]);
83
+ const buildCallbacks = useCallback((overrides) => ({
84
+ onOpen: () => {
85
+ void ensurePlayer();
86
+ overrides?.onOpen?.();
87
+ },
88
+ onAudioDelta: (pcm16) => {
89
+ void ensurePlayer().then(() => playerRef.current?.addAudio(pcm16));
90
+ overrides?.onAudioDelta?.(pcm16);
91
+ },
92
+ onClose: (reason) => {
93
+ cleanup();
94
+ overrides?.onClose?.(reason);
95
+ },
96
+ onError: (err) => {
97
+ cleanup();
98
+ overrides?.onError?.(err);
99
+ },
100
+ }), [cleanup, ensurePlayer]);
101
+ useEffect(() => {
102
+ if (!isVoiceActive) {
103
+ prevCaptureActiveRef.current = false;
104
+ cleanup();
105
+ return;
106
+ }
107
+ const prev = prevCaptureActiveRef.current;
108
+ prevCaptureActiveRef.current = isCaptureActive;
109
+ if (isCaptureActive && !prev)
110
+ void startCapture();
111
+ else if (!isCaptureActive && prev)
112
+ stopCapture(true);
113
+ }, [isCaptureActive, isVoiceActive, startCapture, stopCapture, cleanup]);
114
+ return { bindSession, buildCallbacks, getOutputAudioLevel: () => playerRef.current?.getAudioLevel() ?? 0 };
115
+ }
@@ -0,0 +1,4 @@
1
+ import type { VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
2
+ export declare const ASSISTANT_MIC_HOLD_STATUSES: ReadonlySet<VoiceStatus>;
3
+ export declare const SPEAKER_ACTIVE_STATUSES: ReadonlySet<VoiceStatus>;
4
+ //# sourceMappingURL=voiceMicConstants.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"voiceMicConstants.d.ts","sourceRoot":"","sources":["../../src/voice/voiceMicConstants.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8CAA8C,CAAC;AAEhF,eAAO,MAAM,2BAA2B,EAAE,WAAW,CAAC,WAAW,CAK/D,CAAC;AAEH,eAAO,MAAM,uBAAuB,EAAE,WAAW,CAAC,WAAW,CAG3D,CAAC"}
@@ -0,0 +1,10 @@
1
+ export const ASSISTANT_MIC_HOLD_STATUSES = new Set([
2
+ 'transcribing',
3
+ 'thinking',
4
+ 'generating',
5
+ 'speaking',
6
+ ]);
7
+ export const SPEAKER_ACTIVE_STATUSES = new Set([
8
+ 'generating',
9
+ 'speaking',
10
+ ]);
@@ -0,0 +1,23 @@
1
+ export declare class BrowserWsPcmPlayer {
2
+ private readonly sampleRate;
3
+ private readonly fadeSamples;
4
+ private audioContext;
5
+ private gainNode;
6
+ private analyserNode;
7
+ private analyserBuffer;
8
+ private readonly activeSources;
9
+ private queue;
10
+ private isPlaying;
11
+ private playHead;
12
+ private lastChunkRms;
13
+ private lastChunkAt;
14
+ ensureRunning(): Promise<void>;
15
+ prewarm(): void;
16
+ addAudio(pcm16: Int16Array | ArrayBuffer): void;
17
+ getAudioLevel(): number;
18
+ clearQueue(): void;
19
+ cleanup(): void;
20
+ private ensureContext;
21
+ private pipelineQueue;
22
+ }
23
+ //# sourceMappingURL=BrowserWsPcmPlayer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"BrowserWsPcmPlayer.d.ts","sourceRoot":"","sources":["../../../src/voice/ws/BrowserWsPcmPlayer.ts"],"names":[],"mappings":"AAeA,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA2B;IACtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmD;IAC/E,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAyB;IACzC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoC;IAClE,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,SAAS,CAAQ;IACzB,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,WAAW,CAAK;IAElB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAMpC,OAAO,IAAI,IAAI;IAIf,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,GAAG,IAAI;IAqB/C,aAAa,IAAI,MAAM;IAgBvB,UAAU,IAAI,IAAI;IAWlB,OAAO,IAAI,IAAI;IAUf,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,aAAa;CA2BtB"}
@@ -0,0 +1,138 @@
1
+ import { WS_REALTIME_SAMPLE_RATE } from './wsRealtimeConstants';
2
+ const LEVEL_SCALE = 3.2;
3
+ const LOOKAHEAD_SECONDS = 0.02;
4
+ const CROSSFADE_SECONDS = 0.04;
5
+ function applyFade(buffer, fadeSamples) {
6
+ const fade = Math.min(fadeSamples, Math.floor(buffer.length / 2));
7
+ if (fade <= 0)
8
+ return;
9
+ for (let i = 0; i < fade; i += 1) {
10
+ buffer[i] *= i / fade;
11
+ buffer[buffer.length - 1 - i] *= (fade - i) / fade;
12
+ }
13
+ }
14
+ export class BrowserWsPcmPlayer {
15
+ constructor() {
16
+ this.sampleRate = WS_REALTIME_SAMPLE_RATE;
17
+ this.fadeSamples = Math.max(1, Math.floor(this.sampleRate * 0.01));
18
+ this.audioContext = null;
19
+ this.gainNode = null;
20
+ this.analyserNode = null;
21
+ this.analyserBuffer = null;
22
+ this.activeSources = new Set();
23
+ this.queue = [];
24
+ this.isPlaying = true;
25
+ this.playHead = 0;
26
+ this.lastChunkRms = 0;
27
+ this.lastChunkAt = 0;
28
+ }
29
+ async ensureRunning() {
30
+ const ctx = this.ensureContext();
31
+ if (ctx.state === 'suspended')
32
+ await ctx.resume();
33
+ this.isPlaying = true;
34
+ }
35
+ prewarm() {
36
+ this.ensureContext();
37
+ }
38
+ addAudio(pcm16) {
39
+ const ctx = this.ensureContext();
40
+ const int16 = pcm16 instanceof ArrayBuffer ? new Int16Array(pcm16) : pcm16;
41
+ if (int16.length === 0)
42
+ return;
43
+ const floats = new Float32Array(int16.length);
44
+ let sumSq = 0;
45
+ for (let i = 0; i < int16.length; i += 1) {
46
+ const sample = int16[i] / 32768;
47
+ floats[i] = Math.max(-1, Math.min(1, sample));
48
+ sumSq += sample * sample;
49
+ }
50
+ applyFade(floats, this.fadeSamples);
51
+ this.lastChunkRms = Math.min(1, Math.sqrt(sumSq / int16.length) * LEVEL_SCALE);
52
+ this.lastChunkAt = Date.now();
53
+ const buffer = ctx.createBuffer(1, floats.length, this.sampleRate);
54
+ buffer.getChannelData(0).set(floats);
55
+ this.queue.push(buffer);
56
+ this.isPlaying = true;
57
+ this.pipelineQueue();
58
+ }
59
+ getAudioLevel() {
60
+ const analyser = this.analyserNode;
61
+ const buf = this.analyserBuffer;
62
+ if (analyser && buf && this.activeSources.size > 0) {
63
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
64
+ analyser.getFloatTimeDomainData(buf);
65
+ let sumSq = 0;
66
+ for (let i = 0; i < buf.length; i += 1) {
67
+ const v = buf[i];
68
+ sumSq += v * v;
69
+ }
70
+ return Math.min(1, Math.sqrt(sumSq / buf.length) * LEVEL_SCALE);
71
+ }
72
+ return Date.now() - this.lastChunkAt < 500 ? this.lastChunkRms : 0;
73
+ }
74
+ clearQueue() {
75
+ for (const source of this.activeSources) {
76
+ try {
77
+ source.stop();
78
+ }
79
+ catch { /* noop */ }
80
+ }
81
+ this.activeSources.clear();
82
+ this.queue = [];
83
+ this.playHead = this.audioContext?.currentTime ?? 0;
84
+ this.lastChunkRms = 0;
85
+ this.lastChunkAt = 0;
86
+ }
87
+ cleanup() {
88
+ this.clearQueue();
89
+ this.isPlaying = false;
90
+ void this.audioContext?.close();
91
+ this.analyserNode = null;
92
+ this.analyserBuffer = null;
93
+ this.gainNode = null;
94
+ this.audioContext = null;
95
+ }
96
+ ensureContext() {
97
+ if (!this.audioContext) {
98
+ this.audioContext = new AudioContext({ sampleRate: this.sampleRate });
99
+ this.gainNode = this.audioContext.createGain();
100
+ this.analyserNode = this.audioContext.createAnalyser();
101
+ this.analyserNode.fftSize = 512;
102
+ this.analyserBuffer = new Float32Array(this.analyserNode.fftSize);
103
+ this.gainNode.connect(this.analyserNode);
104
+ this.analyserNode.connect(this.audioContext.destination);
105
+ this.playHead = this.audioContext.currentTime;
106
+ }
107
+ return this.audioContext;
108
+ }
109
+ pipelineQueue() {
110
+ const ctx = this.audioContext;
111
+ const gain = this.gainNode;
112
+ if (!ctx || !gain || !this.isPlaying)
113
+ return;
114
+ while (this.queue.length > 0) {
115
+ const audioBuffer = this.queue.shift();
116
+ if (!audioBuffer)
117
+ return;
118
+ const source = ctx.createBufferSource();
119
+ source.buffer = audioBuffer;
120
+ const perSourceGain = ctx.createGain();
121
+ source.connect(perSourceGain);
122
+ perSourceGain.connect(gain);
123
+ const now = ctx.currentTime;
124
+ const startAt = Math.max((this.playHead || now) - CROSSFADE_SECONDS * 0.75, now + LOOKAHEAD_SECONDS);
125
+ const duration = audioBuffer.duration;
126
+ const endAt = startAt + duration;
127
+ const fade = Math.min(CROSSFADE_SECONDS, duration / 2);
128
+ perSourceGain.gain.setValueAtTime(0, startAt);
129
+ perSourceGain.gain.linearRampToValueAtTime(1, startAt + fade);
130
+ perSourceGain.gain.setValueAtTime(1, endAt - fade);
131
+ perSourceGain.gain.linearRampToValueAtTime(0, endAt);
132
+ this.playHead = endAt;
133
+ this.activeSources.add(source);
134
+ source.start(startAt);
135
+ source.onended = () => this.activeSources.delete(source);
136
+ }
137
+ }
138
+ }
@@ -0,0 +1,19 @@
1
+ import type { VoiceSession } from '@bytexbyte/nxtlinq-ai-agent-core-development';
2
+ /** getUserMedia only on start() — Berify hold-to-talk aligned. */
3
+ export declare class BrowserWsPcmRecorder {
4
+ private session;
5
+ private onRms?;
6
+ private stream;
7
+ private audioContext;
8
+ private processor;
9
+ private source;
10
+ private silentGain;
11
+ private shouldAppend?;
12
+ bindSession(session: VoiceSession | null): void;
13
+ setShouldAppend(handler: () => boolean): void;
14
+ setOnRms(handler: (rms: number) => void): void;
15
+ start(): Promise<void>;
16
+ stop(): void;
17
+ cleanup(): void;
18
+ }
19
+ //# sourceMappingURL=BrowserWsPcmRecorder.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"BrowserWsPcmRecorder.d.ts","sourceRoot":"","sources":["../../../src/voice/ws/BrowserWsPcmRecorder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,8CAA8C,CAAC;AAajF,kEAAkE;AAClE,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,KAAK,CAAC,CAAwB;IACtC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,SAAS,CAAoC;IACrD,OAAO,CAAC,MAAM,CAA2C;IACzD,OAAO,CAAC,UAAU,CAAyB;IAE3C,OAAO,CAAC,YAAY,CAAC,CAAgB;IAErC,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,GAAG,IAAI;IAI/C,eAAe,CAAC,OAAO,EAAE,MAAM,OAAO,GAAG,IAAI;IAI7C,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;IAIxC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA0B5B,IAAI,IAAI,IAAI;IAeZ,OAAO,IAAI,IAAI;CAIhB"}
@@ -0,0 +1,76 @@
1
+ import { float32ToPcm16 } from './float32ToPcm16';
2
+ import { WS_REALTIME_SAMPLE_RATE } from './wsRealtimeConstants';
3
+ function computeRms(channel) {
4
+ let sum = 0;
5
+ for (let i = 0; i < channel.length; i += 1) {
6
+ const v = channel[i];
7
+ sum += v * v;
8
+ }
9
+ return channel.length > 0 ? Math.sqrt(sum / channel.length) : 0;
10
+ }
11
+ /** getUserMedia only on start() — Berify hold-to-talk aligned. */
12
+ export class BrowserWsPcmRecorder {
13
+ constructor() {
14
+ this.session = null;
15
+ this.stream = null;
16
+ this.audioContext = null;
17
+ this.processor = null;
18
+ this.source = null;
19
+ this.silentGain = null;
20
+ }
21
+ bindSession(session) {
22
+ this.session = session;
23
+ }
24
+ setShouldAppend(handler) {
25
+ this.shouldAppend = handler;
26
+ }
27
+ setOnRms(handler) {
28
+ this.onRms = handler;
29
+ }
30
+ async start() {
31
+ if (this.processor)
32
+ return;
33
+ this.stream = await navigator.mediaDevices.getUserMedia({ audio: true });
34
+ const ctx = new AudioContext({ sampleRate: WS_REALTIME_SAMPLE_RATE });
35
+ this.audioContext = ctx;
36
+ this.source = ctx.createMediaStreamSource(this.stream);
37
+ this.processor = ctx.createScriptProcessor(4096, 1, 1);
38
+ this.silentGain = ctx.createGain();
39
+ this.silentGain.gain.value = 0;
40
+ this.source.connect(this.processor);
41
+ this.processor.connect(this.silentGain);
42
+ this.silentGain.connect(ctx.destination);
43
+ this.processor.onaudioprocess = (event) => {
44
+ if (!this.session?.appendInputAudio)
45
+ return;
46
+ const channel = event.inputBuffer.getChannelData(0);
47
+ const rms = computeRms(channel);
48
+ this.onRms?.(rms);
49
+ // Berify: only stream PCM after RMS crosses speech threshold.
50
+ if (this.shouldAppend && !this.shouldAppend())
51
+ return;
52
+ this.session.appendInputAudio(float32ToPcm16(channel));
53
+ };
54
+ if (ctx.state === 'suspended') {
55
+ await ctx.resume();
56
+ }
57
+ }
58
+ stop() {
59
+ this.processor?.disconnect();
60
+ this.source?.disconnect();
61
+ this.silentGain?.disconnect();
62
+ this.processor = null;
63
+ this.source = null;
64
+ this.silentGain = null;
65
+ for (const track of this.stream?.getTracks() ?? []) {
66
+ track.stop();
67
+ }
68
+ this.stream = null;
69
+ void this.audioContext?.close();
70
+ this.audioContext = null;
71
+ }
72
+ cleanup() {
73
+ this.stop();
74
+ this.session = null;
75
+ }
76
+ }
@@ -0,0 +1,2 @@
1
+ export declare function float32ToPcm16(input: Float32Array): Int16Array;
2
+ //# sourceMappingURL=float32ToPcm16.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"float32ToPcm16.d.ts","sourceRoot":"","sources":["../../../src/voice/ws/float32ToPcm16.ts"],"names":[],"mappings":"AAAA,wBAAgB,cAAc,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,CAO9D"}
@@ -0,0 +1,8 @@
1
+ export function float32ToPcm16(input) {
2
+ const out = new Int16Array(input.length);
3
+ for (let i = 0; i < input.length; i += 1) {
4
+ const sample = Math.max(-1, Math.min(1, input[i]));
5
+ out[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
6
+ }
7
+ return out;
8
+ }
@@ -0,0 +1,5 @@
1
+ export declare const MIC_SPEECH_RMS_THRESHOLD = 0.008;
2
+ export declare const MIC_BARGE_IN_RMS_THRESHOLD = 0.045;
3
+ export declare const MIC_SILENCE_COMMIT_MS = 400;
4
+ export declare const MIC_SILENCE_POLL_MS = 100;
5
+ //# sourceMappingURL=voiceSilenceConstants.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"voiceSilenceConstants.d.ts","sourceRoot":"","sources":["../../../src/voice/ws/voiceSilenceConstants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAC9C,eAAO,MAAM,0BAA0B,QAAQ,CAAC;AAChD,eAAO,MAAM,qBAAqB,MAAM,CAAC;AACzC,eAAO,MAAM,mBAAmB,MAAM,CAAC"}
@@ -0,0 +1,4 @@
1
+ export const MIC_SPEECH_RMS_THRESHOLD = 0.008;
2
+ export const MIC_BARGE_IN_RMS_THRESHOLD = 0.045;
3
+ export const MIC_SILENCE_COMMIT_MS = 400;
4
+ export const MIC_SILENCE_POLL_MS = 100;
@@ -0,0 +1,2 @@
1
+ export declare const WS_REALTIME_SAMPLE_RATE = 24000;
2
+ //# sourceMappingURL=wsRealtimeConstants.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"wsRealtimeConstants.d.ts","sourceRoot":"","sources":["../../../src/voice/ws/wsRealtimeConstants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,uBAAuB,QAAQ,CAAC"}
@@ -0,0 +1 @@
1
+ export const WS_REALTIME_SAMPLE_RATE = 24000;
@@ -0,0 +1,9 @@
1
+ import type { AgentConfig } from '@bytexbyte/nxtlinq-ai-agent-core-development';
2
+ /** Browser SDK defaults — integrators do not pass these props. */
3
+ export declare const WEB_AGENT_DEFAULTS: {
4
+ readonly voiceTransport: "webrtc";
5
+ readonly voiceMode: "realtime";
6
+ readonly defaultModel: "open-ai-stream";
7
+ };
8
+ export declare function applyWebAgentDefaults<C extends AgentConfig>(config: C): C;
9
+ //# sourceMappingURL=webAgentDefaults.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"webAgentDefaults.d.ts","sourceRoot":"","sources":["../src/webAgentDefaults.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8CAA8C,CAAC;AAEhF,kEAAkE;AAClE,eAAO,MAAM,kBAAkB;;;;CAIU,CAAC;AAE1C,wBAAgB,qBAAqB,CAAC,CAAC,SAAS,WAAW,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,CAEzE"}
@@ -0,0 +1,9 @@
1
+ /** Browser SDK defaults — integrators do not pass these props. */
2
+ export const WEB_AGENT_DEFAULTS = {
3
+ voiceTransport: 'webrtc',
4
+ voiceMode: 'realtime',
5
+ defaultModel: 'open-ai-stream',
6
+ };
7
+ export function applyWebAgentDefaults(config) {
8
+ return { ...WEB_AGENT_DEFAULTS, ...config };
9
+ }
package/package.json ADDED
@@ -0,0 +1,55 @@
1
+ {
2
+ "name": "@bytexbyte/nxtlinq-ai-agent-web-development",
3
+ "version": "0.1.1",
4
+ "description": "React Web headless SDK for nxtlinq AI Agent — hooks and browser ports",
5
+ "main": "dist/index.js",
6
+ "types": "dist/index.d.ts",
7
+ "files": [
8
+ "dist",
9
+ "src"
10
+ ],
11
+ "scripts": {
12
+ "build": "tsc",
13
+ "clean": "rm -rf dist",
14
+ "prepublishOnly": "yarn build"
15
+ },
16
+ "keywords": [
17
+ "nxtlinq",
18
+ "ai-agent",
19
+ "react",
20
+ "web",
21
+ "sdk"
22
+ ],
23
+ "author": "ByteXByte",
24
+ "license": "MIT",
25
+ "repository": {
26
+ "type": "git",
27
+ "url": "https://dev.azure.com/nxtlinqLLC/nxtlinq/_git/nxtlinq-AI-Agent-SDK",
28
+ "directory": "packages/ai-agent-web"
29
+ },
30
+ "publishConfig": {
31
+ "access": "public",
32
+ "registry": "https://registry.npmjs.org/"
33
+ },
34
+ "sideEffects": false,
35
+ "peerDependencies": {
36
+ "react": ">=18.0.0",
37
+ "react-dom": ">=18.0.0"
38
+ },
39
+ "dependencies": {
40
+ "@bytexbyte/nxtlinq-ai-agent-core-development": "0.3.8",
41
+ "ethers": "^6.16.0",
42
+ "fast-json-stable-stringify": "^2.1.0",
43
+ "metakeep": "^2.2.8",
44
+ "microsoft-cognitiveservices-speech-sdk": "^1.45.0",
45
+ "universal-cookie": "^8.0.1"
46
+ },
47
+ "devDependencies": {
48
+ "@bytexbyte/nxtlinq-ai-agent-core-development": "workspace:^",
49
+ "@types/react": "^18.2.64",
50
+ "@types/react-dom": "^18.2.25",
51
+ "react": "^18.2.0",
52
+ "react-dom": "^18.2.0",
53
+ "typescript": "^5.4.2"
54
+ }
55
+ }
@@ -0,0 +1,79 @@
1
+ import {
2
+ NxtlinqAgent,
3
+ type AgentConfig,
4
+ } from '@bytexbyte/nxtlinq-ai-agent-core-development';
5
+ import React, {
6
+ createContext,
7
+ useContext,
8
+ useEffect,
9
+ useMemo,
10
+ type ReactNode,
11
+ } from 'react';
12
+ import {
13
+ createNxtlinqAgentWeb,
14
+ type CreateNxtlinqAgentWebOptions,
15
+ } from '../createNxtlinqAgent';
16
+
17
+ const NxtlinqAgentContext = createContext<NxtlinqAgent | null>(null);
18
+
19
+ type WebAgentInternalProps =
20
+ | 'voiceTransport'
21
+ | 'voiceMode'
22
+ | 'defaultModel'
23
+ | 'enableWebRTC';
24
+
25
+ export type NxtlinqAgentProviderProps = Omit<
26
+ CreateNxtlinqAgentWebOptions,
27
+ WebAgentInternalProps
28
+ > & {
29
+ children: ReactNode;
30
+ resetOnIdentityChange?: boolean;
31
+ };
32
+
33
+ export function NxtlinqAgentProvider({
34
+ children,
35
+ resetOnIdentityChange = true,
36
+ ...options
37
+ }: NxtlinqAgentProviderProps): React.ReactElement {
38
+ const agent = useMemo(
39
+ () => createNxtlinqAgentWeb(options),
40
+ [
41
+ options.serviceId,
42
+ options.apiKey,
43
+ options.apiSecret,
44
+ options.environment,
45
+ options.pseudoId,
46
+ options.externalId,
47
+ options.conversationId,
48
+ options.storage,
49
+ options.fetchImpl,
50
+ options.getTimezone,
51
+ options.webrtc,
52
+ ],
53
+ );
54
+
55
+ useEffect(() => {
56
+ if (!resetOnIdentityChange) return;
57
+ agent.setMessages([]);
58
+ }, [agent, resetOnIdentityChange, options.pseudoId, options.externalId, options.conversationId]);
59
+
60
+ useEffect(() => () => agent.destroy(), [agent]);
61
+
62
+ return (
63
+ <NxtlinqAgentContext.Provider value={agent}>
64
+ {children}
65
+ </NxtlinqAgentContext.Provider>
66
+ );
67
+ }
68
+
69
+ export function useNxtlinqAgentContext(): NxtlinqAgent {
70
+ const agent = useContext(NxtlinqAgentContext);
71
+ if (!agent) {
72
+ throw new Error(
73
+ 'useNxtlinqAgentContext must be used within <NxtlinqAgentProvider>',
74
+ );
75
+ }
76
+ return agent;
77
+ }
78
+
79
+ export type { AgentConfig };
@@ -0,0 +1,36 @@
1
+ import {
2
+ NxtlinqAgent,
3
+ type AgentConfig,
4
+ type PlatformPorts,
5
+ } from '@bytexbyte/nxtlinq-ai-agent-core-development';
6
+ import {
7
+ createWebPlatformPorts,
8
+ type CreateWebPlatformPortsOptions,
9
+ } from './ports/createWebPlatformPorts';
10
+ import { applyWebAgentDefaults } from './webAgentDefaults';
11
+
12
+ export type CreateNxtlinqAgentWebOptions = AgentConfig & CreateWebPlatformPortsOptions;
13
+
14
+ /**
15
+ * Create a platform-ready {@link NxtlinqAgent} for React Web custom UIs.
16
+ * Applies browser defaults (`voiceTransport: webrtc`, `defaultModel`).
17
+ */
18
+ export function createNxtlinqAgentWeb(options: CreateNxtlinqAgentWebOptions): NxtlinqAgent {
19
+ const {
20
+ storage,
21
+ fetchImpl,
22
+ getTimezone,
23
+ webrtc,
24
+ enableWebRTC,
25
+ ...rawConfig
26
+ } = options;
27
+ const config = applyWebAgentDefaults(rawConfig);
28
+ const ports: PlatformPorts = createWebPlatformPorts({
29
+ storage,
30
+ fetchImpl,
31
+ getTimezone,
32
+ webrtc,
33
+ enableWebRTC,
34
+ });
35
+ return new NxtlinqAgent(config, ports);
36
+ }