@agent-native/toolkit 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/README.md +17 -1
  2. package/dist/chat-history/ChatHistoryList.d.ts +103 -0
  3. package/dist/chat-history/ChatHistoryList.d.ts.map +1 -0
  4. package/dist/chat-history/ChatHistoryList.js +109 -0
  5. package/dist/chat-history/ChatHistoryList.js.map +1 -0
  6. package/dist/chat-history/ChatHistoryList.spec.d.ts +2 -0
  7. package/dist/chat-history/ChatHistoryList.spec.d.ts.map +1 -0
  8. package/dist/chat-history/ChatHistoryList.spec.js +283 -0
  9. package/dist/chat-history/ChatHistoryList.spec.js.map +1 -0
  10. package/dist/chat-history/index.d.ts +2 -0
  11. package/dist/chat-history/index.d.ts.map +1 -0
  12. package/dist/chat-history/index.js +2 -0
  13. package/dist/chat-history/index.js.map +1 -0
  14. package/dist/chat-history.css +314 -0
  15. package/package.json +17 -1
  16. package/src/app-shell/header-actions.tsx +84 -0
  17. package/src/app-shell/index.ts +1 -0
  18. package/src/chat-history/ChatHistoryList.spec.tsx +453 -0
  19. package/src/chat-history/ChatHistoryList.tsx +469 -0
  20. package/src/chat-history/index.ts +7 -0
  21. package/src/chat-history.css +314 -0
  22. package/src/collab-ui/AgentPresenceChip.tsx +69 -0
  23. package/src/collab-ui/LiveCursorOverlay.tsx +241 -0
  24. package/src/collab-ui/PresenceBar.tsx +346 -0
  25. package/src/collab-ui/RecentEditHighlights.tsx +225 -0
  26. package/src/collab-ui/RemoteSelectionRings.tsx +227 -0
  27. package/src/collab-ui/agent-identity.ts +16 -0
  28. package/src/collab-ui/index.ts +39 -0
  29. package/src/collab-ui/lead-client.ts +32 -0
  30. package/src/collab-ui/types.spec.ts +18 -0
  31. package/src/collab-ui/types.ts +96 -0
  32. package/src/composer/AgentComposerFrame.tsx +59 -0
  33. package/src/composer/ComposerPlusMenu.tsx +983 -0
  34. package/src/composer/MentionPopover.tsx +412 -0
  35. package/src/composer/PastedTextChip.tsx +150 -0
  36. package/src/composer/PromptComposer.spec.ts +73 -0
  37. package/src/composer/PromptComposer.tsx +718 -0
  38. package/src/composer/RealtimeVoiceMode.spec.tsx +771 -0
  39. package/src/composer/RealtimeVoiceMode.tsx +856 -0
  40. package/src/composer/TiptapComposer.spec.ts +390 -0
  41. package/src/composer/TiptapComposer.tsx +2936 -0
  42. package/src/composer/VoiceButton.spec.ts +33 -0
  43. package/src/composer/VoiceButton.tsx +255 -0
  44. package/src/composer/asset-picker-url.spec.ts +56 -0
  45. package/src/composer/asset-picker-url.ts +80 -0
  46. package/src/composer/attachment-accept.spec.ts +22 -0
  47. package/src/composer/attachment-accept.ts +71 -0
  48. package/src/composer/draft-key.spec.ts +19 -0
  49. package/src/composer/draft-key.ts +7 -0
  50. package/src/composer/extensions/FileReference.tsx +55 -0
  51. package/src/composer/extensions/MentionReference.tsx +93 -0
  52. package/src/composer/extensions/SkillReference.tsx +56 -0
  53. package/src/composer/index.ts +79 -0
  54. package/src/composer/pasted-text.spec.ts +162 -0
  55. package/src/composer/pasted-text.ts +160 -0
  56. package/src/composer/prompt-attachments.spec.ts +80 -0
  57. package/src/composer/prompt-attachments.ts +93 -0
  58. package/src/composer/realtime-voice-audio-level.spec.ts +32 -0
  59. package/src/composer/realtime-voice-audio-level.ts +64 -0
  60. package/src/composer/realtime-voice-transcript.spec.ts +95 -0
  61. package/src/composer/realtime-voice-transcript.ts +134 -0
  62. package/src/composer/runtime-adapters.tsx +343 -0
  63. package/src/composer/types.ts +66 -0
  64. package/src/composer/use-file-search.ts +64 -0
  65. package/src/composer/use-mention-search.ts +90 -0
  66. package/src/composer/use-skills.ts +45 -0
  67. package/src/composer/useRealtimeVoiceMode.spec.ts +818 -0
  68. package/src/composer/useRealtimeVoiceMode.tsx +2117 -0
  69. package/src/composer/useVoiceDictation.spec.ts +45 -0
  70. package/src/composer/useVoiceDictation.ts +1191 -0
  71. package/src/context-ui/ContextMeter.tsx +101 -0
  72. package/src/context-ui/ContextSegmentRow.tsx +184 -0
  73. package/src/context-ui/ContextTreemap.tsx +125 -0
  74. package/src/context-ui/ContextXRayPanel.tsx +377 -0
  75. package/src/context-ui/format.ts +42 -0
  76. package/src/context-ui/index.ts +15 -0
  77. package/src/context-ui/types.ts +43 -0
  78. package/src/design-tweaks/index.ts +1 -0
  79. package/src/design-tweaks/visual-style-controls.tsx +1084 -0
  80. package/src/editor/BubbleToolbar.tsx +253 -0
  81. package/src/editor/CodeBlockNode.tsx +254 -0
  82. package/src/editor/DragHandle.spec.ts +588 -0
  83. package/src/editor/DragHandle.ts +1437 -0
  84. package/src/editor/ImageExtension.roundtrip.spec.ts +117 -0
  85. package/src/editor/ImageExtension.ts +306 -0
  86. package/src/editor/RegistryBlockContext.tsx +107 -0
  87. package/src/editor/RegistryBlockNode.spec.ts +279 -0
  88. package/src/editor/RegistryBlockNode.tsx +652 -0
  89. package/src/editor/RichMarkdownEditor.collab.spec.ts +79 -0
  90. package/src/editor/RichMarkdownEditor.roundtrip.spec.ts +226 -0
  91. package/src/editor/RichMarkdownEditor.tsx +81 -0
  92. package/src/editor/RunId.ts +61 -0
  93. package/src/editor/SharedRichEditor.tsx +321 -0
  94. package/src/editor/SlashCommandMenu.tsx +310 -0
  95. package/src/editor/disableHistory.spec.ts +85 -0
  96. package/src/editor/extensions.ts +402 -0
  97. package/src/editor/gfmDoc.spec.ts +230 -0
  98. package/src/editor/gfmDoc.ts +92 -0
  99. package/src/editor/index.ts +93 -0
  100. package/src/editor/registrySlashCommands.ts +131 -0
  101. package/src/editor/structuralReorderUndo.spec.ts +144 -0
  102. package/src/editor/surgical-apply.spec.ts +279 -0
  103. package/src/editor/surgical-apply.ts +192 -0
  104. package/src/editor/useCollabReconcile.concurrent.spec.ts +422 -0
  105. package/src/editor/useCollabReconcile.idempotent.spec.ts +324 -0
  106. package/src/editor/useCollabReconcile.ts +589 -0
  107. package/src/editor.css +510 -0
  108. package/src/hooks/index.ts +2 -0
  109. package/src/hooks/use-mobile.tsx +21 -0
  110. package/src/hooks/use-toast.ts +188 -0
  111. package/src/index.ts +8 -0
  112. package/src/onboarding/index.ts +1 -0
  113. package/src/provider.tsx +47 -0
  114. package/src/sharing/VisibilityBadge.spec.ts +96 -0
  115. package/src/sharing/VisibilityBadge.tsx +37 -0
  116. package/src/sharing/index.ts +4 -0
  117. package/src/styles.css +21 -0
  118. package/src/ui/accordion.tsx +56 -0
  119. package/src/ui/alert-dialog.tsx +141 -0
  120. package/src/ui/alert.tsx +59 -0
  121. package/src/ui/aspect-ratio.tsx +5 -0
  122. package/src/ui/avatar.tsx +48 -0
  123. package/src/ui/badge.tsx +37 -0
  124. package/src/ui/breadcrumb.tsx +115 -0
  125. package/src/ui/button.tsx +78 -0
  126. package/src/ui/calendar.tsx +214 -0
  127. package/src/ui/card.tsx +86 -0
  128. package/src/ui/carousel.tsx +260 -0
  129. package/src/ui/chart.tsx +375 -0
  130. package/src/ui/checkbox.tsx +28 -0
  131. package/src/ui/collapsible.tsx +9 -0
  132. package/src/ui/command.spec.tsx +51 -0
  133. package/src/ui/command.tsx +174 -0
  134. package/src/ui/context-menu.tsx +202 -0
  135. package/src/ui/date-picker.tsx +64 -0
  136. package/src/ui/dialog.spec.tsx +43 -0
  137. package/src/ui/dialog.tsx +146 -0
  138. package/src/ui/drawer.tsx +126 -0
  139. package/src/ui/dropdown-menu.tsx +218 -0
  140. package/src/ui/form.tsx +177 -0
  141. package/src/ui/hover-card.tsx +27 -0
  142. package/src/ui/index.ts +52 -0
  143. package/src/ui/input-otp.tsx +69 -0
  144. package/src/ui/input.tsx +22 -0
  145. package/src/ui/label.tsx +24 -0
  146. package/src/ui/logo.tsx +36 -0
  147. package/src/ui/menubar.tsx +235 -0
  148. package/src/ui/navigation-menu.tsx +128 -0
  149. package/src/ui/pagination.tsx +121 -0
  150. package/src/ui/popover.tsx +44 -0
  151. package/src/ui/progress.tsx +26 -0
  152. package/src/ui/radio-group.tsx +42 -0
  153. package/src/ui/resizable.tsx +43 -0
  154. package/src/ui/scroll-area.tsx +46 -0
  155. package/src/ui/select.tsx +158 -0
  156. package/src/ui/separator.tsx +29 -0
  157. package/src/ui/sheet.tsx +160 -0
  158. package/src/ui/sidebar.tsx +777 -0
  159. package/src/ui/skeleton.tsx +18 -0
  160. package/src/ui/slider.tsx +26 -0
  161. package/src/ui/sonner.tsx +58 -0
  162. package/src/ui/spinner.tsx +17 -0
  163. package/src/ui/switch.tsx +27 -0
  164. package/src/ui/table.tsx +117 -0
  165. package/src/ui/tabs.tsx +53 -0
  166. package/src/ui/textarea.tsx +23 -0
  167. package/src/ui/toast.tsx +127 -0
  168. package/src/ui/toaster.tsx +33 -0
  169. package/src/ui/toggle-group.tsx +59 -0
  170. package/src/ui/toggle.tsx +43 -0
  171. package/src/ui/tooltip.tsx +53 -0
  172. package/src/ui/use-toast.ts +3 -0
  173. package/src/utils.ts +8 -0
@@ -0,0 +1,818 @@
1
+ // @vitest-environment happy-dom
2
+
3
+ import { afterEach, describe, expect, it, vi } from "vitest";
4
+
5
+ import {
6
+ createRealtimeVoiceGreetingEvent,
7
+ createRealtimeVoiceGreetingStarter,
8
+ createRealtimeVoicePreferenceUpdate,
9
+ createRealtimeVoiceSession,
10
+ createRealtimeVoiceSessionWithCapability,
11
+ createRealtimeVoiceTranscriptSequencer,
12
+ createRealtimeVoiceConnectionTimeout,
13
+ createRealtimeVoiceConnectionGate,
14
+ createRealtimeVoiceAudioConstraints,
15
+ createRealtimeVoiceToolManifestCoordinator,
16
+ executeRealtimeVoiceTool,
17
+ extractCompletedRealtimeVoiceTranscript,
18
+ extractRealtimeVoiceFunctionCalls,
19
+ extractRealtimeVoiceSessionTools,
20
+ isRealtimeVoiceAbortError,
21
+ isRealtimeVoiceSetupRequiredError,
22
+ listenForRealtimeVoicePageHide,
23
+ mergeRealtimeVoiceToolManifest,
24
+ normalizeRealtimeVoicePreferences,
25
+ REALTIME_VOICE_AUDIO_CONSTRAINTS,
26
+ replaceRealtimeVoiceMicrophone,
27
+ realtimeVoiceReasoningEffort,
28
+ resolveRealtimeVoiceLanguage,
29
+ shouldRestoreRealtimeVoiceTranscriptThread,
30
+ } from "./useRealtimeVoiceMode.js";
31
+ import type { RealtimeVoiceFunctionTool } from "./useRealtimeVoiceMode.js";
32
+
33
+ afterEach(() => {
34
+ vi.useRealTimers();
35
+ vi.unstubAllGlobals();
36
+ });
37
+
38
+ function realtimeTool(name: string): RealtimeVoiceFunctionTool {
39
+ return {
40
+ type: "function",
41
+ name,
42
+ description: `Call ${name}`,
43
+ parameters: { type: "object", properties: {} },
44
+ };
45
+ }
46
+
47
+ describe("Realtime voice client transport", () => {
48
+ it("normalizes persisted preferences and resolves Auto from the browser language", () => {
49
+ expect(
50
+ normalizeRealtimeVoicePreferences({
51
+ language: "en",
52
+ intelligence: "deep",
53
+ voice: "cedar",
54
+ }),
55
+ ).toEqual({ language: "en", intelligence: "deep", voice: "cedar" });
56
+ expect(
57
+ normalizeRealtimeVoicePreferences({
58
+ language: "xx",
59
+ intelligence: "maximum",
60
+ voice: "unknown",
61
+ }),
62
+ ).toEqual({ language: "auto", intelligence: "instant", voice: "marin" });
63
+ expect(resolveRealtimeVoiceLanguage("auto", ["en-US", "fr"])).toBe("en");
64
+ expect(resolveRealtimeVoiceLanguage("auto", ["nl-NL"])).toBe("en");
65
+ });
66
+
67
+ it("maps inline intelligence and language controls to a safe session update", () => {
68
+ expect(realtimeVoiceReasoningEffort("instant")).toBe("minimal");
69
+ expect(realtimeVoiceReasoningEffort("balanced")).toBe("low");
70
+ expect(realtimeVoiceReasoningEffort("deep")).toBe("medium");
71
+ expect(
72
+ createRealtimeVoicePreferenceUpdate(
73
+ { language: "auto", intelligence: "balanced", voice: "cedar" },
74
+ { browserLanguages: ["en-US"], includeVoice: true },
75
+ ),
76
+ ).toEqual({
77
+ type: "session.update",
78
+ session: {
79
+ type: "realtime",
80
+ reasoning: { effort: "low" },
81
+ audio: {
82
+ input: {
83
+ transcription: {
84
+ model: "gpt-4o-mini-transcribe",
85
+ language: "en",
86
+ },
87
+ },
88
+ output: { voice: "cedar" },
89
+ },
90
+ },
91
+ });
92
+ });
93
+
94
+ it("prefers the browser and OS default microphone without requiring it", () => {
95
+ expect(REALTIME_VOICE_AUDIO_CONSTRAINTS).toEqual(
96
+ expect.objectContaining({
97
+ deviceId: { ideal: "default" },
98
+ echoCancellation: true,
99
+ noiseSuppression: true,
100
+ autoGainControl: true,
101
+ }),
102
+ );
103
+ expect(createRealtimeVoiceAudioConstraints("studio", true)).toEqual(
104
+ expect.objectContaining({ deviceId: { exact: "studio" } }),
105
+ );
106
+ expect(createRealtimeVoiceAudioConstraints("studio")).toEqual(
107
+ expect.objectContaining({ deviceId: { ideal: "studio" } }),
108
+ );
109
+ });
110
+
111
+ it("replaces the live microphone before stopping the previous track", async () => {
112
+ const events: string[] = [];
113
+ const oldTrack = {
114
+ kind: "audio",
115
+ onended: vi.fn(),
116
+ stop: vi.fn(() => events.push("old-stopped")),
117
+ };
118
+ const replacementTrack = { kind: "audio", stop: vi.fn() };
119
+ const replacementStream = {
120
+ getAudioTracks: () => [replacementTrack],
121
+ getTracks: () => [replacementTrack],
122
+ };
123
+ const sender = {
124
+ track: oldTrack,
125
+ replaceTrack: vi.fn(async () => {
126
+ events.push("replaced");
127
+ }),
128
+ };
129
+
130
+ await expect(
131
+ replaceRealtimeVoiceMicrophone({
132
+ mediaDevices: {
133
+ getUserMedia: vi.fn(async () => replacementStream),
134
+ } as unknown as Pick<MediaDevices, "getUserMedia">,
135
+ peer: {
136
+ getSenders: () => [sender],
137
+ } as unknown as Pick<RTCPeerConnection, "getSenders">,
138
+ currentStream: {
139
+ getTracks: () => [oldTrack],
140
+ } as unknown as MediaStream,
141
+ deviceId: "studio",
142
+ }),
143
+ ).resolves.toBe(replacementStream);
144
+
145
+ expect(events).toEqual(["replaced", "old-stopped"]);
146
+ expect(oldTrack.onended).toBeNull();
147
+ expect(replacementTrack.stop).not.toHaveBeenCalled();
148
+ });
149
+
150
+ it("keeps the current microphone when replacing its track fails", async () => {
151
+ const oldTrack = { kind: "audio", stop: vi.fn() };
152
+ const replacementTrack = { kind: "audio", stop: vi.fn() };
153
+ const replacementStream = {
154
+ getAudioTracks: () => [replacementTrack],
155
+ getTracks: () => [replacementTrack],
156
+ };
157
+
158
+ await expect(
159
+ replaceRealtimeVoiceMicrophone({
160
+ mediaDevices: {
161
+ getUserMedia: vi.fn(async () => replacementStream),
162
+ } as unknown as Pick<MediaDevices, "getUserMedia">,
163
+ peer: {
164
+ getSenders: () => [
165
+ {
166
+ track: oldTrack,
167
+ replaceTrack: vi.fn(async () => {
168
+ throw new Error("replace failed");
169
+ }),
170
+ },
171
+ ],
172
+ } as unknown as Pick<RTCPeerConnection, "getSenders">,
173
+ currentStream: {
174
+ getTracks: () => [oldTrack],
175
+ } as unknown as MediaStream,
176
+ deviceId: "studio",
177
+ }),
178
+ ).rejects.toThrow("replace failed");
179
+
180
+ expect(oldTrack.stop).not.toHaveBeenCalled();
181
+ expect(replacementTrack.stop).toHaveBeenCalledOnce();
182
+ });
183
+
184
+ it("times out a connection attempt and supports idempotent cancellation", () => {
185
+ vi.useFakeTimers();
186
+ const onTimeout = vi.fn();
187
+ const cancelFirst = createRealtimeVoiceConnectionTimeout(onTimeout, 1_000);
188
+
189
+ cancelFirst();
190
+ cancelFirst();
191
+ vi.advanceTimersByTime(1_000);
192
+ expect(onTimeout).not.toHaveBeenCalled();
193
+
194
+ const cancelSecond = createRealtimeVoiceConnectionTimeout(onTimeout, 1_000);
195
+
196
+ vi.advanceTimersByTime(999);
197
+ expect(onTimeout).not.toHaveBeenCalled();
198
+ vi.advanceTimersByTime(1);
199
+ expect(onTimeout).toHaveBeenCalledOnce();
200
+
201
+ cancelSecond();
202
+ vi.useRealTimers();
203
+ });
204
+
205
+ it("keeps the handshake deadline armed until session.created", () => {
206
+ vi.useFakeTimers();
207
+ const onTimeout = vi.fn();
208
+ const transportOnly = createRealtimeVoiceConnectionGate(onTimeout, 1_000);
209
+
210
+ transportOnly.markTransportReady();
211
+ vi.advanceTimersByTime(1_000);
212
+ expect(onTimeout).toHaveBeenCalledOnce();
213
+
214
+ const liveSession = createRealtimeVoiceConnectionGate(onTimeout, 1_000);
215
+ liveSession.markTransportReady();
216
+ liveSession.markSessionCreated();
217
+ vi.advanceTimersByTime(1_000);
218
+ expect(onTimeout).toHaveBeenCalledOnce();
219
+ vi.useRealTimers();
220
+ });
221
+
222
+ it("recognizes abort-like DOM errors without relying on Error identity", () => {
223
+ expect(
224
+ isRealtimeVoiceAbortError(
225
+ new DOMException("The operation was aborted", "AbortError"),
226
+ ),
227
+ ).toBe(true);
228
+ expect(isRealtimeVoiceAbortError({ name: "AbortError" })).toBe(true);
229
+ expect(isRealtimeVoiceAbortError(new Error("signal was aborted"))).toBe(
230
+ false,
231
+ );
232
+ });
233
+
234
+ it("cleans up realtime transport when the page is hidden", () => {
235
+ const cleanup = vi.fn();
236
+ const stopListening = listenForRealtimeVoicePageHide(cleanup);
237
+
238
+ window.dispatchEvent(new Event("pagehide"));
239
+ expect(cleanup).toHaveBeenCalledOnce();
240
+
241
+ stopListening();
242
+ window.dispatchEvent(new Event("pagehide"));
243
+ expect(cleanup).toHaveBeenCalledOnce();
244
+ });
245
+
246
+ it("creates a same-origin SDP session without exposing a provider key", async () => {
247
+ const fetchMock = vi.fn(
248
+ async () =>
249
+ new Response("answer-sdp", {
250
+ status: 200,
251
+ headers: {
252
+ "Content-Type": "application/sdp",
253
+ "X-Agent-Native-Realtime-Capability": "capability-1",
254
+ },
255
+ }),
256
+ );
257
+ vi.stubGlobal("fetch", fetchMock);
258
+
259
+ await expect(
260
+ createRealtimeVoiceSession("offer-sdp", {
261
+ browserTabId: "tab-1",
262
+ preferences: {
263
+ language: "auto",
264
+ intelligence: "instant",
265
+ voice: "marin",
266
+ },
267
+ browserLanguages: ["en-US"],
268
+ }),
269
+ ).resolves.toBe("answer-sdp");
270
+ await expect(
271
+ createRealtimeVoiceSessionWithCapability("offer-sdp"),
272
+ ).resolves.toEqual({
273
+ sdp: "answer-sdp",
274
+ capability: "capability-1",
275
+ });
276
+ expect(fetchMock).toHaveBeenCalledWith(
277
+ "/_agent-native/realtime-voice/session",
278
+ expect.objectContaining({
279
+ method: "POST",
280
+ body: "offer-sdp",
281
+ headers: {
282
+ "Content-Type": "application/sdp",
283
+ "X-Agent-Native-Browser-Tab": "tab-1",
284
+ "X-Agent-Native-Realtime-Language": "en",
285
+ "X-Agent-Native-Realtime-Intelligence": "instant",
286
+ "X-Agent-Native-Realtime-Voice": "marin",
287
+ },
288
+ }),
289
+ );
290
+ expect(JSON.stringify(fetchMock.mock.calls)).not.toContain(
291
+ "OPENAI_API_KEY",
292
+ );
293
+ });
294
+
295
+ it("sends function calls to the authenticated Agent Native tool bridge", async () => {
296
+ const fetchMock = vi.fn(async () =>
297
+ Response.json({
298
+ callId: "call-1",
299
+ status: "completed",
300
+ output: '{"ok":true}',
301
+ }),
302
+ );
303
+ vi.stubGlobal("fetch", fetchMock);
304
+
305
+ await expect(
306
+ executeRealtimeVoiceTool({
307
+ name: "navigate",
308
+ args: { path: "/inbox" },
309
+ callId: "call-1",
310
+ sessionId: "session-1",
311
+ browserTabId: "tab-1",
312
+ capability: "capability-1",
313
+ }),
314
+ ).resolves.toEqual({
315
+ callId: "call-1",
316
+ status: "completed",
317
+ output: '{"ok":true}',
318
+ });
319
+ expect(fetchMock).toHaveBeenCalledWith(
320
+ "/_agent-native/realtime-voice/tool",
321
+ expect.objectContaining({
322
+ method: "POST",
323
+ headers: expect.objectContaining({
324
+ "X-Agent-Native-Browser-Tab": "tab-1",
325
+ "X-Agent-Native-Realtime-Capability": "capability-1",
326
+ }),
327
+ body: JSON.stringify({
328
+ name: "navigate",
329
+ args: { path: "/inbox" },
330
+ callId: "call-1",
331
+ sessionId: "session-1",
332
+ browserTabId: "tab-1",
333
+ }),
334
+ }),
335
+ );
336
+ });
337
+ });
338
+
339
+ describe("Realtime voice dynamic tool manifests", () => {
340
+ it("reads full tool manifests from session lifecycle events", () => {
341
+ const tools = [realtimeTool("navigate"), realtimeTool("tool-search")];
342
+ expect(
343
+ extractRealtimeVoiceSessionTools({
344
+ type: "session.created",
345
+ session: { id: "session-1", tools },
346
+ }),
347
+ ).toEqual(tools);
348
+ expect(
349
+ extractRealtimeVoiceSessionTools({
350
+ type: "session.updated",
351
+ session: { tools },
352
+ }),
353
+ ).toEqual(tools);
354
+ expect(
355
+ extractRealtimeVoiceSessionTools({
356
+ type: "response.done",
357
+ response: {},
358
+ }),
359
+ ).toBeNull();
360
+ });
361
+
362
+ it("keeps pinned tools and newest discoveries when the manifest is full", () => {
363
+ const pinned = [
364
+ "navigate",
365
+ "set-url-path",
366
+ "set-search-params",
367
+ "view-screen",
368
+ "tool-search",
369
+ ].map(realtimeTool);
370
+ const current = [
371
+ ...pinned,
372
+ ...Array.from({ length: 27 }, (_, index) => realtimeTool(`old-${index}`)),
373
+ ];
374
+
375
+ const merged = mergeRealtimeVoiceToolManifest(current, [
376
+ realtimeTool("open-dashboard"),
377
+ realtimeTool("list-dashboards"),
378
+ ]);
379
+ const names = merged.map((tool) => tool.name);
380
+
381
+ expect(merged).toHaveLength(32);
382
+ expect(names.slice(0, 5)).toEqual(pinned.map((tool) => tool.name));
383
+ expect(names).toContain("open-dashboard");
384
+ expect(names).toContain("list-dashboards");
385
+ expect(names).not.toContain("old-26");
386
+ });
387
+
388
+ it("packs dynamic schemas within the realtime session byte budget", () => {
389
+ const largeTool = (name: string): RealtimeVoiceFunctionTool => ({
390
+ ...realtimeTool(name),
391
+ description: "x".repeat(25_000),
392
+ });
393
+
394
+ const merged = mergeRealtimeVoiceToolManifest(
395
+ [],
396
+ [largeTool("large-1"), largeTool("large-2"), largeTool("large-3")],
397
+ );
398
+
399
+ expect(merged.map((tool) => tool.name)).toEqual(["large-1", "large-2"]);
400
+ });
401
+
402
+ it("waits for session.updated before exposing discovered tools to the model", () => {
403
+ const sent: Record<string, unknown>[] = [];
404
+ const coordinator = createRealtimeVoiceToolManifestCoordinator((event) =>
405
+ sent.push(event),
406
+ );
407
+ coordinator.setSessionTools([
408
+ realtimeTool("navigate"),
409
+ realtimeTool("tool-search"),
410
+ ]);
411
+ coordinator.enqueue({
412
+ callId: "call-search",
413
+ status: "completed",
414
+ output: '{"matches":["open-dashboard"]}',
415
+ expandedTools: [realtimeTool("open-dashboard")],
416
+ });
417
+
418
+ expect(sent.map((event) => event.type)).toEqual(["session.update"]);
419
+ expect(sent[0]?.event_id).toBe("realtime_tool_manifest_1");
420
+ const updateTools = (
421
+ sent[0]?.session as { tools: RealtimeVoiceFunctionTool[] }
422
+ ).tools;
423
+ coordinator.setSessionTools([realtimeTool("navigate")]);
424
+ expect(sent.map((event) => event.type)).toEqual(["session.update"]);
425
+
426
+ coordinator.setSessionTools(updateTools);
427
+ expect(sent.map((event) => event.type)).toEqual([
428
+ "session.update",
429
+ "conversation.item.create",
430
+ "response.create",
431
+ ]);
432
+ const output = JSON.parse(
433
+ String(
434
+ (sent[1]?.item as { output?: unknown } | undefined)?.output ?? "{}",
435
+ ),
436
+ ) as Record<string, unknown>;
437
+ expect(output.output).toBe('{"matches":["open-dashboard"]}');
438
+ expect(output).not.toHaveProperty("expandedTools");
439
+ });
440
+
441
+ it("handles a rejected manifest update without failing the voice session", () => {
442
+ const sent: Record<string, unknown>[] = [];
443
+ const coordinator = createRealtimeVoiceToolManifestCoordinator((event) =>
444
+ sent.push(event),
445
+ );
446
+ coordinator.setSessionTools([realtimeTool("tool-search")]);
447
+ coordinator.enqueue({
448
+ callId: "call-search",
449
+ status: "completed",
450
+ output: "Found open-dashboard",
451
+ expandedTools: [realtimeTool("open-dashboard")],
452
+ });
453
+
454
+ expect(
455
+ coordinator.handleError(
456
+ "realtime_tool_manifest_1",
457
+ "Invalid session tools",
458
+ ),
459
+ ).toBe(true);
460
+ expect(sent.map((event) => event.type)).toEqual([
461
+ "session.update",
462
+ "conversation.item.create",
463
+ "response.create",
464
+ ]);
465
+ const failure = JSON.parse(
466
+ String((sent[1]?.item as { output: string }).output),
467
+ ) as Record<string, unknown>;
468
+ expect(failure.status).toBe("failed");
469
+ expect(failure.output).toContain("Invalid session tools");
470
+ expect(failure.output).toContain("Found open-dashboard");
471
+ expect(coordinator.handleError("some_other_event", "unrelated")).toBe(
472
+ false,
473
+ );
474
+ });
475
+
476
+ it("serializes updates and falls back after a missing confirmation", () => {
477
+ vi.useFakeTimers();
478
+ const sent: Record<string, unknown>[] = [];
479
+ const coordinator = createRealtimeVoiceToolManifestCoordinator(
480
+ (event) => sent.push(event),
481
+ 1_000,
482
+ );
483
+ coordinator.setSessionTools([realtimeTool("tool-search")]);
484
+ coordinator.enqueue({
485
+ callId: "call-1",
486
+ status: "completed",
487
+ output: "first",
488
+ expandedTools: [realtimeTool("first-tool")],
489
+ });
490
+ coordinator.enqueue({
491
+ callId: "call-2",
492
+ status: "completed",
493
+ output: "second",
494
+ expandedTools: [realtimeTool("second-tool")],
495
+ });
496
+
497
+ expect(sent.map((event) => event.type)).toEqual(["session.update"]);
498
+ vi.advanceTimersByTime(1_000);
499
+ expect(sent.map((event) => event.type)).toEqual([
500
+ "session.update",
501
+ "conversation.item.create",
502
+ "response.create",
503
+ "session.update",
504
+ ]);
505
+ expect(
506
+ JSON.parse(String((sent[1]?.item as { output: string }).output)).status,
507
+ ).toBe("failed");
508
+ vi.advanceTimersByTime(1_000);
509
+ expect(sent.map((event) => event.type)).toEqual([
510
+ "session.update",
511
+ "conversation.item.create",
512
+ "response.create",
513
+ "session.update",
514
+ "conversation.item.create",
515
+ "response.create",
516
+ ]);
517
+ expect(
518
+ JSON.parse(String((sent[4]?.item as { output: string }).output)).status,
519
+ ).toBe("failed");
520
+ });
521
+ });
522
+
523
+ describe("extractRealtimeVoiceFunctionCalls", () => {
524
+ it("uses the low-latency completed-arguments event", () => {
525
+ expect(
526
+ extractRealtimeVoiceFunctionCalls({
527
+ type: "response.function_call_arguments.done",
528
+ name: "navigate",
529
+ call_id: "call-1",
530
+ arguments: '{"path":"/inbox"}',
531
+ }),
532
+ ).toEqual([
533
+ {
534
+ name: "navigate",
535
+ callId: "call-1",
536
+ argumentsText: '{"path":"/inbox"}',
537
+ },
538
+ ]);
539
+ });
540
+
541
+ it("falls back to completed function items on response.done", () => {
542
+ expect(
543
+ extractRealtimeVoiceFunctionCalls({
544
+ type: "response.done",
545
+ response: {
546
+ output: [
547
+ { type: "message", role: "assistant" },
548
+ {
549
+ type: "function_call",
550
+ name: "view-screen",
551
+ call_id: "call-2",
552
+ arguments: "{}",
553
+ },
554
+ ],
555
+ },
556
+ }),
557
+ ).toEqual([
558
+ {
559
+ name: "view-screen",
560
+ callId: "call-2",
561
+ argumentsText: "{}",
562
+ },
563
+ ]);
564
+ });
565
+ });
566
+
567
+ describe("extractCompletedRealtimeVoiceTranscript", () => {
568
+ it("accepts completed user and assistant transcripts with stable provider ids", () => {
569
+ expect(
570
+ extractCompletedRealtimeVoiceTranscript({
571
+ type: "conversation.item.input_audio_transcription.completed",
572
+ transcript: " Find the latest report. ",
573
+ item_id: "item-1",
574
+ }),
575
+ ).toEqual({
576
+ role: "user",
577
+ text: "Find the latest report.",
578
+ providerId: "item-1",
579
+ });
580
+
581
+ expect(
582
+ extractCompletedRealtimeVoiceTranscript({
583
+ type: "response.output_audio_transcript.done",
584
+ transcript: "I found it.",
585
+ response_id: "response-1",
586
+ }),
587
+ ).toEqual({
588
+ role: "assistant",
589
+ text: "I found it.",
590
+ providerId: "response-1",
591
+ });
592
+ });
593
+
594
+ it("ignores transcript deltas, unrelated events, and empty completed text", () => {
595
+ expect(
596
+ extractCompletedRealtimeVoiceTranscript({
597
+ type: "response.output_audio_transcript.delta",
598
+ transcript: "partial",
599
+ }),
600
+ ).toBeNull();
601
+ expect(
602
+ extractCompletedRealtimeVoiceTranscript({
603
+ type: "response.output_audio_transcript.done",
604
+ transcript: " ",
605
+ }),
606
+ ).toBeNull();
607
+ });
608
+ });
609
+
610
+ describe("Realtime voice startup and transcript ordering", () => {
611
+ it("requests one brief uncapped greeting when the live session starts", () => {
612
+ expect(createRealtimeVoiceGreetingEvent()).toEqual({
613
+ type: "response.create",
614
+ response: {
615
+ output_modalities: ["audio"],
616
+ instructions:
617
+ 'Say exactly: "How can I help you?" Do not add anything else.',
618
+ },
619
+ });
620
+ });
621
+
622
+ it("starts the greeting exactly once across duplicate session lifecycle events", () => {
623
+ const send = vi.fn();
624
+ const greeting = createRealtimeVoiceGreetingStarter(send);
625
+
626
+ expect(greeting.start()).toBe(true);
627
+ expect(greeting.start()).toBe(false);
628
+ expect(send).toHaveBeenCalledOnce();
629
+
630
+ greeting.reset();
631
+ expect(greeting.start()).toBe(true);
632
+ expect(send).toHaveBeenCalledTimes(2);
633
+ });
634
+
635
+ it("publishes in conversation order when user ASR finishes after the assistant", () => {
636
+ const published: Array<{ role: string; text: string }> = [];
637
+ const sequencer = createRealtimeVoiceTranscriptSequencer((transcript) => {
638
+ published.push(transcript);
639
+ });
640
+
641
+ sequencer.handle({
642
+ type: "conversation.item.added",
643
+ item: { id: "user-1", type: "message", role: "user" },
644
+ });
645
+ sequencer.handle({
646
+ type: "conversation.item.added",
647
+ item: { id: "assistant-1", type: "message", role: "assistant" },
648
+ });
649
+ sequencer.handle({
650
+ type: "response.output_audio_transcript.done",
651
+ item_id: "assistant-1",
652
+ response_id: "response-1",
653
+ transcript: "I can help with that.",
654
+ });
655
+
656
+ expect(published).toEqual([]);
657
+
658
+ sequencer.handle({
659
+ type: "conversation.item.input_audio_transcription.completed",
660
+ item_id: "user-1",
661
+ transcript: "Can you help me?",
662
+ });
663
+
664
+ expect(published).toEqual([
665
+ expect.objectContaining({ role: "user", text: "Can you help me?" }),
666
+ expect.objectContaining({
667
+ role: "assistant",
668
+ text: "I can help with that.",
669
+ }),
670
+ ]);
671
+ });
672
+
673
+ it("does not deadlock later turns when input transcription fails", () => {
674
+ const published: Array<{ role: string; text: string }> = [];
675
+ const sequencer = createRealtimeVoiceTranscriptSequencer((transcript) => {
676
+ published.push(transcript);
677
+ });
678
+
679
+ sequencer.handle({
680
+ type: "conversation.item.created",
681
+ item: { id: "user-1", type: "message", role: "user" },
682
+ });
683
+ sequencer.handle({
684
+ type: "conversation.item.created",
685
+ item: { id: "assistant-1", type: "message", role: "assistant" },
686
+ });
687
+ sequencer.handle({
688
+ type: "response.output_audio_transcript.done",
689
+ item_id: "assistant-1",
690
+ transcript: "Still here.",
691
+ });
692
+ sequencer.handle({
693
+ type: "conversation.item.input_audio_transcription.failed",
694
+ item_id: "user-1",
695
+ });
696
+
697
+ expect(published).toEqual([
698
+ expect.objectContaining({ role: "assistant", text: "Still here." }),
699
+ ]);
700
+ });
701
+
702
+ it("matches legacy completions without item_id to a reserved role slot", () => {
703
+ const published: Array<{ role: string; text: string }> = [];
704
+ const sequencer = createRealtimeVoiceTranscriptSequencer((transcript) => {
705
+ published.push(transcript);
706
+ });
707
+
708
+ sequencer.handle({
709
+ type: "conversation.item.added",
710
+ item: { id: "user-1", type: "message", role: "user" },
711
+ });
712
+ sequencer.handle({
713
+ type: "conversation.item.added",
714
+ item: { id: "assistant-1", type: "message", role: "assistant" },
715
+ });
716
+ sequencer.handle({
717
+ type: "response.output_audio_transcript.done",
718
+ response_id: "response-1",
719
+ transcript: "First answer.",
720
+ });
721
+ sequencer.handle({
722
+ type: "conversation.item.input_audio_transcription.completed",
723
+ item_id: "user-1",
724
+ transcript: "First question.",
725
+ });
726
+ sequencer.handle({
727
+ type: "conversation.item.added",
728
+ item: { id: "assistant-2", type: "message", role: "assistant" },
729
+ });
730
+ sequencer.handle({
731
+ type: "response.output_audio_transcript.done",
732
+ item_id: "assistant-2",
733
+ transcript: "Second answer.",
734
+ });
735
+
736
+ expect(published.map(({ text }) => text)).toEqual([
737
+ "First question.",
738
+ "First answer.",
739
+ "Second answer.",
740
+ ]);
741
+ });
742
+
743
+ it("ignores duplicate completion events and releases interrupted output", () => {
744
+ const published: Array<{ role: string; text: string }> = [];
745
+ const sequencer = createRealtimeVoiceTranscriptSequencer((transcript) => {
746
+ published.push(transcript);
747
+ });
748
+
749
+ sequencer.handle({
750
+ type: "conversation.item.added",
751
+ item: { id: "assistant-1", type: "message", role: "assistant" },
752
+ });
753
+ const completed = {
754
+ type: "response.output_audio_transcript.done",
755
+ item_id: "assistant-1",
756
+ transcript: "Only once.",
757
+ };
758
+ sequencer.handle(completed);
759
+ sequencer.handle(completed);
760
+ sequencer.handle({
761
+ type: "conversation.item.added",
762
+ item: { id: "assistant-interrupted", type: "message", role: "assistant" },
763
+ });
764
+ sequencer.handle({
765
+ type: "response.done",
766
+ response: {
767
+ status: "cancelled",
768
+ output: [{ id: "assistant-interrupted", type: "message" }],
769
+ },
770
+ });
771
+ sequencer.handle({
772
+ type: "conversation.item.added",
773
+ item: { id: "assistant-2", type: "message", role: "assistant" },
774
+ });
775
+ sequencer.handle({
776
+ type: "response.output_audio_transcript.done",
777
+ item_id: "assistant-2",
778
+ transcript: "After interruption.",
779
+ });
780
+
781
+ expect(published.map(({ text }) => text)).toEqual([
782
+ "Only once.",
783
+ "After interruption.",
784
+ ]);
785
+ });
786
+ });
787
+
788
+ describe("shouldRestoreRealtimeVoiceTranscriptThread", () => {
789
+ it("restores the captured transcript when it remains active or chat has no active thread", () => {
790
+ expect(
791
+ shouldRestoreRealtimeVoiceTranscriptThread(
792
+ "voice-thread",
793
+ "voice-thread",
794
+ ),
795
+ ).toBe(true);
796
+ expect(
797
+ shouldRestoreRealtimeVoiceTranscriptThread("voice-thread", undefined),
798
+ ).toBe(true);
799
+ });
800
+
801
+ it("does not restore over a thread selected while voice mode was active", () => {
802
+ expect(
803
+ shouldRestoreRealtimeVoiceTranscriptThread(
804
+ "voice-thread",
805
+ "other-thread",
806
+ ),
807
+ ).toBe(false);
808
+ expect(
809
+ shouldRestoreRealtimeVoiceTranscriptThread(undefined, "other-thread"),
810
+ ).toBe(false);
811
+ });
812
+
813
+ it("recognizes the authoritative missing-provider response", () => {
814
+ expect(isRealtimeVoiceSetupRequiredError({ status: 409 })).toBe(true);
815
+ expect(isRealtimeVoiceSetupRequiredError({ status: 400 })).toBe(false);
816
+ expect(isRealtimeVoiceSetupRequiredError(new Error("offline"))).toBe(false);
817
+ });
818
+ });