@runtypelabs/persona 3.34.1 → 3.35.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/README.md +11 -11
  2. package/dist/animations/glyph-cycle.d.cts +1 -1
  3. package/dist/animations/glyph-cycle.d.ts +1 -1
  4. package/dist/animations/{types-CthJFfNx.d.cts → types-DgYsuwXL.d.cts} +15 -14
  5. package/dist/animations/{types-CthJFfNx.d.ts → types-DgYsuwXL.d.ts} +15 -14
  6. package/dist/animations/wipe.d.cts +1 -1
  7. package/dist/animations/wipe.d.ts +1 -1
  8. package/dist/codegen.cjs +11 -11
  9. package/dist/codegen.js +10 -10
  10. package/dist/index.cjs +46 -44
  11. package/dist/index.cjs.map +1 -1
  12. package/dist/index.d.cts +514 -158
  13. package/dist/index.d.ts +514 -158
  14. package/dist/index.global.js +41 -89
  15. package/dist/index.global.js.map +1 -1
  16. package/dist/index.js +46 -44
  17. package/dist/index.js.map +1 -1
  18. package/dist/install.global.js +1 -1
  19. package/dist/install.global.js.map +1 -1
  20. package/dist/launcher.global.js +2 -2
  21. package/dist/launcher.global.js.map +1 -1
  22. package/dist/markdown-parsers.js +53 -0
  23. package/dist/plugin-kit.d.cts +4 -4
  24. package/dist/plugin-kit.d.ts +4 -4
  25. package/dist/runtype-tts.js +1 -0
  26. package/dist/smart-dom-reader.cjs +1 -1
  27. package/dist/smart-dom-reader.d.cts +352 -103
  28. package/dist/smart-dom-reader.d.ts +352 -103
  29. package/dist/smart-dom-reader.js +1 -1
  30. package/dist/theme-editor-preview.cjs +203 -0
  31. package/dist/theme-editor-preview.d.cts +5748 -0
  32. package/dist/theme-editor-preview.d.ts +5748 -0
  33. package/dist/theme-editor-preview.js +203 -0
  34. package/dist/theme-editor.cjs +14 -131
  35. package/dist/theme-editor.d.cts +362 -574
  36. package/dist/theme-editor.d.ts +362 -574
  37. package/dist/theme-editor.js +14 -131
  38. package/dist/theme-reference.cjs +1 -1
  39. package/dist/theme-reference.d.cts +1 -1
  40. package/dist/theme-reference.d.ts +1 -1
  41. package/dist/theme-reference.js +1 -1
  42. package/dist/voice-worklet-player.cjs +80 -0
  43. package/dist/voice-worklet-player.d.cts +215 -0
  44. package/dist/voice-worklet-player.d.ts +215 -0
  45. package/dist/voice-worklet-player.js +80 -0
  46. package/dist/widget.css +54 -36
  47. package/package.json +16 -2
  48. package/src/animations/glyph-cycle.ts +11 -14
  49. package/src/animations/wipe.ts +1 -1
  50. package/src/ask-user-question-tool.test.ts +1 -1
  51. package/src/ask-user-question-tool.ts +6 -8
  52. package/src/client.test.ts +12 -12
  53. package/src/client.ts +30 -31
  54. package/src/codegen.test.ts +1 -1
  55. package/src/codegen.ts +1 -1
  56. package/src/components/approval-bubble.test.ts +1 -1
  57. package/src/components/approval-bubble.ts +3 -3
  58. package/src/components/artifact-pane.ts +3 -3
  59. package/src/components/ask-user-question-bubble.test.ts +14 -14
  60. package/src/components/ask-user-question-bubble.ts +10 -10
  61. package/src/components/composer-builder.ts +17 -16
  62. package/src/components/composer-parts.test.ts +54 -0
  63. package/src/components/composer-parts.ts +95 -126
  64. package/src/components/demo-carousel.ts +1 -1
  65. package/src/components/event-stream-view.test.ts +7 -7
  66. package/src/components/event-stream-view.ts +5 -5
  67. package/src/components/header-builder.ts +33 -27
  68. package/src/components/header-layouts.ts +1 -1
  69. package/src/components/header-parts.test.ts +90 -0
  70. package/src/components/header-parts.ts +62 -93
  71. package/src/components/launcher.test.ts +11 -0
  72. package/src/components/launcher.ts +1 -1
  73. package/src/components/message-bubble.test.ts +74 -2
  74. package/src/components/message-bubble.ts +25 -11
  75. package/src/components/messages.ts +2 -2
  76. package/src/components/panel.test.ts +1 -1
  77. package/src/components/panel.ts +92 -97
  78. package/src/components/pill-composer-builder.ts +28 -24
  79. package/src/components/reasoning-bubble.ts +2 -2
  80. package/src/components/suggestions.ts +1 -1
  81. package/src/components/tool-bubble.ts +1 -1
  82. package/src/components/widget-view.test.ts +181 -0
  83. package/src/components/widget-view.ts +234 -0
  84. package/src/defaults.ts +54 -51
  85. package/src/generated/runtype-openapi-contract.ts +5 -0
  86. package/src/index-core.ts +30 -4
  87. package/src/index-global.ts +73 -6
  88. package/src/index.ts +7 -1
  89. package/src/install.test.ts +11 -11
  90. package/src/install.ts +28 -18
  91. package/src/launcher-global.ts +25 -14
  92. package/src/markdown-parsers-eager.ts +20 -0
  93. package/src/markdown-parsers-entry.ts +4 -0
  94. package/src/markdown-parsers-loader.ts +49 -0
  95. package/src/plugin-kit.test.ts +1 -1
  96. package/src/plugin-kit.ts +7 -7
  97. package/src/plugins/types.ts +5 -5
  98. package/src/postprocessors.ts +27 -14
  99. package/src/runtime/host-layout.test.ts +1 -1
  100. package/src/runtime/host-layout.ts +4 -5
  101. package/src/runtime/init.ts +2 -2
  102. package/src/session.test.ts +5 -6
  103. package/src/session.ts +333 -227
  104. package/src/session.voice.test.ts +146 -0
  105. package/src/session.webmcp.test.ts +24 -25
  106. package/src/smart-dom-reader.ts +3 -4
  107. package/src/styles/widget.css +54 -36
  108. package/src/suggest-replies-tool.test.ts +3 -3
  109. package/src/suggest-replies-tool.ts +5 -5
  110. package/src/testing/index.ts +1 -1
  111. package/src/theme-editor/color-utils.ts +2 -2
  112. package/src/theme-editor/index.ts +4 -13
  113. package/src/theme-editor/presets.ts +1 -1
  114. package/src/theme-editor/preview-utils.ts +22 -20
  115. package/src/theme-editor/preview.ts +4 -1
  116. package/src/theme-editor/role-mappings.ts +3 -3
  117. package/src/theme-editor/sections.ts +21 -20
  118. package/src/theme-editor/types.ts +1 -1
  119. package/src/theme-editor/webmcp/index.ts +1 -1
  120. package/src/theme-editor/webmcp/summary.ts +1 -2
  121. package/src/theme-editor/webmcp/tools.ts +30 -30
  122. package/src/theme-editor/webmcp/types.ts +2 -2
  123. package/src/theme-editor-preview.ts +10 -0
  124. package/src/theme-reference.ts +26 -26
  125. package/src/types.ts +368 -99
  126. package/src/ui.approval-plugin.test.ts +3 -3
  127. package/src/ui.ask-user-question-plugin.test.ts +9 -9
  128. package/src/ui.component-directive.test.ts +4 -4
  129. package/src/ui.composer-bar.test.ts +10 -11
  130. package/src/ui.composer-keyboard.test.ts +5 -5
  131. package/src/ui.header-icon-color.test.ts +59 -0
  132. package/src/ui.scroll.test.ts +5 -6
  133. package/src/ui.stop-button.test.ts +3 -4
  134. package/src/ui.suggest-replies.test.ts +2 -2
  135. package/src/ui.ts +292 -131
  136. package/src/utils/buttons.ts +1 -1
  137. package/src/utils/code-generators.ts +1 -1
  138. package/src/utils/composer-history.ts +3 -3
  139. package/src/utils/dom-context.test.ts +2 -2
  140. package/src/utils/dom-context.ts +7 -7
  141. package/src/utils/dom.test.ts +67 -0
  142. package/src/utils/dom.ts +75 -0
  143. package/src/utils/dropdown.ts +2 -2
  144. package/src/utils/event-stream-buffer.test.ts +2 -2
  145. package/src/utils/message-fingerprint.test.ts +9 -0
  146. package/src/utils/message-fingerprint.ts +5 -0
  147. package/src/utils/morph.ts +1 -1
  148. package/src/utils/sanitize.ts +32 -16
  149. package/src/utils/sequence-buffer.test.ts +8 -8
  150. package/src/utils/sequence-buffer.ts +4 -4
  151. package/src/utils/smart-dom-adapter.test.ts +1 -1
  152. package/src/utils/smart-dom-adapter.ts +4 -5
  153. package/src/utils/speech-text.test.ts +122 -0
  154. package/src/utils/speech-text.ts +101 -0
  155. package/src/utils/stream-animation.test.ts +3 -3
  156. package/src/utils/stream-animation.ts +6 -6
  157. package/src/utils/theme.test.ts +1 -1
  158. package/src/utils/throughput-tracker.test.ts +11 -11
  159. package/src/utils/throughput-tracker.ts +9 -9
  160. package/src/utils/tokens.ts +13 -13
  161. package/src/utils/virtual-scroller.ts +1 -1
  162. package/src/vendor/smart-dom-reader/README.md +4 -4
  163. package/src/vendor/smart-dom-reader/index.d.ts +1 -1
  164. package/src/vendor/smart-dom-reader/index.js +2 -2
  165. package/src/version.ts +1 -1
  166. package/src/voice/audio-playback-manager.test.ts +159 -0
  167. package/src/voice/audio-playback-manager.ts +145 -30
  168. package/src/voice/browser-speech-engine.ts +130 -0
  169. package/src/voice/fallback-speech-engine.ts +89 -0
  170. package/src/voice/index.ts +21 -0
  171. package/src/voice/read-aloud-controller.test.ts +143 -0
  172. package/src/voice/read-aloud-controller.ts +136 -0
  173. package/src/voice/runtype-speech-engine.test.ts +280 -0
  174. package/src/voice/runtype-speech-engine.ts +211 -0
  175. package/src/voice/runtype-tts-entry.ts +10 -0
  176. package/src/voice/runtype-tts-loader.test.ts +47 -0
  177. package/src/voice/runtype-tts-loader.ts +42 -0
  178. package/src/voice/runtype-voice-provider.ts +332 -537
  179. package/src/voice/voice-factory.ts +26 -4
  180. package/src/voice/voice.test.ts +321 -7
  181. package/src/voice/worklet-playback-engine.ts +272 -0
  182. package/src/voice-worklet-player.ts +39 -0
  183. package/src/webmcp-bridge.test.ts +6 -6
  184. package/src/webmcp-bridge.ts +24 -25
  185. package/src/webmcp-polyfill.ts +1 -1
  186. package/src/voice/voice-activity-detector.ts +0 -90
@@ -19,9 +19,26 @@ export function createVoiceProvider(config: VoiceConfig): VoiceProvider {
19
19
  }
20
20
  return new BrowserVoiceProvider(config.browser || {});
21
21
 
22
- case 'custom':
23
- throw new Error('Custom voice providers not yet implemented');
24
-
22
+ case 'custom': {
23
+ // Bring-your-own provider: `custom` is either a ready VoiceProvider
24
+ // instance or a `() => VoiceProvider` factory (deferred construction, so
25
+ // resources like a WebSocket or AudioContext are only created when voice
26
+ // is actually set up). Resolve and sanity-check the shape.
27
+ const custom = config.custom;
28
+ if (!custom) {
29
+ throw new Error(
30
+ 'Custom voice provider requires a `custom` provider instance or factory'
31
+ );
32
+ }
33
+ const provider = typeof custom === 'function' ? custom() : custom;
34
+ if (!provider || typeof provider.startListening !== 'function') {
35
+ throw new Error(
36
+ 'Custom voice provider `custom` must be a VoiceProvider (or a factory returning one)'
37
+ );
38
+ }
39
+ return provider;
40
+ }
41
+
25
42
  default:
26
43
  throw new Error(`Unknown voice provider type: ${config.type}`);
27
44
  }
@@ -29,11 +46,16 @@ export function createVoiceProvider(config: VoiceConfig): VoiceProvider {
29
46
 
30
47
  // Auto-select the best available provider
31
48
  export function createBestAvailableVoiceProvider(config?: Partial<VoiceConfig>): VoiceProvider {
49
+ // Honor an explicit bring-your-own provider before any built-in.
50
+ if (config?.type === 'custom' && config.custom) {
51
+ return createVoiceProvider({ type: 'custom', custom: config.custom });
52
+ }
53
+
32
54
  // Prefer Runtype if configured
33
55
  if (config?.type === 'runtype' && config.runtype) {
34
56
  return createVoiceProvider({ type: 'runtype', runtype: config.runtype });
35
57
  }
36
-
58
+
37
59
  // Fall back to browser if supported
38
60
  if (BrowserVoiceProvider.isSupported()) {
39
61
  return createVoiceProvider({
@@ -1,5 +1,5 @@
1
1
  // Voice SDK Tests
2
- import { describe, it, expect, beforeAll, afterAll } from 'vitest';
2
+ import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
3
3
  import type { VoiceConfig } from '../types';
4
4
  import { RuntypeVoiceProvider } from './runtype-voice-provider';
5
5
  import { BrowserVoiceProvider } from './browser-voice-provider';
@@ -32,6 +32,20 @@ function mockBrowserSupport(supported: boolean) {
32
32
  }
33
33
  }
34
34
 
35
+ /** Minimal VoiceProvider-shaped object for the bring-your-own (`custom`) tests. */
36
+ function makeFakeProvider() {
37
+ return {
38
+ type: 'custom' as const,
39
+ connect: async () => {},
40
+ disconnect: async () => {},
41
+ startListening: async () => {},
42
+ stopListening: async () => {},
43
+ onResult: () => {},
44
+ onError: () => {},
45
+ onStatusChange: () => {},
46
+ };
47
+ }
48
+
35
49
  describe('BrowserVoiceProvider', () => {
36
50
  it('should check browser support', () => {
37
51
  // Test supported
@@ -86,15 +100,43 @@ describe('Voice Factory', () => {
86
100
  expect(() => createVoiceProvider(config)).toThrow('Browser speech recognition not supported');
87
101
  });
88
102
 
89
- it('should throw error for custom provider', () => {
103
+ it('should throw when a custom provider is configured without `custom`', () => {
90
104
  const config: VoiceConfig = {
105
+ type: 'custom'
106
+ };
107
+
108
+ expect(() => createVoiceProvider(config)).toThrow('requires a `custom` provider');
109
+ });
110
+
111
+ it('should return a bring-your-own custom provider instance', () => {
112
+ const byo = makeFakeProvider();
113
+ const provider = createVoiceProvider({ type: 'custom', custom: byo });
114
+ expect(provider).toBe(byo);
115
+ });
116
+
117
+ it('should resolve a custom provider factory', () => {
118
+ const byo = makeFakeProvider();
119
+ let calls = 0;
120
+ const provider = createVoiceProvider({
91
121
  type: 'custom',
92
- custom: {}
122
+ custom: () => {
123
+ calls += 1;
124
+ return byo;
125
+ },
126
+ });
127
+ expect(provider).toBe(byo);
128
+ expect(calls).toBe(1);
129
+ });
130
+
131
+ it('should throw when a custom factory returns a non-provider', () => {
132
+ const config = {
133
+ type: 'custom' as const,
134
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
135
+ custom: (() => ({})) as any,
93
136
  };
94
-
95
- expect(() => createVoiceProvider(config)).toThrow('Custom voice providers not yet implemented');
137
+ expect(() => createVoiceProvider(config)).toThrow('must be a VoiceProvider');
96
138
  });
97
-
139
+
98
140
  it('should throw error for unknown provider type', () => {
99
141
  const config = {
100
142
  type: 'unknown' as any
@@ -163,7 +205,279 @@ describe('Voice Support Check', () => {
163
205
  it('should return false when no voice support available', () => {
164
206
  // Mock no browser support
165
207
  mockBrowserSupport(false);
166
-
208
+
167
209
  expect(isVoiceSupported()).toBe(false);
168
210
  });
211
+ });
212
+
213
+ // ---------------------------------------------------------------------------
214
+ // Realtime streaming protocol (RuntypeVoiceProvider rewrite)
215
+ // ---------------------------------------------------------------------------
216
+
217
+ const WS_OPEN = 1;
218
+
219
+ class MockWebSocket {
220
+ static OPEN = WS_OPEN;
221
+ static instances: MockWebSocket[] = [];
222
+
223
+ url: string;
224
+ protocols: string | string[] | undefined;
225
+ binaryType = '';
226
+ readyState = 0;
227
+ sent: ArrayBuffer[] = [];
228
+ closeCalls: Array<{ code?: number; reason?: string }> = [];
229
+ onopen: (() => void) | null = null;
230
+ onmessage: ((ev: { data: unknown }) => void) | null = null;
231
+ onclose: ((ev: { code: number; reason?: string; wasClean: boolean }) => void) | null = null;
232
+ onerror: ((ev: unknown) => void) | null = null;
233
+
234
+ constructor(url: string, protocols?: string | string[]) {
235
+ this.url = url;
236
+ this.protocols = protocols;
237
+ MockWebSocket.instances.push(this);
238
+ }
239
+
240
+ send(data: ArrayBuffer) {
241
+ this.sent.push(data);
242
+ }
243
+
244
+ close(code?: number, reason?: string) {
245
+ this.readyState = 3;
246
+ this.closeCalls.push({ code, reason });
247
+ this.onclose?.({ code: code ?? 1000, reason, wasClean: (code ?? 1000) === 1000 });
248
+ }
249
+
250
+ // --- test triggers ---
251
+ triggerOpen() {
252
+ this.readyState = WS_OPEN;
253
+ this.onopen?.();
254
+ }
255
+ triggerMessage(data: unknown) {
256
+ this.onmessage?.({ data });
257
+ }
258
+ triggerClose(code: number, reason?: string) {
259
+ this.readyState = 3;
260
+ this.onclose?.({ code, reason, wasClean: code === 1000 });
261
+ }
262
+ triggerError() {
263
+ this.onerror?.({});
264
+ }
265
+ }
266
+
267
+ class MockAudioContext {
268
+ state = 'running';
269
+ destination = {};
270
+ sampleRate: number;
271
+ constructor(opts?: { sampleRate?: number }) {
272
+ this.sampleRate = opts?.sampleRate ?? 44100;
273
+ }
274
+ async resume() {
275
+ this.state = 'running';
276
+ }
277
+ async close() {
278
+ this.state = 'closed';
279
+ }
280
+ createMediaStreamSource() {
281
+ return { connect() {}, disconnect() {} };
282
+ }
283
+ createScriptProcessor() {
284
+ return { connect() {}, disconnect() {}, onaudioprocess: null as unknown };
285
+ }
286
+ }
287
+
288
+ function makeStream() {
289
+ const track = { stopped: false, stop() { track.stopped = true; } };
290
+ return { stream: { getTracks: () => [track] }, track };
291
+ }
292
+
293
+ function makeFakeEngine() {
294
+ const engine = {
295
+ enqueued: [] as Uint8Array[],
296
+ streamEnded: false,
297
+ flushed: false,
298
+ destroyed: false,
299
+ finishedCb: null as null | (() => void),
300
+ enqueue(p: Uint8Array) { engine.enqueued.push(p); },
301
+ markStreamEnd() { engine.streamEnded = true; },
302
+ flush() { engine.flushed = true; },
303
+ onFinished(cb: () => void) { engine.finishedCb = cb; },
304
+ destroy() { engine.destroyed = true; },
305
+ };
306
+ return engine;
307
+ }
308
+
309
+ /** Build a WAV-wrapped frame: 44-byte RIFF header + the given PCM bytes. */
310
+ function makeWavFrame(pcmBytes: number[]): ArrayBuffer {
311
+ const buf = new ArrayBuffer(44 + pcmBytes.length);
312
+ const view = new DataView(buf);
313
+ view.setUint32(0, 0x52494646, false); // "RIFF"
314
+ new Uint8Array(buf).set(pcmBytes, 44);
315
+ return buf;
316
+ }
317
+
318
+ describe('RuntypeVoiceProvider (realtime streaming)', () => {
319
+ let getUserMedia: ReturnType<typeof vi.fn>;
320
+ let currentStream: ReturnType<typeof makeStream>;
321
+
322
+ beforeEach(() => {
323
+ MockWebSocket.instances = [];
324
+ currentStream = makeStream();
325
+ getUserMedia = vi.fn(async () => currentStream.stream);
326
+
327
+ vi.stubGlobal('WebSocket', MockWebSocket);
328
+ vi.stubGlobal('navigator', { mediaDevices: { getUserMedia } });
329
+ (globalThis as any).window.AudioContext = MockAudioContext;
330
+ (globalThis as any).window.webkitAudioContext = MockAudioContext;
331
+ (globalThis as any).window.location = { protocol: 'https:' };
332
+ });
333
+
334
+ afterEach(() => {
335
+ vi.unstubAllGlobals();
336
+ delete (globalThis as any).window.AudioContext;
337
+ delete (globalThis as any).window.webkitAudioContext;
338
+ delete (globalThis as any).window.location;
339
+ });
340
+
341
+ const baseConfig = () => ({
342
+ agentId: 'a1',
343
+ clientToken: 'ct_secret',
344
+ host: 'https://api.example.com',
345
+ });
346
+
347
+ const lastWs = () => MockWebSocket.instances[MockWebSocket.instances.length - 1];
348
+
349
+ it('connects to /voice with subprotocol auth and arraybuffer (no token in URL)', async () => {
350
+ const provider = new RuntypeVoiceProvider(baseConfig());
351
+ await provider.startListening();
352
+
353
+ const ws = lastWs();
354
+ expect(ws.url).toBe('wss://api.example.com/ws/agents/a1/voice');
355
+ expect(ws.protocols).toEqual(['runtype.bearer', 'ct_secret']);
356
+ expect(ws.binaryType).toBe('arraybuffer');
357
+ expect(ws.url).not.toContain('token=');
358
+ expect(ws.url).not.toContain('ct_secret');
359
+ });
360
+
361
+ it.each([
362
+ ['https://api.example.com', 'wss://api.example.com'],
363
+ ['http://localhost:8787', 'ws://localhost:8787'],
364
+ ['wss://api.example.com', 'wss://api.example.com'],
365
+ ['api.example.com', 'wss://api.example.com'], // bare host → window.location.protocol
366
+ ])('derives ws base %s -> %s', async (host, expected) => {
367
+ const provider = new RuntypeVoiceProvider({ ...baseConfig(), host });
368
+ await provider.startListening();
369
+ expect(lastWs().url).toBe(`${expected}/ws/agents/a1/voice`);
370
+ });
371
+
372
+ it('drives status + onTranscript from control frames', async () => {
373
+ const statuses: string[] = [];
374
+ const transcripts: Array<[string, string, boolean]> = [];
375
+ const provider = new RuntypeVoiceProvider(baseConfig());
376
+ provider.onStatusChange((s) => statuses.push(s));
377
+ provider.onTranscript((role, text, isFinal) => transcripts.push([role, text, isFinal]));
378
+
379
+ await provider.startListening();
380
+ lastWs().triggerOpen();
381
+ lastWs().triggerMessage(JSON.stringify({ type: 'transcript_interim', text: 'hel' }));
382
+ lastWs().triggerMessage(JSON.stringify({ type: 'transcript_final', role: 'user', text: 'hello' }));
383
+ lastWs().triggerMessage(JSON.stringify({ type: 'transcript_final', role: 'assistant', text: 'hi there' }));
384
+
385
+ expect(transcripts).toEqual([
386
+ ['user', 'hel', false],
387
+ ['user', 'hello', true],
388
+ ['assistant', 'hi there', true],
389
+ ]);
390
+ expect(statuses).toContain('listening'); // ws open + interim
391
+ expect(statuses).toContain('processing'); // user final
392
+ expect(statuses).toContain('speaking'); // assistant final
393
+ });
394
+
395
+ it('strips the WAV header, enqueues raw PCM, and drains on audio_end', async () => {
396
+ const engine = makeFakeEngine();
397
+ const provider = new RuntypeVoiceProvider({ ...baseConfig(), createPlaybackEngine: () => engine });
398
+ await provider.startListening();
399
+ lastWs().triggerOpen();
400
+
401
+ lastWs().triggerMessage(makeWavFrame([1, 2, 3, 4]));
402
+ expect(engine.enqueued).toHaveLength(1);
403
+ expect(Array.from(engine.enqueued[0])).toEqual([1, 2, 3, 4]); // 44-byte header stripped
404
+
405
+ lastWs().triggerMessage(JSON.stringify({ type: 'audio_end' }));
406
+ expect(engine.streamEnded).toBe(true);
407
+ });
408
+
409
+ it('treats a non-WAV binary frame as raw PCM', async () => {
410
+ const engine = makeFakeEngine();
411
+ const provider = new RuntypeVoiceProvider({ ...baseConfig(), createPlaybackEngine: () => engine });
412
+ await provider.startListening();
413
+ lastWs().triggerOpen();
414
+
415
+ const raw = new Uint8Array([9, 8, 7, 6]).buffer;
416
+ lastWs().triggerMessage(raw);
417
+ expect(Array.from(engine.enqueued[0])).toEqual([9, 8, 7, 6]);
418
+ });
419
+
420
+ it('emits onMetrics with camelCase from the snake_case frame', async () => {
421
+ const metrics: unknown[] = [];
422
+ const provider = new RuntypeVoiceProvider(baseConfig());
423
+ provider.onMetrics((m) => metrics.push(m));
424
+ await provider.startListening();
425
+ lastWs().triggerOpen();
426
+ // Raw JSON string: the wire frame is snake_case (decoded to camelCase by
427
+ // the provider); a literal avoids the no-snake_case-property lint rule.
428
+ lastWs().triggerMessage(
429
+ '{"type":"metrics","llm_ms":120,"tts_ms":80,"first_audio_ms":200,"total_ms":400}',
430
+ );
431
+ expect(metrics).toEqual([{ llmMs: 120, ttsMs: 80, firstAudioMs: 200, totalMs: 400 }]);
432
+ });
433
+
434
+ it('reports barge-in semantics while the call is live', async () => {
435
+ const provider = new RuntypeVoiceProvider(baseConfig());
436
+ expect(provider.getInterruptionMode()).toBe('barge-in');
437
+ expect(provider.isBargeInActive()).toBe(false);
438
+ await provider.startListening();
439
+ expect(provider.isBargeInActive()).toBe(true);
440
+ await provider.stopListening();
441
+ expect(provider.isBargeInActive()).toBe(false);
442
+ });
443
+
444
+ it('hangs up cleanly and drops late frames after teardown', async () => {
445
+ const engine = makeFakeEngine();
446
+ const provider = new RuntypeVoiceProvider({ ...baseConfig(), createPlaybackEngine: () => engine });
447
+ await provider.startListening();
448
+ lastWs().triggerOpen();
449
+ const ws = lastWs();
450
+
451
+ await provider.stopListening();
452
+
453
+ expect(ws.closeCalls).toEqual([{ code: 1000, reason: 'client ended call' }]);
454
+ expect(currentStream.track.stopped).toBe(true);
455
+ expect(engine.destroyed).toBe(true);
456
+
457
+ // A frame arriving after teardown is dropped by the generation guard.
458
+ const before = engine.enqueued.length;
459
+ ws.triggerMessage(makeWavFrame([1, 2]));
460
+ expect(engine.enqueued.length).toBe(before);
461
+ });
462
+
463
+ it('startListening is idempotent while a call is live', async () => {
464
+ const provider = new RuntypeVoiceProvider(baseConfig());
465
+ await provider.startListening();
466
+ await provider.startListening();
467
+ expect(MockWebSocket.instances).toHaveLength(1);
468
+ expect(getUserMedia).toHaveBeenCalledTimes(1);
469
+ });
470
+
471
+ it('returns to listening when playback drains (call stays open)', async () => {
472
+ const engine = makeFakeEngine();
473
+ const statuses: string[] = [];
474
+ const provider = new RuntypeVoiceProvider({ ...baseConfig(), createPlaybackEngine: () => engine });
475
+ provider.onStatusChange((s) => statuses.push(s));
476
+ await provider.startListening();
477
+ lastWs().triggerOpen();
478
+ lastWs().triggerMessage(makeWavFrame([1, 2])); // → speaking
479
+ statuses.length = 0;
480
+ engine.finishedCb?.(); // playback drained
481
+ expect(statuses).toEqual(['listening']);
482
+ });
169
483
  });
@@ -0,0 +1,272 @@
1
+ // Jitter-buffered AudioWorklet PCM player.
2
+ //
3
+ // A reusable playback primitive for raw PCM16 / 24 kHz / mono streams. A
4
+ // worklet-side sample queue is drained continuously by the audio thread, with a
5
+ // configurable prebuffer "waterline" that absorbs network jitter. On underrun it
6
+ // goes silent and re-buffers (the audio thread outputs pre-zeroed silence)
7
+ // rather than snapping a schedule clock — so a late chunk produces a clean pause,
8
+ // not a click. This is the production-grade alternative to hand-scheduling
9
+ // AudioBufferSourceNodes (`AudioPlaybackManager`), which is simpler but clicks
10
+ // under jitter.
11
+ //
12
+ // Two consumers:
13
+ // - The realtime `runtype` voice provider, via `createWorkletPlaybackEngine`
14
+ // (injected through `voiceRecognition.provider.runtype.createPlaybackEngine`).
15
+ // - Any hosted `SpeechEngine` (e.g. server/ElevenLabs/OpenAI TTS), via
16
+ // `createPcmStreamPlayer({ prebufferMs })` — same engine, plus pause/resume
17
+ // and a tunable prebuffer for bursty HTTP-streamed audio.
18
+ //
19
+ // Shipped from a separate subpath (`@runtypelabs/persona/voice-worklet-player`)
20
+ // so the worklet source stays out of the main bundle.
21
+
22
+ import type { PcmStreamPlayer, VoicePlaybackEngine } from "../types";
23
+
24
+ const PLAYBACK_SAMPLE_RATE = 24000;
25
+ // Default prebuffer: ~150ms. Enough for realtime, server-paced audio; HTTP-pulled
26
+ // TTS is burstier and should pass a larger `prebufferMs`.
27
+ const DEFAULT_PREBUFFER_MS = 150;
28
+
29
+ // The worklet reads its waterline from `processorOptions.waterlineSamples`, so a
30
+ // single registered processor serves any prebuffer size.
31
+ const WORKLET_SOURCE = `
32
+ class PersonaPcmPlayerProcessor extends AudioWorkletProcessor {
33
+ constructor(options) {
34
+ super()
35
+ const opts = (options && options.processorOptions) || {}
36
+ this.waterline = opts.waterlineSamples > 0 ? opts.waterlineSamples : 3600
37
+ this.chunks = []
38
+ this.readOffset = 0
39
+ this.buffered = 0
40
+ this.waiting = true
41
+ // 'drained' must mean "the reply finished playing", not "momentary
42
+ // underrun": a jitter gap empties the buffer mid-reply too, and firing
43
+ // there would flap the UI status and race the audio_end handler. Only
44
+ // report drained once eos has been signalled.
45
+ this.eosSeen = false
46
+ // Fire 'started' once, when the prebuffer first releases into playback, so a
47
+ // consumer can flip UI from loading→playing only when audio is truly audible.
48
+ // A mid-reply underrun re-buffers (waiting=true) but must NOT re-signal.
49
+ this.startedSignaled = false
50
+ this.port.onmessage = (e) => {
51
+ const msg = e.data
52
+ if (msg.type === 'push') {
53
+ this.eosSeen = false
54
+ this.chunks.push(msg.samples)
55
+ this.buffered += msg.samples.length
56
+ if (this.waiting && this.buffered >= this.waterline) {
57
+ this.waiting = false
58
+ this.signalStarted()
59
+ }
60
+ } else if (msg.type === 'eos') {
61
+ this.eosSeen = true
62
+ if (this.waiting && this.buffered > 0) {
63
+ this.waiting = false
64
+ this.signalStarted()
65
+ }
66
+ if (this.buffered === 0) {
67
+ this.eosSeen = false
68
+ this.port.postMessage({ type: 'drained' })
69
+ }
70
+ } else if (msg.type === 'clear') {
71
+ this.chunks = []
72
+ this.readOffset = 0
73
+ this.buffered = 0
74
+ this.waiting = true
75
+ this.eosSeen = false
76
+ this.startedSignaled = false
77
+ }
78
+ }
79
+ }
80
+ signalStarted() {
81
+ if (!this.startedSignaled) {
82
+ this.startedSignaled = true
83
+ this.port.postMessage({ type: 'started' })
84
+ }
85
+ }
86
+ process(inputs, outputs) {
87
+ const out = outputs[0][0]
88
+ if (!out || this.waiting) return true // outputs are pre-zeroed: silence
89
+ let i = 0
90
+ while (i < out.length && this.buffered > 0) {
91
+ const chunk = this.chunks[0]
92
+ out[i++] = chunk[this.readOffset++]
93
+ this.buffered--
94
+ if (this.readOffset >= chunk.length) {
95
+ this.chunks.shift()
96
+ this.readOffset = 0
97
+ }
98
+ }
99
+ if (this.buffered === 0) {
100
+ this.waiting = true // mid-reply underrun: re-buffer silently
101
+ if (this.eosSeen) {
102
+ this.eosSeen = false
103
+ this.port.postMessage({ type: 'drained' })
104
+ }
105
+ }
106
+ return true
107
+ }
108
+ }
109
+ registerProcessor('persona-pcm-player', PersonaPcmPlayerProcessor)
110
+ `;
111
+
112
+ /** Convert raw PCM16 LE mono to Float32 samples in [-1, 1]. */
113
+ function pcm16ToFloat32(pcm: Uint8Array): Float32Array {
114
+ const count = pcm.length >> 1;
115
+ const out = new Float32Array(count);
116
+ const view = new DataView(pcm.buffer, pcm.byteOffset, pcm.byteLength);
117
+ for (let i = 0; i < count; i++) {
118
+ out[i] = view.getInt16(i * 2, true) / 0x8000;
119
+ }
120
+ return out;
121
+ }
122
+
123
+ export interface PcmStreamPlayerOptions {
124
+ /**
125
+ * Audio (ms) to buffer before the first sample, and to re-buffer after an
126
+ * underrun. Higher = smoother on bursty/jittery streams, at the cost of a
127
+ * slightly later first sound (latency ↔ smoothness). Default 150 — good for
128
+ * realtime, server-paced audio; HTTP-pulled TTS often wants ~400–600.
129
+ */
130
+ prebufferMs?: number;
131
+ }
132
+
133
+ /**
134
+ * Create a jitter-buffered AudioWorklet PCM player with pause/resume.
135
+ *
136
+ * Feed it raw PCM16 / 24 kHz / mono via `enqueue()`; it handles prebuffering,
137
+ * gapless playback, and graceful underrun. Reuse it inside a hosted
138
+ * {@link SpeechEngine} so streamed TTS plays smoothly:
139
+ *
140
+ * @example
141
+ * import { createPcmStreamPlayer } from '@runtypelabs/persona/voice-worklet-player'
142
+ * const player = await createPcmStreamPlayer({ prebufferMs: 500 })
143
+ * // for each streamed chunk: player.enqueue(pcmChunk)
144
+ * player.markStreamEnd()
145
+ */
146
+ export async function createPcmStreamPlayer(
147
+ options: PcmStreamPlayerOptions = {},
148
+ ): Promise<PcmStreamPlayer> {
149
+ const prebufferMs = options.prebufferMs ?? DEFAULT_PREBUFFER_MS;
150
+ const waterlineSamples = Math.max(
151
+ 1,
152
+ Math.round((PLAYBACK_SAMPLE_RATE * prebufferMs) / 1000),
153
+ );
154
+
155
+ const AudioCtx =
156
+ (window as any).AudioContext || (window as any).webkitAudioContext;
157
+ // Match the context rate to the PCM rate so the worklet maps samples 1:1;
158
+ // the browser resamples to the hardware rate at the output.
159
+ const context: AudioContext = new AudioCtx({ sampleRate: PLAYBACK_SAMPLE_RATE });
160
+ if (context.state === "suspended") {
161
+ await context.resume().catch(() => {});
162
+ }
163
+
164
+ const moduleUrl = URL.createObjectURL(
165
+ new Blob([WORKLET_SOURCE], { type: "application/javascript" }),
166
+ );
167
+ try {
168
+ await context.audioWorklet.addModule(moduleUrl);
169
+ } catch (err) {
170
+ context.close().catch(() => {});
171
+ throw err;
172
+ } finally {
173
+ URL.revokeObjectURL(moduleUrl);
174
+ }
175
+
176
+ const node = new AudioWorkletNode(context, "persona-pcm-player", {
177
+ numberOfInputs: 0,
178
+ numberOfOutputs: 1,
179
+ outputChannelCount: [1],
180
+ processorOptions: { waterlineSamples },
181
+ });
182
+ node.connect(context.destination);
183
+
184
+ let finishedCallbacks: (() => void)[] = [];
185
+ let startedCallbacks: (() => void)[] = [];
186
+ // Carry an orphaned odd byte across chunks so a sample split at a frame
187
+ // boundary isn't dropped.
188
+ let remainder: Uint8Array | null = null;
189
+
190
+ node.port.onmessage = (e: MessageEvent) => {
191
+ const type = e.data?.type;
192
+ if (type === "started") {
193
+ const cbs = startedCallbacks.slice();
194
+ startedCallbacks = [];
195
+ cbs.forEach((cb) => cb());
196
+ } else if (type === "drained") {
197
+ const cbs = finishedCallbacks.slice();
198
+ finishedCallbacks = [];
199
+ cbs.forEach((cb) => cb());
200
+ }
201
+ };
202
+
203
+ return {
204
+ enqueue(pcm: Uint8Array) {
205
+ let data = pcm;
206
+ if (remainder) {
207
+ const merged = new Uint8Array(remainder.length + pcm.length);
208
+ merged.set(remainder);
209
+ merged.set(pcm, remainder.length);
210
+ data = merged;
211
+ remainder = null;
212
+ }
213
+ if (data.length % 2 !== 0) {
214
+ remainder = new Uint8Array([data[data.length - 1]]);
215
+ data = data.subarray(0, data.length - 1);
216
+ }
217
+ if (data.length === 0) return;
218
+ const samples = pcm16ToFloat32(data);
219
+ if (samples.length === 0) return;
220
+ node.port.postMessage({ type: "push", samples }, [samples.buffer]);
221
+ },
222
+ markStreamEnd() {
223
+ node.port.postMessage({ type: "eos" });
224
+ },
225
+ flush() {
226
+ remainder = null;
227
+ finishedCallbacks = [];
228
+ startedCallbacks = [];
229
+ node.port.postMessage({ type: "clear" });
230
+ },
231
+ onFinished(callback: () => void) {
232
+ finishedCallbacks.push(callback);
233
+ },
234
+ onStarted(callback: () => void) {
235
+ startedCallbacks.push(callback);
236
+ },
237
+ pause() {
238
+ // Suspending the context freezes the audio clock; queued samples stay put
239
+ // and resume() continues exactly where playback left off.
240
+ if (context.state === "running") void context.suspend();
241
+ },
242
+ resume() {
243
+ if (context.state === "suspended") void context.resume();
244
+ },
245
+ destroy() {
246
+ node.port.onmessage = null;
247
+ try {
248
+ node.disconnect();
249
+ } catch {
250
+ // ignore
251
+ }
252
+ return context.close().catch(() => {});
253
+ },
254
+ };
255
+ }
256
+
257
+ /**
258
+ * Realtime-named alias of {@link createPcmStreamPlayer} (default prebuffer),
259
+ * typed as a plain {@link VoicePlaybackEngine}. Pass it to the realtime voice
260
+ * provider's `createPlaybackEngine`:
261
+ *
262
+ * @example
263
+ * import { createWorkletPlaybackEngine } from '@runtypelabs/persona/voice-worklet-player'
264
+ *
265
+ * initAgentWidget({ config: { voiceRecognition: { enabled: true, provider: {
266
+ * type: 'runtype',
267
+ * runtype: { agentId, createPlaybackEngine: createWorkletPlaybackEngine },
268
+ * } } } })
269
+ */
270
+ export function createWorkletPlaybackEngine(): Promise<VoicePlaybackEngine> {
271
+ return createPcmStreamPlayer();
272
+ }