@runtypelabs/persona 3.34.0 → 3.35.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/README.md +11 -11
  2. package/dist/animations/glyph-cycle.d.cts +1 -1
  3. package/dist/animations/glyph-cycle.d.ts +1 -1
  4. package/dist/animations/{types-CthJFfNx.d.cts → types-DgYsuwXL.d.cts} +15 -14
  5. package/dist/animations/{types-CthJFfNx.d.ts → types-DgYsuwXL.d.ts} +15 -14
  6. package/dist/animations/wipe.d.cts +1 -1
  7. package/dist/animations/wipe.d.ts +1 -1
  8. package/dist/codegen.cjs +11 -11
  9. package/dist/codegen.js +10 -10
  10. package/dist/index.cjs +46 -44
  11. package/dist/index.cjs.map +1 -1
  12. package/dist/index.d.cts +514 -158
  13. package/dist/index.d.ts +514 -158
  14. package/dist/index.global.js +41 -89
  15. package/dist/index.global.js.map +1 -1
  16. package/dist/index.js +46 -44
  17. package/dist/index.js.map +1 -1
  18. package/dist/install.global.js +1 -1
  19. package/dist/install.global.js.map +1 -1
  20. package/dist/launcher.global.js +2 -2
  21. package/dist/launcher.global.js.map +1 -1
  22. package/dist/markdown-parsers.js +53 -0
  23. package/dist/plugin-kit.d.cts +4 -4
  24. package/dist/plugin-kit.d.ts +4 -4
  25. package/dist/runtype-tts.js +1 -0
  26. package/dist/smart-dom-reader.cjs +1 -1
  27. package/dist/smart-dom-reader.d.cts +352 -103
  28. package/dist/smart-dom-reader.d.ts +352 -103
  29. package/dist/smart-dom-reader.js +1 -1
  30. package/dist/theme-editor-preview.cjs +203 -0
  31. package/dist/theme-editor-preview.d.cts +5748 -0
  32. package/dist/theme-editor-preview.d.ts +5748 -0
  33. package/dist/theme-editor-preview.js +203 -0
  34. package/dist/theme-editor.cjs +14 -131
  35. package/dist/theme-editor.d.cts +362 -574
  36. package/dist/theme-editor.d.ts +362 -574
  37. package/dist/theme-editor.js +14 -131
  38. package/dist/theme-reference.cjs +1 -1
  39. package/dist/theme-reference.d.cts +1 -1
  40. package/dist/theme-reference.d.ts +1 -1
  41. package/dist/theme-reference.js +1 -1
  42. package/dist/voice-worklet-player.cjs +80 -0
  43. package/dist/voice-worklet-player.d.cts +215 -0
  44. package/dist/voice-worklet-player.d.ts +215 -0
  45. package/dist/voice-worklet-player.js +80 -0
  46. package/dist/widget.css +54 -36
  47. package/package.json +16 -2
  48. package/src/animations/glyph-cycle.ts +11 -14
  49. package/src/animations/wipe.ts +1 -1
  50. package/src/ask-user-question-tool.test.ts +1 -1
  51. package/src/ask-user-question-tool.ts +6 -8
  52. package/src/client.test.ts +67 -12
  53. package/src/client.ts +40 -32
  54. package/src/codegen.test.ts +1 -1
  55. package/src/codegen.ts +1 -1
  56. package/src/components/approval-bubble.test.ts +1 -1
  57. package/src/components/approval-bubble.ts +3 -3
  58. package/src/components/artifact-pane.ts +3 -3
  59. package/src/components/ask-user-question-bubble.test.ts +14 -14
  60. package/src/components/ask-user-question-bubble.ts +10 -10
  61. package/src/components/composer-builder.ts +17 -16
  62. package/src/components/composer-parts.test.ts +54 -0
  63. package/src/components/composer-parts.ts +95 -126
  64. package/src/components/demo-carousel.ts +1 -1
  65. package/src/components/event-stream-view.test.ts +7 -7
  66. package/src/components/event-stream-view.ts +5 -5
  67. package/src/components/header-builder.ts +33 -27
  68. package/src/components/header-layouts.ts +1 -1
  69. package/src/components/header-parts.test.ts +90 -0
  70. package/src/components/header-parts.ts +62 -93
  71. package/src/components/launcher.test.ts +11 -0
  72. package/src/components/launcher.ts +1 -1
  73. package/src/components/message-bubble.test.ts +74 -2
  74. package/src/components/message-bubble.ts +25 -11
  75. package/src/components/messages.ts +2 -2
  76. package/src/components/panel.test.ts +1 -1
  77. package/src/components/panel.ts +92 -97
  78. package/src/components/pill-composer-builder.ts +28 -24
  79. package/src/components/reasoning-bubble.ts +2 -2
  80. package/src/components/suggestions.ts +1 -1
  81. package/src/components/tool-bubble.ts +1 -1
  82. package/src/components/widget-view.test.ts +181 -0
  83. package/src/components/widget-view.ts +234 -0
  84. package/src/defaults.ts +54 -51
  85. package/src/generated/runtype-openapi-contract.ts +5 -0
  86. package/src/index-core.ts +30 -4
  87. package/src/index-global.ts +73 -6
  88. package/src/index.ts +7 -1
  89. package/src/install.test.ts +11 -11
  90. package/src/install.ts +28 -18
  91. package/src/launcher-global.ts +25 -14
  92. package/src/markdown-parsers-eager.ts +20 -0
  93. package/src/markdown-parsers-entry.ts +4 -0
  94. package/src/markdown-parsers-loader.ts +49 -0
  95. package/src/plugin-kit.test.ts +1 -1
  96. package/src/plugin-kit.ts +7 -7
  97. package/src/plugins/types.ts +5 -5
  98. package/src/postprocessors.ts +27 -14
  99. package/src/runtime/host-layout.test.ts +1 -1
  100. package/src/runtime/host-layout.ts +4 -5
  101. package/src/runtime/init.ts +2 -2
  102. package/src/session.test.ts +5 -6
  103. package/src/session.ts +333 -227
  104. package/src/session.voice.test.ts +146 -0
  105. package/src/session.webmcp.test.ts +24 -25
  106. package/src/smart-dom-reader.ts +3 -4
  107. package/src/styles/widget.css +54 -36
  108. package/src/suggest-replies-tool.test.ts +3 -3
  109. package/src/suggest-replies-tool.ts +5 -5
  110. package/src/testing/index.ts +1 -1
  111. package/src/theme-editor/color-utils.ts +2 -2
  112. package/src/theme-editor/index.ts +4 -13
  113. package/src/theme-editor/presets.ts +1 -1
  114. package/src/theme-editor/preview-utils.ts +22 -20
  115. package/src/theme-editor/preview.ts +4 -1
  116. package/src/theme-editor/role-mappings.ts +3 -3
  117. package/src/theme-editor/sections.ts +21 -20
  118. package/src/theme-editor/types.ts +1 -1
  119. package/src/theme-editor/webmcp/index.ts +1 -1
  120. package/src/theme-editor/webmcp/summary.ts +1 -2
  121. package/src/theme-editor/webmcp/tools.ts +30 -30
  122. package/src/theme-editor/webmcp/types.ts +2 -2
  123. package/src/theme-editor-preview.ts +10 -0
  124. package/src/theme-reference.ts +26 -26
  125. package/src/types.ts +368 -99
  126. package/src/ui.approval-plugin.test.ts +3 -3
  127. package/src/ui.ask-user-question-plugin.test.ts +9 -9
  128. package/src/ui.component-directive.test.ts +4 -4
  129. package/src/ui.composer-bar.test.ts +10 -11
  130. package/src/ui.composer-keyboard.test.ts +5 -5
  131. package/src/ui.header-icon-color.test.ts +59 -0
  132. package/src/ui.scroll.test.ts +5 -6
  133. package/src/ui.stop-button.test.ts +3 -4
  134. package/src/ui.suggest-replies.test.ts +2 -2
  135. package/src/ui.ts +292 -131
  136. package/src/utils/buttons.ts +1 -1
  137. package/src/utils/code-generators.ts +1 -1
  138. package/src/utils/composer-history.ts +3 -3
  139. package/src/utils/dom-context.test.ts +2 -2
  140. package/src/utils/dom-context.ts +7 -7
  141. package/src/utils/dom.test.ts +67 -0
  142. package/src/utils/dom.ts +75 -0
  143. package/src/utils/dropdown.ts +2 -2
  144. package/src/utils/event-stream-buffer.test.ts +2 -2
  145. package/src/utils/message-fingerprint.test.ts +9 -0
  146. package/src/utils/message-fingerprint.ts +5 -0
  147. package/src/utils/morph.ts +1 -1
  148. package/src/utils/sanitize.ts +32 -16
  149. package/src/utils/sequence-buffer.test.ts +8 -8
  150. package/src/utils/sequence-buffer.ts +4 -4
  151. package/src/utils/smart-dom-adapter.test.ts +1 -1
  152. package/src/utils/smart-dom-adapter.ts +4 -5
  153. package/src/utils/speech-text.test.ts +122 -0
  154. package/src/utils/speech-text.ts +101 -0
  155. package/src/utils/stream-animation.test.ts +3 -3
  156. package/src/utils/stream-animation.ts +6 -6
  157. package/src/utils/theme.test.ts +1 -1
  158. package/src/utils/throughput-tracker.test.ts +11 -11
  159. package/src/utils/throughput-tracker.ts +9 -9
  160. package/src/utils/tokens.ts +13 -13
  161. package/src/utils/virtual-scroller.ts +1 -1
  162. package/src/vendor/smart-dom-reader/README.md +4 -4
  163. package/src/vendor/smart-dom-reader/index.d.ts +1 -1
  164. package/src/vendor/smart-dom-reader/index.js +2 -2
  165. package/src/version.ts +1 -1
  166. package/src/voice/audio-playback-manager.test.ts +159 -0
  167. package/src/voice/audio-playback-manager.ts +145 -30
  168. package/src/voice/browser-speech-engine.ts +130 -0
  169. package/src/voice/fallback-speech-engine.ts +89 -0
  170. package/src/voice/index.ts +21 -0
  171. package/src/voice/read-aloud-controller.test.ts +143 -0
  172. package/src/voice/read-aloud-controller.ts +136 -0
  173. package/src/voice/runtype-speech-engine.test.ts +280 -0
  174. package/src/voice/runtype-speech-engine.ts +211 -0
  175. package/src/voice/runtype-tts-entry.ts +10 -0
  176. package/src/voice/runtype-tts-loader.test.ts +47 -0
  177. package/src/voice/runtype-tts-loader.ts +42 -0
  178. package/src/voice/runtype-voice-provider.ts +332 -537
  179. package/src/voice/voice-factory.ts +26 -4
  180. package/src/voice/voice.test.ts +321 -7
  181. package/src/voice/worklet-playback-engine.ts +272 -0
  182. package/src/voice-worklet-player.ts +39 -0
  183. package/src/webmcp-bridge.test.ts +6 -6
  184. package/src/webmcp-bridge.ts +24 -25
  185. package/src/webmcp-polyfill.ts +1 -1
  186. package/src/voice/voice-activity-detector.ts +0 -90
@@ -0,0 +1,159 @@
1
+ // Tests for the AudioPlaybackManager PcmStreamPlayer surface: prebuffer gating,
2
+ // onStarted-once, short/empty-reply finish, and pause/resume via AudioContext
3
+ // suspension. Web Audio is faked — these assert scheduling decisions, not sound.
4
+ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
5
+ import { AudioPlaybackManager } from "./audio-playback-manager";
6
+
7
+ class MockBufferSource {
8
+ buffer: { duration: number } | null = null;
9
+ onended: (() => void) | null = null;
10
+ startTime = -1;
11
+ connect() {}
12
+ disconnect() {}
13
+ start(t: number) {
14
+ this.startTime = t;
15
+ }
16
+ stop() {}
17
+ }
18
+
19
+ class MockAudioContext {
20
+ static instances: MockAudioContext[] = [];
21
+ state: "running" | "suspended" | "closed" = "running";
22
+ currentTime = 0;
23
+ destination = {};
24
+ sampleRate: number;
25
+ sources: MockBufferSource[] = [];
26
+ suspendCalls = 0;
27
+ resumeCalls = 0;
28
+
29
+ constructor(opts?: { sampleRate?: number }) {
30
+ this.sampleRate = opts?.sampleRate ?? 44100;
31
+ MockAudioContext.instances.push(this);
32
+ }
33
+ createBuffer(_channels: number, length: number, rate: number) {
34
+ const data = new Float32Array(length);
35
+ return { length, duration: length / rate, getChannelData: () => data };
36
+ }
37
+ createBufferSource() {
38
+ const s = new MockBufferSource();
39
+ this.sources.push(s);
40
+ return s as unknown as AudioBufferSourceNode;
41
+ }
42
+ async suspend() {
43
+ this.suspendCalls++;
44
+ this.state = "suspended";
45
+ }
46
+ async resume() {
47
+ this.resumeCalls++;
48
+ this.state = "running";
49
+ }
50
+ async close() {
51
+ this.state = "closed";
52
+ }
53
+ }
54
+
55
+ /** A PCM16 chunk of `n` samples (content is irrelevant — we assert scheduling). */
56
+ const pcm = (n: number) => new Uint8Array(n * 2);
57
+
58
+ describe("AudioPlaybackManager (PcmStreamPlayer surface)", () => {
59
+ beforeEach(() => {
60
+ MockAudioContext.instances = [];
61
+ (global as any).window = {
62
+ AudioContext: MockAudioContext,
63
+ webkitAudioContext: MockAudioContext,
64
+ };
65
+ });
66
+
67
+ afterEach(() => {
68
+ delete (global as any).window;
69
+ });
70
+
71
+ // waterline @ 24kHz / 200ms = round(24000 * 200 / 1000) = 4800 samples.
72
+ it("holds audio until the prebuffer waterline, then releases and fires onStarted once", () => {
73
+ const m = new AudioPlaybackManager(24000, { prebufferMs: 200 });
74
+ const started = vi.fn();
75
+ m.onStarted(started);
76
+
77
+ // Below the waterline: nothing is scheduled and no AudioContext is created.
78
+ m.enqueue(pcm(1000));
79
+ expect(MockAudioContext.instances).toHaveLength(0);
80
+ expect(started).not.toHaveBeenCalled();
81
+
82
+ // Crossing the waterline releases the whole held batch as scheduled sources.
83
+ m.enqueue(pcm(4000)); // 1000 + 4000 = 5000 >= 4800
84
+ const ctx = MockAudioContext.instances[0];
85
+ expect(ctx.sources).toHaveLength(2);
86
+ expect(started).toHaveBeenCalledTimes(1);
87
+
88
+ // Subsequent post-release chunks schedule directly and never re-fire onStarted.
89
+ m.enqueue(pcm(100));
90
+ expect(ctx.sources).toHaveLength(3);
91
+ expect(started).toHaveBeenCalledTimes(1);
92
+ });
93
+
94
+ it("plays a reply shorter than the prebuffer (markStreamEnd flushes the gate)", () => {
95
+ const m = new AudioPlaybackManager(24000, { prebufferMs: 200 });
96
+ const finished = vi.fn();
97
+ m.onFinished(finished);
98
+
99
+ m.enqueue(pcm(500)); // well below the 4800 waterline
100
+ expect(MockAudioContext.instances).toHaveLength(0);
101
+
102
+ m.markStreamEnd();
103
+ const ctx = MockAudioContext.instances[0];
104
+ expect(ctx.sources).toHaveLength(1); // released so it still plays
105
+ expect(finished).not.toHaveBeenCalled(); // not until the source ends
106
+
107
+ ctx.sources[0].onended?.();
108
+ expect(finished).toHaveBeenCalledTimes(1);
109
+ });
110
+
111
+ it("resolves an empty reply to finished without ever opening an AudioContext", () => {
112
+ const m = new AudioPlaybackManager(24000, { prebufferMs: 200 });
113
+ const finished = vi.fn();
114
+ m.onFinished(finished);
115
+
116
+ m.markStreamEnd(); // no audio ever enqueued
117
+ expect(finished).toHaveBeenCalledTimes(1);
118
+ expect(MockAudioContext.instances).toHaveLength(0);
119
+ });
120
+
121
+ it("pauses via AudioContext.suspend and does not auto-resume on further audio", () => {
122
+ const m = new AudioPlaybackManager(24000); // prebufferMs 0 → realtime, no gate
123
+ const started = vi.fn();
124
+ m.onStarted(started);
125
+
126
+ m.enqueue(pcm(100)); // schedules immediately
127
+ const ctx = MockAudioContext.instances[0];
128
+ expect(ctx.sources).toHaveLength(1);
129
+ expect(started).toHaveBeenCalledTimes(1);
130
+ expect(ctx.state).toBe("running");
131
+
132
+ m.pause();
133
+ expect(ctx.suspendCalls).toBe(1);
134
+ expect(ctx.state).toBe("suspended");
135
+
136
+ // More audio arriving while paused must NOT silently un-pause the context.
137
+ m.enqueue(pcm(100));
138
+ expect(ctx.state).toBe("suspended");
139
+
140
+ m.resume();
141
+ expect(ctx.resumeCalls).toBeGreaterThanOrEqual(1);
142
+ expect(ctx.state).toBe("running");
143
+ });
144
+
145
+ it("clears the started latch on flush so the next reply can fire onStarted again", () => {
146
+ const m = new AudioPlaybackManager(24000); // realtime: schedules on first chunk
147
+ const started = vi.fn();
148
+ m.onStarted(started);
149
+
150
+ m.enqueue(pcm(100));
151
+ expect(started).toHaveBeenCalledTimes(1);
152
+
153
+ m.flush();
154
+ const startedAgain = vi.fn();
155
+ m.onStarted(startedAgain);
156
+ m.enqueue(pcm(100));
157
+ expect(startedAgain).toHaveBeenCalledTimes(1);
158
+ });
159
+ });
@@ -1,3 +1,5 @@
1
+ import type { PcmStreamPlayer } from "../types";
2
+
1
3
  /**
2
4
  * AudioPlaybackManager
3
5
  *
@@ -7,26 +9,60 @@
7
9
  * playback using AudioBufferSourceNode.
8
10
  *
9
11
  * Works on all browsers including iOS Safari (no MediaSource dependency).
12
+ *
13
+ * This is the default {@link PcmStreamPlayer} for both the realtime voice
14
+ * provider and the `provider: 'runtype'` "Read aloud" path. It runs entirely on
15
+ * the main thread, so it stays in the main bundle (no AudioWorklet module). The
16
+ * jitter-buffered AudioWorklet player (`createPcmStreamPlayer` from
17
+ * `@runtypelabs/persona/voice-worklet-player`) is the higher-quality, opt-in
18
+ * alternative — inject it via `voiceRecognition.provider.runtype.createPlaybackEngine`
19
+ * or `textToSpeech.createPlaybackEngine` and it lands in the consumer's bundle.
20
+ *
21
+ * With the default `prebufferMs: 0` this behaves exactly like a bare scheduler
22
+ * (the realtime path), so that path is unchanged. A non-zero `prebufferMs`
23
+ * (the read-aloud path passes ~200) holds incoming audio until a waterline of
24
+ * samples is buffered before starting, and re-enters buffering on underrun —
25
+ * softening the schedule-clock snap that a hand-scheduled BufferSource otherwise
26
+ * turns into a click on bursty HTTP-streamed audio. This is a pragmatic
27
+ * approximation of the worklet's audio-thread silence, not parity.
10
28
  */
11
- export class AudioPlaybackManager {
29
+ export class AudioPlaybackManager implements PcmStreamPlayer {
12
30
  private ctx: AudioContext | null = null;
13
31
  private nextStartTime = 0;
14
32
  private activeSources: AudioBufferSourceNode[] = [];
15
33
  private finishedCallbacks: (() => void)[] = [];
34
+ private startedCallbacks: (() => void)[] = [];
16
35
  private playing = false;
17
36
  private streamEnded = false;
18
37
  private pendingCount = 0;
38
+ // Fires once per playback session when the first sample is actually scheduled
39
+ // (loading→playing). Cleared by flush(); a mid-reply underrun does not re-fire.
40
+ private started = false;
41
+ // Explicit user pause via pause(); kept separate from the AudioContext's
42
+ // autoplay-policy suspension so ensureContext() doesn't auto-resume over it.
43
+ private userPaused = false;
44
+
45
+ // Prebuffer gate: while `buffering`, incoming samples accumulate in
46
+ // `pendingBuffers` until they reach `waterlineSamples`, then release into the
47
+ // scheduler. With `waterlineSamples === 0` the gate is off (realtime default).
48
+ private buffering: boolean;
49
+ private pendingBuffers: Float32Array[] = [];
50
+ private pendingSamples = 0;
19
51
 
20
52
  // PCM format constants
21
53
  private readonly sampleRate: number;
54
+ private readonly waterlineSamples: number;
22
55
 
23
56
  // Remainder byte from a previous chunk when the chunk had an odd byte count.
24
57
  // Network chunks don't respect 2-byte sample boundaries, so we carry over
25
58
  // the orphaned byte and prepend it to the next chunk.
26
59
  private remainder: Uint8Array | null = null;
27
60
 
28
- constructor(sampleRate = 24000) {
61
+ constructor(sampleRate = 24000, options: { prebufferMs?: number } = {}) {
29
62
  this.sampleRate = sampleRate;
63
+ const prebufferMs = Math.max(0, options.prebufferMs ?? 0);
64
+ this.waterlineSamples = Math.round((sampleRate * prebufferMs) / 1000);
65
+ this.buffering = this.waterlineSamples > 0;
30
66
  }
31
67
 
32
68
  /**
@@ -41,8 +77,9 @@ export class AudioPlaybackManager {
41
77
  this.ctx = new AudioCtx({ sampleRate: this.sampleRate }) as AudioContext;
42
78
  }
43
79
  const ctx = this.ctx!;
44
- // Resume if suspended (autoplay policy)
45
- if (ctx.state === "suspended") {
80
+ // Resume if suspended (autoplay policy) — but never override an explicit
81
+ // user pause(): more audio may still stream in while paused.
82
+ if (ctx.state === "suspended" && !this.userPaused) {
46
83
  ctx.resume();
47
84
  }
48
85
  return ctx;
@@ -73,34 +110,17 @@ export class AudioPlaybackManager {
73
110
 
74
111
  if (data.length === 0) return;
75
112
 
76
- const ctx = this.ensureContext();
77
113
  const float32 = this.pcmToFloat32(data);
114
+ if (float32.length === 0) return;
78
115
 
79
- const buffer = ctx.createBuffer(1, float32.length, this.sampleRate);
80
- buffer.getChannelData(0).set(float32);
81
-
82
- const source = ctx.createBufferSource();
83
- source.buffer = buffer;
84
- source.connect(ctx.destination);
85
-
86
- // Schedule gap-free playback
87
- const now = ctx.currentTime;
88
- if (this.nextStartTime < now) {
89
- this.nextStartTime = now;
116
+ if (this.buffering) {
117
+ // Hold until the prebuffer waterline fills, then release as a batch.
118
+ this.pendingBuffers.push(float32);
119
+ this.pendingSamples += float32.length;
120
+ if (this.pendingSamples >= this.waterlineSamples) this.releaseBuffer();
121
+ } else {
122
+ this.scheduleSamples(float32);
90
123
  }
91
- source.start(this.nextStartTime);
92
- this.nextStartTime += buffer.duration;
93
-
94
- this.activeSources.push(source);
95
- this.pendingCount++;
96
- this.playing = true;
97
-
98
- source.onended = () => {
99
- const idx = this.activeSources.indexOf(source);
100
- if (idx !== -1) this.activeSources.splice(idx, 1);
101
- this.pendingCount--;
102
- this.checkFinished();
103
- };
104
124
  }
105
125
 
106
126
  /**
@@ -108,6 +128,9 @@ export class AudioPlaybackManager {
108
128
  * The onFinished callback fires after all queued audio has played.
109
129
  */
110
130
  markStreamEnd(): void {
131
+ // A reply shorter than the prebuffer never reaches the waterline; release
132
+ // whatever we held so it still plays.
133
+ if (this.pendingBuffers.length > 0) this.releaseBuffer();
111
134
  this.streamEnded = true;
112
135
  this.checkFinished();
113
136
  }
@@ -130,7 +153,13 @@ export class AudioPlaybackManager {
130
153
  this.playing = false;
131
154
  this.streamEnded = false;
132
155
  this.finishedCallbacks = [];
156
+ this.startedCallbacks = [];
133
157
  this.remainder = null;
158
+ // Reset the prebuffer gate and the started latch for the next reply.
159
+ this.pendingBuffers = [];
160
+ this.pendingSamples = 0;
161
+ this.buffering = this.waterlineSamples > 0;
162
+ this.started = false;
134
163
  }
135
164
 
136
165
  /**
@@ -147,6 +176,30 @@ export class AudioPlaybackManager {
147
176
  this.finishedCallbacks.push(callback);
148
177
  }
149
178
 
179
+ /**
180
+ * Register a callback fired once when audible playback first begins (the first
181
+ * sample is scheduled, after any prebuffer). Cleared by {@link flush}; a
182
+ * mid-reply underrun re-buffer does not re-fire it.
183
+ */
184
+ onStarted(callback: () => void): void {
185
+ this.startedCallbacks.push(callback);
186
+ }
187
+
188
+ /**
189
+ * Pause playback. Suspends the AudioContext clock; queued/scheduled audio
190
+ * freezes in place and {@link resume} continues exactly where it left off.
191
+ */
192
+ pause(): void {
193
+ this.userPaused = true;
194
+ if (this.ctx && this.ctx.state === "running") void this.ctx.suspend();
195
+ }
196
+
197
+ /** Resume playback after {@link pause}. */
198
+ resume(): void {
199
+ this.userPaused = false;
200
+ if (this.ctx && this.ctx.state === "suspended") void this.ctx.resume();
201
+ }
202
+
150
203
  /**
151
204
  * Clean up AudioContext resources.
152
205
  */
@@ -158,8 +211,70 @@ export class AudioPlaybackManager {
158
211
  }
159
212
  }
160
213
 
214
+ /** Release held prebuffer samples into the scheduler in arrival order. */
215
+ private releaseBuffer(): void {
216
+ this.buffering = false;
217
+ const held = this.pendingBuffers;
218
+ this.pendingBuffers = [];
219
+ this.pendingSamples = 0;
220
+ for (const samples of held) this.scheduleSamples(samples);
221
+ }
222
+
223
+ /** Schedule one Float32 sample block for gap-free playback. */
224
+ private scheduleSamples(float32: Float32Array): void {
225
+ if (float32.length === 0) return;
226
+ const ctx = this.ensureContext();
227
+
228
+ const buffer = ctx.createBuffer(1, float32.length, this.sampleRate);
229
+ buffer.getChannelData(0).set(float32);
230
+
231
+ const source = ctx.createBufferSource();
232
+ source.buffer = buffer;
233
+ source.connect(ctx.destination);
234
+
235
+ const now = ctx.currentTime;
236
+ if (this.nextStartTime === 0) {
237
+ // Fresh start (first audio of this session, or after flush).
238
+ this.nextStartTime = now;
239
+ } else if (this.nextStartTime < now) {
240
+ // Underrun: the playhead caught up to the queue. Snap to now (a small,
241
+ // unavoidable gap) and, when a prebuffer is configured, re-enter buffering
242
+ // so subsequent chunks re-accumulate before scheduling — collapsing a
243
+ // train of clicks into a single rebuffer.
244
+ this.nextStartTime = now;
245
+ if (this.waterlineSamples > 0) this.buffering = true;
246
+ }
247
+ source.start(this.nextStartTime);
248
+ this.nextStartTime += buffer.duration;
249
+
250
+ this.activeSources.push(source);
251
+ this.pendingCount++;
252
+ this.playing = true;
253
+
254
+ if (!this.started) {
255
+ this.started = true;
256
+ const cbs = this.startedCallbacks.slice();
257
+ this.startedCallbacks = [];
258
+ for (const cb of cbs) cb();
259
+ }
260
+
261
+ source.onended = () => {
262
+ const idx = this.activeSources.indexOf(source);
263
+ if (idx !== -1) this.activeSources.splice(idx, 1);
264
+ this.pendingCount--;
265
+ this.checkFinished();
266
+ };
267
+ }
268
+
161
269
  private checkFinished(): void {
162
- if (this.streamEnded && this.pendingCount <= 0 && this.playing) {
270
+ // Fire once the stream has ended and nothing is scheduled or held. No
271
+ // `playing` precondition: an empty reply (markStreamEnd with no audio) must
272
+ // still resolve to idle, matching the worklet's immediate 'drained'.
273
+ if (
274
+ this.streamEnded &&
275
+ this.pendingCount <= 0 &&
276
+ this.pendingBuffers.length === 0
277
+ ) {
163
278
  this.playing = false;
164
279
  this.streamEnded = false;
165
280
  const cbs = this.finishedCallbacks.slice();
@@ -0,0 +1,130 @@
1
+ // Browser Speech Engine
2
+ //
3
+ // Default `SpeechEngine` for the per-message "Read aloud" action and the
4
+ // auto-speak path, backed by the browser Web Speech API
5
+ // (`window.speechSynthesis`). Zero-backend and offline-capable, but limited to
6
+ // the OS/browser voice set.
7
+ //
8
+ // A hosted engine (Runtype TTS, ElevenLabs, a server proxy, …) implements the
9
+ // same `SpeechEngine` interface and is supplied via
10
+ // `textToSpeech.createEngine`; such an engine can stream PCM into the realtime
11
+ // voice `VoicePlaybackEngine` (see `audio-playback-manager.ts`). Nothing in the
12
+ // `ReadAloudController` is browser-specific — only this file is.
13
+
14
+ import type { SpeechCallbacks, SpeechEngine, SpeechRequest } from "../types";
15
+
16
+ /**
17
+ * Pick the best available English voice from the browser's voice list.
18
+ * Prefers high-quality remote/natural voices, then enhanced local voices,
19
+ * then standard local voices, then any English voice, then the first voice.
20
+ */
21
+ export function pickBestVoice(voices: SpeechSynthesisVoice[]): SpeechSynthesisVoice {
22
+ // Priority list: high-quality voices across browsers/platforms.
23
+ const preferred = [
24
+ // Edge Online Natural (highest quality)
25
+ "Microsoft Jenny Online (Natural) - English (United States)",
26
+ "Microsoft Aria Online (Natural) - English (United States)",
27
+ "Microsoft Guy Online (Natural) - English (United States)",
28
+ // Google remote (good quality, cross-platform in Chrome)
29
+ "Google US English",
30
+ "Google UK English Female",
31
+ // Apple premium/enhanced (macOS)
32
+ "Ava (Premium)",
33
+ "Evan (Enhanced)",
34
+ "Samantha (Enhanced)",
35
+ // Apple standard (macOS/iOS)
36
+ "Samantha",
37
+ "Daniel",
38
+ "Karen",
39
+ // Windows SAPI
40
+ "Microsoft David Desktop - English (United States)",
41
+ "Microsoft Zira Desktop - English (United States)",
42
+ ];
43
+
44
+ for (const name of preferred) {
45
+ const match = voices.find((v) => v.name === name);
46
+ if (match) return match;
47
+ }
48
+
49
+ // Fallback: any English voice, then first available.
50
+ return voices.find((v) => v.lang.startsWith("en")) ?? voices[0];
51
+ }
52
+
53
+ export interface BrowserSpeechEngineOptions {
54
+ /** Custom voice picker, used when no exact `voice` name is requested. */
55
+ pickVoice?: (voices: SpeechSynthesisVoice[]) => SpeechSynthesisVoice;
56
+ }
57
+
58
+ /** Default `SpeechEngine` backed by the browser Web Speech API. */
59
+ export class BrowserSpeechEngine implements SpeechEngine {
60
+ readonly id = "browser";
61
+ // speechSynthesis exposes pause()/resume(); reliable for start/stop and
62
+ // serviceable for pause on most engines (Chrome has known quirks on resume).
63
+ readonly supportsPause = true;
64
+
65
+ constructor(private options: BrowserSpeechEngineOptions = {}) {}
66
+
67
+ /** Whether the Web Speech API is available in this environment. */
68
+ static isSupported(): boolean {
69
+ return typeof window !== "undefined" && "speechSynthesis" in window;
70
+ }
71
+
72
+ speak(request: SpeechRequest, callbacks: SpeechCallbacks): void {
73
+ if (!BrowserSpeechEngine.isSupported()) {
74
+ callbacks.onError?.(new Error("Web Speech API is unavailable"));
75
+ return;
76
+ }
77
+
78
+ const synth = window.speechSynthesis;
79
+ synth.cancel();
80
+
81
+ const utterance = new SpeechSynthesisUtterance(request.text);
82
+ const voices = synth.getVoices();
83
+ if (request.voice) {
84
+ const match = voices.find((v) => v.name === request.voice);
85
+ if (match) utterance.voice = match;
86
+ } else if (voices.length > 0) {
87
+ utterance.voice = this.options.pickVoice
88
+ ? this.options.pickVoice(voices)
89
+ : pickBestVoice(voices);
90
+ }
91
+ if (request.rate !== undefined) utterance.rate = request.rate;
92
+ if (request.pitch !== undefined) utterance.pitch = request.pitch;
93
+
94
+ utterance.onend = () => callbacks.onEnd?.();
95
+ utterance.onerror = (event) => {
96
+ // A stop()/superseding utterance fires "canceled"/"interrupted" — that's
97
+ // a normal end of playback, not an error worth surfacing. `event` is a
98
+ // SpeechSynthesisErrorEvent, so `.error` is the failure reason.
99
+ const reason = event.error;
100
+ if (reason === "canceled" || reason === "interrupted") {
101
+ callbacks.onEnd?.();
102
+ } else {
103
+ callbacks.onError?.(new Error(reason || "Speech synthesis failed"));
104
+ }
105
+ };
106
+
107
+ // Chrome bug: cancel() immediately followed by speak() can drop rate/pitch.
108
+ // A short delay lets the engine reset before the new utterance starts.
109
+ setTimeout(() => {
110
+ synth.speak(utterance);
111
+ // `utterance.onstart` is unreliable across browsers — Chrome can leave it
112
+ // unfired even while actively speaking, which would strand the UI in the
113
+ // "loading" state. Browser TTS has no real async prepare phase, so treat
114
+ // scheduling as the start; onend/onerror still drive the return to idle.
115
+ callbacks.onStart?.();
116
+ }, 50);
117
+ }
118
+
119
+ pause(): void {
120
+ if (BrowserSpeechEngine.isSupported()) window.speechSynthesis.pause();
121
+ }
122
+
123
+ resume(): void {
124
+ if (BrowserSpeechEngine.isSupported()) window.speechSynthesis.resume();
125
+ }
126
+
127
+ stop(): void {
128
+ if (BrowserSpeechEngine.isSupported()) window.speechSynthesis.cancel();
129
+ }
130
+ }
@@ -0,0 +1,89 @@
1
+ // Fallback Speech Engine
2
+ //
3
+ // Composes a primary `SpeechEngine` with a fallback one (typically a hosted
4
+ // engine + the browser Web Speech API). If the primary fails *before* any audio
5
+ // is audible — a missing/404 endpoint, a network error, an auth failure — the
6
+ // fallback transparently takes over for that utterance, so the "Read aloud"
7
+ // button is never left broken. A failure that happens *after* playback has
8
+ // started (a mid-stream drop) is surfaced as a real error instead of restarting
9
+ // from the top.
10
+ //
11
+ // This is what makes `textToSpeech: { provider: 'runtype' }` safe to ship before
12
+ // (or independently of) the Runtype TTS endpoint: it speaks with the browser
13
+ // voice today and auto-upgrades to Runtype voices the moment the endpoint
14
+ // answers. Set `browserFallback: false` to opt out and surface Runtype errors.
15
+
16
+ import type { SpeechCallbacks, SpeechEngine, SpeechRequest } from "../types";
17
+
18
+ export interface FallbackSpeechEngineOptions {
19
+ /**
20
+ * Called once when the primary engine fails before audio starts and the
21
+ * fallback takes over — so a silent downgrade is still observable in dev/
22
+ * telemetry even though the user keeps hearing speech.
23
+ */
24
+ onFallback?: (error: Error) => void;
25
+ }
26
+
27
+ /** A `SpeechEngine` that falls back from `primary` to `fallback` per utterance. */
28
+ export class FallbackSpeechEngine implements SpeechEngine {
29
+ readonly id = "fallback";
30
+
31
+ // Whichever engine is currently driving playback, so pause/resume/stop route
32
+ // to the right one after a fallback has (or hasn't) kicked in.
33
+ private active: SpeechEngine;
34
+
35
+ constructor(
36
+ private readonly primary: SpeechEngine,
37
+ private readonly fallback: SpeechEngine,
38
+ private readonly options: FallbackSpeechEngineOptions = {},
39
+ ) {
40
+ this.active = primary;
41
+ }
42
+
43
+ // Pause/resume only matters once something is playing, and both built-in
44
+ // engines support it; report the active engine's capability.
45
+ get supportsPause(): boolean {
46
+ return this.active.supportsPause;
47
+ }
48
+
49
+ speak(request: SpeechRequest, callbacks: SpeechCallbacks): void {
50
+ this.active = this.primary;
51
+ let started = false;
52
+
53
+ this.primary.speak(request, {
54
+ onStart: () => {
55
+ started = true;
56
+ callbacks.onStart?.();
57
+ },
58
+ onEnd: () => callbacks.onEnd?.(),
59
+ onError: (error) => {
60
+ // A failure once audio is playing is a genuine error — don't restart.
61
+ if (started) {
62
+ callbacks.onError?.(error);
63
+ return;
64
+ }
65
+ // Pre-start failure: silently hand the utterance to the fallback.
66
+ this.options.onFallback?.(error);
67
+ this.active = this.fallback;
68
+ this.fallback.speak(request, callbacks);
69
+ },
70
+ });
71
+ }
72
+
73
+ pause(): void {
74
+ this.active.pause();
75
+ }
76
+
77
+ resume(): void {
78
+ this.active.resume();
79
+ }
80
+
81
+ stop(): void {
82
+ this.active.stop();
83
+ }
84
+
85
+ destroy(): void {
86
+ this.primary.destroy?.();
87
+ this.fallback.destroy?.();
88
+ }
89
+ }
@@ -14,3 +14,24 @@ export {
14
14
  createBestAvailableVoiceProvider,
15
15
  isVoiceSupported
16
16
  } from './voice-factory';
17
+
18
+ export {
19
+ BrowserSpeechEngine,
20
+ pickBestVoice,
21
+ type BrowserSpeechEngineOptions
22
+ } from './browser-speech-engine';
23
+
24
+ // NOTE: RuntypeSpeechEngine is intentionally NOT re-exported here. It pairs with
25
+ // the AudioWorklet player (`worklet-playback-engine`) and ships from the
26
+ // `@runtypelabs/persona/voice-worklet-player` subpath; `session.ts` lazy-imports
27
+ // it so its module only evaluates when `provider: 'runtype'` actually plays audio.
28
+ // FallbackSpeechEngine's value export lives on the
29
+ // `@runtypelabs/persona/voice-worklet-player` subpath so the hosted read-aloud
30
+ // path can be deferred to the `runtype-tts.js` chunk; only the (erased) type is
31
+ // re-exported here for `session.ts`.
32
+ export type { FallbackSpeechEngineOptions } from './fallback-speech-engine';
33
+
34
+ export {
35
+ ReadAloudController,
36
+ type ReadAloudListener
37
+ } from './read-aloud-controller';