@runtypelabs/persona 3.34.1 → 3.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -11
- package/dist/animations/glyph-cycle.d.cts +1 -1
- package/dist/animations/glyph-cycle.d.ts +1 -1
- package/dist/animations/{types-CthJFfNx.d.cts → types-DgYsuwXL.d.cts} +15 -14
- package/dist/animations/{types-CthJFfNx.d.ts → types-DgYsuwXL.d.ts} +15 -14
- package/dist/animations/wipe.d.cts +1 -1
- package/dist/animations/wipe.d.ts +1 -1
- package/dist/codegen.cjs +11 -11
- package/dist/codegen.js +10 -10
- package/dist/index.cjs +46 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +514 -158
- package/dist/index.d.ts +514 -158
- package/dist/index.global.js +41 -89
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +46 -44
- package/dist/index.js.map +1 -1
- package/dist/install.global.js +1 -1
- package/dist/install.global.js.map +1 -1
- package/dist/launcher.global.js +2 -2
- package/dist/launcher.global.js.map +1 -1
- package/dist/markdown-parsers.js +53 -0
- package/dist/plugin-kit.d.cts +4 -4
- package/dist/plugin-kit.d.ts +4 -4
- package/dist/runtype-tts.js +1 -0
- package/dist/smart-dom-reader.cjs +1 -1
- package/dist/smart-dom-reader.d.cts +352 -103
- package/dist/smart-dom-reader.d.ts +352 -103
- package/dist/smart-dom-reader.js +1 -1
- package/dist/theme-editor-preview.cjs +203 -0
- package/dist/theme-editor-preview.d.cts +5748 -0
- package/dist/theme-editor-preview.d.ts +5748 -0
- package/dist/theme-editor-preview.js +203 -0
- package/dist/theme-editor.cjs +14 -131
- package/dist/theme-editor.d.cts +362 -574
- package/dist/theme-editor.d.ts +362 -574
- package/dist/theme-editor.js +14 -131
- package/dist/theme-reference.cjs +1 -1
- package/dist/theme-reference.d.cts +1 -1
- package/dist/theme-reference.d.ts +1 -1
- package/dist/theme-reference.js +1 -1
- package/dist/voice-worklet-player.cjs +80 -0
- package/dist/voice-worklet-player.d.cts +215 -0
- package/dist/voice-worklet-player.d.ts +215 -0
- package/dist/voice-worklet-player.js +80 -0
- package/dist/widget.css +54 -36
- package/package.json +16 -2
- package/src/animations/glyph-cycle.ts +11 -14
- package/src/animations/wipe.ts +1 -1
- package/src/ask-user-question-tool.test.ts +1 -1
- package/src/ask-user-question-tool.ts +6 -8
- package/src/client.test.ts +56 -12
- package/src/client.ts +35 -31
- package/src/codegen.test.ts +1 -1
- package/src/codegen.ts +1 -1
- package/src/components/approval-bubble.test.ts +1 -1
- package/src/components/approval-bubble.ts +3 -3
- package/src/components/artifact-pane.ts +3 -3
- package/src/components/ask-user-question-bubble.test.ts +14 -14
- package/src/components/ask-user-question-bubble.ts +10 -10
- package/src/components/composer-builder.ts +17 -16
- package/src/components/composer-parts.test.ts +54 -0
- package/src/components/composer-parts.ts +95 -126
- package/src/components/demo-carousel.ts +1 -1
- package/src/components/event-stream-view.test.ts +7 -7
- package/src/components/event-stream-view.ts +5 -5
- package/src/components/header-builder.ts +33 -27
- package/src/components/header-layouts.ts +1 -1
- package/src/components/header-parts.test.ts +90 -0
- package/src/components/header-parts.ts +62 -93
- package/src/components/launcher.test.ts +11 -0
- package/src/components/launcher.ts +1 -1
- package/src/components/message-bubble.test.ts +74 -2
- package/src/components/message-bubble.ts +25 -11
- package/src/components/messages.ts +2 -2
- package/src/components/panel.test.ts +1 -1
- package/src/components/panel.ts +92 -97
- package/src/components/pill-composer-builder.ts +28 -24
- package/src/components/reasoning-bubble.ts +2 -2
- package/src/components/suggestions.ts +1 -1
- package/src/components/tool-bubble.ts +1 -1
- package/src/components/widget-view.test.ts +181 -0
- package/src/components/widget-view.ts +234 -0
- package/src/defaults.ts +54 -51
- package/src/generated/runtype-openapi-contract.ts +5 -0
- package/src/index-core.ts +30 -4
- package/src/index-global.ts +73 -6
- package/src/index.ts +7 -1
- package/src/install.test.ts +11 -11
- package/src/install.ts +28 -18
- package/src/launcher-global.ts +25 -14
- package/src/markdown-parsers-eager.ts +20 -0
- package/src/markdown-parsers-entry.ts +4 -0
- package/src/markdown-parsers-loader.ts +49 -0
- package/src/plugin-kit.test.ts +1 -1
- package/src/plugin-kit.ts +7 -7
- package/src/plugins/types.ts +5 -5
- package/src/postprocessors.ts +27 -14
- package/src/runtime/host-layout.test.ts +1 -1
- package/src/runtime/host-layout.ts +4 -5
- package/src/runtime/init.ts +2 -2
- package/src/session.test.ts +5 -6
- package/src/session.ts +333 -227
- package/src/session.voice.test.ts +146 -0
- package/src/session.webmcp.test.ts +24 -25
- package/src/smart-dom-reader.ts +3 -4
- package/src/styles/widget.css +54 -36
- package/src/suggest-replies-tool.test.ts +3 -3
- package/src/suggest-replies-tool.ts +5 -5
- package/src/testing/index.ts +1 -1
- package/src/theme-editor/color-utils.ts +2 -2
- package/src/theme-editor/index.ts +4 -13
- package/src/theme-editor/presets.ts +1 -1
- package/src/theme-editor/preview-utils.ts +22 -20
- package/src/theme-editor/preview.ts +4 -1
- package/src/theme-editor/role-mappings.ts +3 -3
- package/src/theme-editor/sections.ts +21 -20
- package/src/theme-editor/types.ts +1 -1
- package/src/theme-editor/webmcp/index.ts +1 -1
- package/src/theme-editor/webmcp/summary.ts +1 -2
- package/src/theme-editor/webmcp/tools.ts +30 -30
- package/src/theme-editor/webmcp/types.ts +2 -2
- package/src/theme-editor-preview.ts +10 -0
- package/src/theme-reference.ts +26 -26
- package/src/types.ts +368 -99
- package/src/ui.approval-plugin.test.ts +3 -3
- package/src/ui.ask-user-question-plugin.test.ts +9 -9
- package/src/ui.component-directive.test.ts +4 -4
- package/src/ui.composer-bar.test.ts +10 -11
- package/src/ui.composer-keyboard.test.ts +5 -5
- package/src/ui.header-icon-color.test.ts +59 -0
- package/src/ui.scroll.test.ts +5 -6
- package/src/ui.stop-button.test.ts +3 -4
- package/src/ui.suggest-replies.test.ts +2 -2
- package/src/ui.ts +292 -131
- package/src/utils/buttons.ts +1 -1
- package/src/utils/code-generators.ts +1 -1
- package/src/utils/composer-history.ts +3 -3
- package/src/utils/dom-context.test.ts +2 -2
- package/src/utils/dom-context.ts +7 -7
- package/src/utils/dom.test.ts +67 -0
- package/src/utils/dom.ts +75 -0
- package/src/utils/dropdown.ts +2 -2
- package/src/utils/event-stream-buffer.test.ts +2 -2
- package/src/utils/message-fingerprint.test.ts +9 -0
- package/src/utils/message-fingerprint.ts +5 -0
- package/src/utils/morph.ts +1 -1
- package/src/utils/sanitize.ts +32 -16
- package/src/utils/sequence-buffer.test.ts +8 -8
- package/src/utils/sequence-buffer.ts +4 -4
- package/src/utils/smart-dom-adapter.test.ts +1 -1
- package/src/utils/smart-dom-adapter.ts +4 -5
- package/src/utils/speech-text.test.ts +122 -0
- package/src/utils/speech-text.ts +101 -0
- package/src/utils/stream-animation.test.ts +3 -3
- package/src/utils/stream-animation.ts +6 -6
- package/src/utils/theme.test.ts +1 -1
- package/src/utils/throughput-tracker.test.ts +11 -11
- package/src/utils/throughput-tracker.ts +9 -9
- package/src/utils/tokens.ts +13 -13
- package/src/utils/virtual-scroller.ts +1 -1
- package/src/vendor/smart-dom-reader/README.md +4 -4
- package/src/vendor/smart-dom-reader/index.d.ts +1 -1
- package/src/vendor/smart-dom-reader/index.js +2 -2
- package/src/version.ts +1 -1
- package/src/voice/audio-playback-manager.test.ts +159 -0
- package/src/voice/audio-playback-manager.ts +145 -30
- package/src/voice/browser-speech-engine.ts +130 -0
- package/src/voice/fallback-speech-engine.ts +89 -0
- package/src/voice/index.ts +21 -0
- package/src/voice/read-aloud-controller.test.ts +143 -0
- package/src/voice/read-aloud-controller.ts +136 -0
- package/src/voice/runtype-speech-engine.test.ts +280 -0
- package/src/voice/runtype-speech-engine.ts +211 -0
- package/src/voice/runtype-tts-entry.ts +10 -0
- package/src/voice/runtype-tts-loader.test.ts +47 -0
- package/src/voice/runtype-tts-loader.ts +42 -0
- package/src/voice/runtype-voice-provider.ts +332 -537
- package/src/voice/voice-factory.ts +26 -4
- package/src/voice/voice.test.ts +321 -7
- package/src/voice/worklet-playback-engine.ts +272 -0
- package/src/voice-worklet-player.ts +39 -0
- package/src/webmcp-bridge.test.ts +6 -6
- package/src/webmcp-bridge.ts +24 -25
- package/src/webmcp-polyfill.ts +1 -1
- package/src/voice/voice-activity-detector.ts +0 -90
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// Tests for the AudioPlaybackManager PcmStreamPlayer surface: prebuffer gating,
|
|
2
|
+
// onStarted-once, short/empty-reply finish, and pause/resume via AudioContext
|
|
3
|
+
// suspension. Web Audio is faked — these assert scheduling decisions, not sound.
|
|
4
|
+
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
|
5
|
+
import { AudioPlaybackManager } from "./audio-playback-manager";
|
|
6
|
+
|
|
7
|
+
class MockBufferSource {
|
|
8
|
+
buffer: { duration: number } | null = null;
|
|
9
|
+
onended: (() => void) | null = null;
|
|
10
|
+
startTime = -1;
|
|
11
|
+
connect() {}
|
|
12
|
+
disconnect() {}
|
|
13
|
+
start(t: number) {
|
|
14
|
+
this.startTime = t;
|
|
15
|
+
}
|
|
16
|
+
stop() {}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
class MockAudioContext {
|
|
20
|
+
static instances: MockAudioContext[] = [];
|
|
21
|
+
state: "running" | "suspended" | "closed" = "running";
|
|
22
|
+
currentTime = 0;
|
|
23
|
+
destination = {};
|
|
24
|
+
sampleRate: number;
|
|
25
|
+
sources: MockBufferSource[] = [];
|
|
26
|
+
suspendCalls = 0;
|
|
27
|
+
resumeCalls = 0;
|
|
28
|
+
|
|
29
|
+
constructor(opts?: { sampleRate?: number }) {
|
|
30
|
+
this.sampleRate = opts?.sampleRate ?? 44100;
|
|
31
|
+
MockAudioContext.instances.push(this);
|
|
32
|
+
}
|
|
33
|
+
createBuffer(_channels: number, length: number, rate: number) {
|
|
34
|
+
const data = new Float32Array(length);
|
|
35
|
+
return { length, duration: length / rate, getChannelData: () => data };
|
|
36
|
+
}
|
|
37
|
+
createBufferSource() {
|
|
38
|
+
const s = new MockBufferSource();
|
|
39
|
+
this.sources.push(s);
|
|
40
|
+
return s as unknown as AudioBufferSourceNode;
|
|
41
|
+
}
|
|
42
|
+
async suspend() {
|
|
43
|
+
this.suspendCalls++;
|
|
44
|
+
this.state = "suspended";
|
|
45
|
+
}
|
|
46
|
+
async resume() {
|
|
47
|
+
this.resumeCalls++;
|
|
48
|
+
this.state = "running";
|
|
49
|
+
}
|
|
50
|
+
async close() {
|
|
51
|
+
this.state = "closed";
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** A PCM16 chunk of `n` samples (content is irrelevant — we assert scheduling). */
|
|
56
|
+
const pcm = (n: number) => new Uint8Array(n * 2);
|
|
57
|
+
|
|
58
|
+
describe("AudioPlaybackManager (PcmStreamPlayer surface)", () => {
|
|
59
|
+
beforeEach(() => {
|
|
60
|
+
MockAudioContext.instances = [];
|
|
61
|
+
(global as any).window = {
|
|
62
|
+
AudioContext: MockAudioContext,
|
|
63
|
+
webkitAudioContext: MockAudioContext,
|
|
64
|
+
};
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
afterEach(() => {
|
|
68
|
+
delete (global as any).window;
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// waterline @ 24kHz / 200ms = round(24000 * 200 / 1000) = 4800 samples.
|
|
72
|
+
it("holds audio until the prebuffer waterline, then releases and fires onStarted once", () => {
|
|
73
|
+
const m = new AudioPlaybackManager(24000, { prebufferMs: 200 });
|
|
74
|
+
const started = vi.fn();
|
|
75
|
+
m.onStarted(started);
|
|
76
|
+
|
|
77
|
+
// Below the waterline: nothing is scheduled and no AudioContext is created.
|
|
78
|
+
m.enqueue(pcm(1000));
|
|
79
|
+
expect(MockAudioContext.instances).toHaveLength(0);
|
|
80
|
+
expect(started).not.toHaveBeenCalled();
|
|
81
|
+
|
|
82
|
+
// Crossing the waterline releases the whole held batch as scheduled sources.
|
|
83
|
+
m.enqueue(pcm(4000)); // 1000 + 4000 = 5000 >= 4800
|
|
84
|
+
const ctx = MockAudioContext.instances[0];
|
|
85
|
+
expect(ctx.sources).toHaveLength(2);
|
|
86
|
+
expect(started).toHaveBeenCalledTimes(1);
|
|
87
|
+
|
|
88
|
+
// Subsequent post-release chunks schedule directly and never re-fire onStarted.
|
|
89
|
+
m.enqueue(pcm(100));
|
|
90
|
+
expect(ctx.sources).toHaveLength(3);
|
|
91
|
+
expect(started).toHaveBeenCalledTimes(1);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it("plays a reply shorter than the prebuffer (markStreamEnd flushes the gate)", () => {
|
|
95
|
+
const m = new AudioPlaybackManager(24000, { prebufferMs: 200 });
|
|
96
|
+
const finished = vi.fn();
|
|
97
|
+
m.onFinished(finished);
|
|
98
|
+
|
|
99
|
+
m.enqueue(pcm(500)); // well below the 4800 waterline
|
|
100
|
+
expect(MockAudioContext.instances).toHaveLength(0);
|
|
101
|
+
|
|
102
|
+
m.markStreamEnd();
|
|
103
|
+
const ctx = MockAudioContext.instances[0];
|
|
104
|
+
expect(ctx.sources).toHaveLength(1); // released so it still plays
|
|
105
|
+
expect(finished).not.toHaveBeenCalled(); // not until the source ends
|
|
106
|
+
|
|
107
|
+
ctx.sources[0].onended?.();
|
|
108
|
+
expect(finished).toHaveBeenCalledTimes(1);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it("resolves an empty reply to finished without ever opening an AudioContext", () => {
|
|
112
|
+
const m = new AudioPlaybackManager(24000, { prebufferMs: 200 });
|
|
113
|
+
const finished = vi.fn();
|
|
114
|
+
m.onFinished(finished);
|
|
115
|
+
|
|
116
|
+
m.markStreamEnd(); // no audio ever enqueued
|
|
117
|
+
expect(finished).toHaveBeenCalledTimes(1);
|
|
118
|
+
expect(MockAudioContext.instances).toHaveLength(0);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it("pauses via AudioContext.suspend and does not auto-resume on further audio", () => {
|
|
122
|
+
const m = new AudioPlaybackManager(24000); // prebufferMs 0 → realtime, no gate
|
|
123
|
+
const started = vi.fn();
|
|
124
|
+
m.onStarted(started);
|
|
125
|
+
|
|
126
|
+
m.enqueue(pcm(100)); // schedules immediately
|
|
127
|
+
const ctx = MockAudioContext.instances[0];
|
|
128
|
+
expect(ctx.sources).toHaveLength(1);
|
|
129
|
+
expect(started).toHaveBeenCalledTimes(1);
|
|
130
|
+
expect(ctx.state).toBe("running");
|
|
131
|
+
|
|
132
|
+
m.pause();
|
|
133
|
+
expect(ctx.suspendCalls).toBe(1);
|
|
134
|
+
expect(ctx.state).toBe("suspended");
|
|
135
|
+
|
|
136
|
+
// More audio arriving while paused must NOT silently un-pause the context.
|
|
137
|
+
m.enqueue(pcm(100));
|
|
138
|
+
expect(ctx.state).toBe("suspended");
|
|
139
|
+
|
|
140
|
+
m.resume();
|
|
141
|
+
expect(ctx.resumeCalls).toBeGreaterThanOrEqual(1);
|
|
142
|
+
expect(ctx.state).toBe("running");
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it("clears the started latch on flush so the next reply can fire onStarted again", () => {
|
|
146
|
+
const m = new AudioPlaybackManager(24000); // realtime: schedules on first chunk
|
|
147
|
+
const started = vi.fn();
|
|
148
|
+
m.onStarted(started);
|
|
149
|
+
|
|
150
|
+
m.enqueue(pcm(100));
|
|
151
|
+
expect(started).toHaveBeenCalledTimes(1);
|
|
152
|
+
|
|
153
|
+
m.flush();
|
|
154
|
+
const startedAgain = vi.fn();
|
|
155
|
+
m.onStarted(startedAgain);
|
|
156
|
+
m.enqueue(pcm(100));
|
|
157
|
+
expect(startedAgain).toHaveBeenCalledTimes(1);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { PcmStreamPlayer } from "../types";
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* AudioPlaybackManager
|
|
3
5
|
*
|
|
@@ -7,26 +9,60 @@
|
|
|
7
9
|
* playback using AudioBufferSourceNode.
|
|
8
10
|
*
|
|
9
11
|
* Works on all browsers including iOS Safari (no MediaSource dependency).
|
|
12
|
+
*
|
|
13
|
+
* This is the default {@link PcmStreamPlayer} for both the realtime voice
|
|
14
|
+
* provider and the `provider: 'runtype'` "Read aloud" path. It runs entirely on
|
|
15
|
+
* the main thread, so it stays in the main bundle (no AudioWorklet module). The
|
|
16
|
+
* jitter-buffered AudioWorklet player (`createPcmStreamPlayer` from
|
|
17
|
+
* `@runtypelabs/persona/voice-worklet-player`) is the higher-quality, opt-in
|
|
18
|
+
* alternative — inject it via `voiceRecognition.provider.runtype.createPlaybackEngine`
|
|
19
|
+
* or `textToSpeech.createPlaybackEngine` and it lands in the consumer's bundle.
|
|
20
|
+
*
|
|
21
|
+
* With the default `prebufferMs: 0` this behaves exactly like a bare scheduler
|
|
22
|
+
* (the realtime path), so that path is unchanged. A non-zero `prebufferMs`
|
|
23
|
+
* (the read-aloud path passes ~200) holds incoming audio until a waterline of
|
|
24
|
+
* samples is buffered before starting, and re-enters buffering on underrun —
|
|
25
|
+
* softening the schedule-clock snap that a hand-scheduled BufferSource otherwise
|
|
26
|
+
* turns into a click on bursty HTTP-streamed audio. This is a pragmatic
|
|
27
|
+
* approximation of the worklet's audio-thread silence, not parity.
|
|
10
28
|
*/
|
|
11
|
-
export class AudioPlaybackManager {
|
|
29
|
+
export class AudioPlaybackManager implements PcmStreamPlayer {
|
|
12
30
|
private ctx: AudioContext | null = null;
|
|
13
31
|
private nextStartTime = 0;
|
|
14
32
|
private activeSources: AudioBufferSourceNode[] = [];
|
|
15
33
|
private finishedCallbacks: (() => void)[] = [];
|
|
34
|
+
private startedCallbacks: (() => void)[] = [];
|
|
16
35
|
private playing = false;
|
|
17
36
|
private streamEnded = false;
|
|
18
37
|
private pendingCount = 0;
|
|
38
|
+
// Fires once per playback session when the first sample is actually scheduled
|
|
39
|
+
// (loading→playing). Cleared by flush(); a mid-reply underrun does not re-fire.
|
|
40
|
+
private started = false;
|
|
41
|
+
// Explicit user pause via pause(); kept separate from the AudioContext's
|
|
42
|
+
// autoplay-policy suspension so ensureContext() doesn't auto-resume over it.
|
|
43
|
+
private userPaused = false;
|
|
44
|
+
|
|
45
|
+
// Prebuffer gate: while `buffering`, incoming samples accumulate in
|
|
46
|
+
// `pendingBuffers` until they reach `waterlineSamples`, then release into the
|
|
47
|
+
// scheduler. With `waterlineSamples === 0` the gate is off (realtime default).
|
|
48
|
+
private buffering: boolean;
|
|
49
|
+
private pendingBuffers: Float32Array[] = [];
|
|
50
|
+
private pendingSamples = 0;
|
|
19
51
|
|
|
20
52
|
// PCM format constants
|
|
21
53
|
private readonly sampleRate: number;
|
|
54
|
+
private readonly waterlineSamples: number;
|
|
22
55
|
|
|
23
56
|
// Remainder byte from a previous chunk when the chunk had an odd byte count.
|
|
24
57
|
// Network chunks don't respect 2-byte sample boundaries, so we carry over
|
|
25
58
|
// the orphaned byte and prepend it to the next chunk.
|
|
26
59
|
private remainder: Uint8Array | null = null;
|
|
27
60
|
|
|
28
|
-
constructor(sampleRate = 24000) {
|
|
61
|
+
constructor(sampleRate = 24000, options: { prebufferMs?: number } = {}) {
|
|
29
62
|
this.sampleRate = sampleRate;
|
|
63
|
+
const prebufferMs = Math.max(0, options.prebufferMs ?? 0);
|
|
64
|
+
this.waterlineSamples = Math.round((sampleRate * prebufferMs) / 1000);
|
|
65
|
+
this.buffering = this.waterlineSamples > 0;
|
|
30
66
|
}
|
|
31
67
|
|
|
32
68
|
/**
|
|
@@ -41,8 +77,9 @@ export class AudioPlaybackManager {
|
|
|
41
77
|
this.ctx = new AudioCtx({ sampleRate: this.sampleRate }) as AudioContext;
|
|
42
78
|
}
|
|
43
79
|
const ctx = this.ctx!;
|
|
44
|
-
// Resume if suspended (autoplay policy)
|
|
45
|
-
|
|
80
|
+
// Resume if suspended (autoplay policy) — but never override an explicit
|
|
81
|
+
// user pause(): more audio may still stream in while paused.
|
|
82
|
+
if (ctx.state === "suspended" && !this.userPaused) {
|
|
46
83
|
ctx.resume();
|
|
47
84
|
}
|
|
48
85
|
return ctx;
|
|
@@ -73,34 +110,17 @@ export class AudioPlaybackManager {
|
|
|
73
110
|
|
|
74
111
|
if (data.length === 0) return;
|
|
75
112
|
|
|
76
|
-
const ctx = this.ensureContext();
|
|
77
113
|
const float32 = this.pcmToFloat32(data);
|
|
114
|
+
if (float32.length === 0) return;
|
|
78
115
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
// Schedule gap-free playback
|
|
87
|
-
const now = ctx.currentTime;
|
|
88
|
-
if (this.nextStartTime < now) {
|
|
89
|
-
this.nextStartTime = now;
|
|
116
|
+
if (this.buffering) {
|
|
117
|
+
// Hold until the prebuffer waterline fills, then release as a batch.
|
|
118
|
+
this.pendingBuffers.push(float32);
|
|
119
|
+
this.pendingSamples += float32.length;
|
|
120
|
+
if (this.pendingSamples >= this.waterlineSamples) this.releaseBuffer();
|
|
121
|
+
} else {
|
|
122
|
+
this.scheduleSamples(float32);
|
|
90
123
|
}
|
|
91
|
-
source.start(this.nextStartTime);
|
|
92
|
-
this.nextStartTime += buffer.duration;
|
|
93
|
-
|
|
94
|
-
this.activeSources.push(source);
|
|
95
|
-
this.pendingCount++;
|
|
96
|
-
this.playing = true;
|
|
97
|
-
|
|
98
|
-
source.onended = () => {
|
|
99
|
-
const idx = this.activeSources.indexOf(source);
|
|
100
|
-
if (idx !== -1) this.activeSources.splice(idx, 1);
|
|
101
|
-
this.pendingCount--;
|
|
102
|
-
this.checkFinished();
|
|
103
|
-
};
|
|
104
124
|
}
|
|
105
125
|
|
|
106
126
|
/**
|
|
@@ -108,6 +128,9 @@ export class AudioPlaybackManager {
|
|
|
108
128
|
* The onFinished callback fires after all queued audio has played.
|
|
109
129
|
*/
|
|
110
130
|
markStreamEnd(): void {
|
|
131
|
+
// A reply shorter than the prebuffer never reaches the waterline; release
|
|
132
|
+
// whatever we held so it still plays.
|
|
133
|
+
if (this.pendingBuffers.length > 0) this.releaseBuffer();
|
|
111
134
|
this.streamEnded = true;
|
|
112
135
|
this.checkFinished();
|
|
113
136
|
}
|
|
@@ -130,7 +153,13 @@ export class AudioPlaybackManager {
|
|
|
130
153
|
this.playing = false;
|
|
131
154
|
this.streamEnded = false;
|
|
132
155
|
this.finishedCallbacks = [];
|
|
156
|
+
this.startedCallbacks = [];
|
|
133
157
|
this.remainder = null;
|
|
158
|
+
// Reset the prebuffer gate and the started latch for the next reply.
|
|
159
|
+
this.pendingBuffers = [];
|
|
160
|
+
this.pendingSamples = 0;
|
|
161
|
+
this.buffering = this.waterlineSamples > 0;
|
|
162
|
+
this.started = false;
|
|
134
163
|
}
|
|
135
164
|
|
|
136
165
|
/**
|
|
@@ -147,6 +176,30 @@ export class AudioPlaybackManager {
|
|
|
147
176
|
this.finishedCallbacks.push(callback);
|
|
148
177
|
}
|
|
149
178
|
|
|
179
|
+
/**
|
|
180
|
+
* Register a callback fired once when audible playback first begins (the first
|
|
181
|
+
* sample is scheduled, after any prebuffer). Cleared by {@link flush}; a
|
|
182
|
+
* mid-reply underrun re-buffer does not re-fire it.
|
|
183
|
+
*/
|
|
184
|
+
onStarted(callback: () => void): void {
|
|
185
|
+
this.startedCallbacks.push(callback);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Pause playback. Suspends the AudioContext clock; queued/scheduled audio
|
|
190
|
+
* freezes in place and {@link resume} continues exactly where it left off.
|
|
191
|
+
*/
|
|
192
|
+
pause(): void {
|
|
193
|
+
this.userPaused = true;
|
|
194
|
+
if (this.ctx && this.ctx.state === "running") void this.ctx.suspend();
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Resume playback after {@link pause}. */
|
|
198
|
+
resume(): void {
|
|
199
|
+
this.userPaused = false;
|
|
200
|
+
if (this.ctx && this.ctx.state === "suspended") void this.ctx.resume();
|
|
201
|
+
}
|
|
202
|
+
|
|
150
203
|
/**
|
|
151
204
|
* Clean up AudioContext resources.
|
|
152
205
|
*/
|
|
@@ -158,8 +211,70 @@ export class AudioPlaybackManager {
|
|
|
158
211
|
}
|
|
159
212
|
}
|
|
160
213
|
|
|
214
|
+
/** Release held prebuffer samples into the scheduler in arrival order. */
|
|
215
|
+
private releaseBuffer(): void {
|
|
216
|
+
this.buffering = false;
|
|
217
|
+
const held = this.pendingBuffers;
|
|
218
|
+
this.pendingBuffers = [];
|
|
219
|
+
this.pendingSamples = 0;
|
|
220
|
+
for (const samples of held) this.scheduleSamples(samples);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** Schedule one Float32 sample block for gap-free playback. */
|
|
224
|
+
private scheduleSamples(float32: Float32Array): void {
|
|
225
|
+
if (float32.length === 0) return;
|
|
226
|
+
const ctx = this.ensureContext();
|
|
227
|
+
|
|
228
|
+
const buffer = ctx.createBuffer(1, float32.length, this.sampleRate);
|
|
229
|
+
buffer.getChannelData(0).set(float32);
|
|
230
|
+
|
|
231
|
+
const source = ctx.createBufferSource();
|
|
232
|
+
source.buffer = buffer;
|
|
233
|
+
source.connect(ctx.destination);
|
|
234
|
+
|
|
235
|
+
const now = ctx.currentTime;
|
|
236
|
+
if (this.nextStartTime === 0) {
|
|
237
|
+
// Fresh start (first audio of this session, or after flush).
|
|
238
|
+
this.nextStartTime = now;
|
|
239
|
+
} else if (this.nextStartTime < now) {
|
|
240
|
+
// Underrun: the playhead caught up to the queue. Snap to now (a small,
|
|
241
|
+
// unavoidable gap) and, when a prebuffer is configured, re-enter buffering
|
|
242
|
+
// so subsequent chunks re-accumulate before scheduling — collapsing a
|
|
243
|
+
// train of clicks into a single rebuffer.
|
|
244
|
+
this.nextStartTime = now;
|
|
245
|
+
if (this.waterlineSamples > 0) this.buffering = true;
|
|
246
|
+
}
|
|
247
|
+
source.start(this.nextStartTime);
|
|
248
|
+
this.nextStartTime += buffer.duration;
|
|
249
|
+
|
|
250
|
+
this.activeSources.push(source);
|
|
251
|
+
this.pendingCount++;
|
|
252
|
+
this.playing = true;
|
|
253
|
+
|
|
254
|
+
if (!this.started) {
|
|
255
|
+
this.started = true;
|
|
256
|
+
const cbs = this.startedCallbacks.slice();
|
|
257
|
+
this.startedCallbacks = [];
|
|
258
|
+
for (const cb of cbs) cb();
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
source.onended = () => {
|
|
262
|
+
const idx = this.activeSources.indexOf(source);
|
|
263
|
+
if (idx !== -1) this.activeSources.splice(idx, 1);
|
|
264
|
+
this.pendingCount--;
|
|
265
|
+
this.checkFinished();
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
161
269
|
private checkFinished(): void {
|
|
162
|
-
|
|
270
|
+
// Fire once the stream has ended and nothing is scheduled or held. No
|
|
271
|
+
// `playing` precondition: an empty reply (markStreamEnd with no audio) must
|
|
272
|
+
// still resolve to idle, matching the worklet's immediate 'drained'.
|
|
273
|
+
if (
|
|
274
|
+
this.streamEnded &&
|
|
275
|
+
this.pendingCount <= 0 &&
|
|
276
|
+
this.pendingBuffers.length === 0
|
|
277
|
+
) {
|
|
163
278
|
this.playing = false;
|
|
164
279
|
this.streamEnded = false;
|
|
165
280
|
const cbs = this.finishedCallbacks.slice();
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
// Browser Speech Engine
|
|
2
|
+
//
|
|
3
|
+
// Default `SpeechEngine` for the per-message "Read aloud" action and the
|
|
4
|
+
// auto-speak path, backed by the browser Web Speech API
|
|
5
|
+
// (`window.speechSynthesis`). Zero-backend and offline-capable, but limited to
|
|
6
|
+
// the OS/browser voice set.
|
|
7
|
+
//
|
|
8
|
+
// A hosted engine (Runtype TTS, ElevenLabs, a server proxy, …) implements the
|
|
9
|
+
// same `SpeechEngine` interface and is supplied via
|
|
10
|
+
// `textToSpeech.createEngine`; such an engine can stream PCM into the realtime
|
|
11
|
+
// voice `VoicePlaybackEngine` (see `audio-playback-manager.ts`). Nothing in the
|
|
12
|
+
// `ReadAloudController` is browser-specific — only this file is.
|
|
13
|
+
|
|
14
|
+
import type { SpeechCallbacks, SpeechEngine, SpeechRequest } from "../types";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Pick the best available English voice from the browser's voice list.
|
|
18
|
+
* Prefers high-quality remote/natural voices, then enhanced local voices,
|
|
19
|
+
* then standard local voices, then any English voice, then the first voice.
|
|
20
|
+
*/
|
|
21
|
+
export function pickBestVoice(voices: SpeechSynthesisVoice[]): SpeechSynthesisVoice {
|
|
22
|
+
// Priority list: high-quality voices across browsers/platforms.
|
|
23
|
+
const preferred = [
|
|
24
|
+
// Edge Online Natural (highest quality)
|
|
25
|
+
"Microsoft Jenny Online (Natural) - English (United States)",
|
|
26
|
+
"Microsoft Aria Online (Natural) - English (United States)",
|
|
27
|
+
"Microsoft Guy Online (Natural) - English (United States)",
|
|
28
|
+
// Google remote (good quality, cross-platform in Chrome)
|
|
29
|
+
"Google US English",
|
|
30
|
+
"Google UK English Female",
|
|
31
|
+
// Apple premium/enhanced (macOS)
|
|
32
|
+
"Ava (Premium)",
|
|
33
|
+
"Evan (Enhanced)",
|
|
34
|
+
"Samantha (Enhanced)",
|
|
35
|
+
// Apple standard (macOS/iOS)
|
|
36
|
+
"Samantha",
|
|
37
|
+
"Daniel",
|
|
38
|
+
"Karen",
|
|
39
|
+
// Windows SAPI
|
|
40
|
+
"Microsoft David Desktop - English (United States)",
|
|
41
|
+
"Microsoft Zira Desktop - English (United States)",
|
|
42
|
+
];
|
|
43
|
+
|
|
44
|
+
for (const name of preferred) {
|
|
45
|
+
const match = voices.find((v) => v.name === name);
|
|
46
|
+
if (match) return match;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Fallback: any English voice, then first available.
|
|
50
|
+
return voices.find((v) => v.lang.startsWith("en")) ?? voices[0];
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface BrowserSpeechEngineOptions {
|
|
54
|
+
/** Custom voice picker, used when no exact `voice` name is requested. */
|
|
55
|
+
pickVoice?: (voices: SpeechSynthesisVoice[]) => SpeechSynthesisVoice;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Default `SpeechEngine` backed by the browser Web Speech API. */
|
|
59
|
+
export class BrowserSpeechEngine implements SpeechEngine {
|
|
60
|
+
readonly id = "browser";
|
|
61
|
+
// speechSynthesis exposes pause()/resume(); reliable for start/stop and
|
|
62
|
+
// serviceable for pause on most engines (Chrome has known quirks on resume).
|
|
63
|
+
readonly supportsPause = true;
|
|
64
|
+
|
|
65
|
+
constructor(private options: BrowserSpeechEngineOptions = {}) {}
|
|
66
|
+
|
|
67
|
+
/** Whether the Web Speech API is available in this environment. */
|
|
68
|
+
static isSupported(): boolean {
|
|
69
|
+
return typeof window !== "undefined" && "speechSynthesis" in window;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
speak(request: SpeechRequest, callbacks: SpeechCallbacks): void {
|
|
73
|
+
if (!BrowserSpeechEngine.isSupported()) {
|
|
74
|
+
callbacks.onError?.(new Error("Web Speech API is unavailable"));
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const synth = window.speechSynthesis;
|
|
79
|
+
synth.cancel();
|
|
80
|
+
|
|
81
|
+
const utterance = new SpeechSynthesisUtterance(request.text);
|
|
82
|
+
const voices = synth.getVoices();
|
|
83
|
+
if (request.voice) {
|
|
84
|
+
const match = voices.find((v) => v.name === request.voice);
|
|
85
|
+
if (match) utterance.voice = match;
|
|
86
|
+
} else if (voices.length > 0) {
|
|
87
|
+
utterance.voice = this.options.pickVoice
|
|
88
|
+
? this.options.pickVoice(voices)
|
|
89
|
+
: pickBestVoice(voices);
|
|
90
|
+
}
|
|
91
|
+
if (request.rate !== undefined) utterance.rate = request.rate;
|
|
92
|
+
if (request.pitch !== undefined) utterance.pitch = request.pitch;
|
|
93
|
+
|
|
94
|
+
utterance.onend = () => callbacks.onEnd?.();
|
|
95
|
+
utterance.onerror = (event) => {
|
|
96
|
+
// A stop()/superseding utterance fires "canceled"/"interrupted" — that's
|
|
97
|
+
// a normal end of playback, not an error worth surfacing. `event` is a
|
|
98
|
+
// SpeechSynthesisErrorEvent, so `.error` is the failure reason.
|
|
99
|
+
const reason = event.error;
|
|
100
|
+
if (reason === "canceled" || reason === "interrupted") {
|
|
101
|
+
callbacks.onEnd?.();
|
|
102
|
+
} else {
|
|
103
|
+
callbacks.onError?.(new Error(reason || "Speech synthesis failed"));
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
// Chrome bug: cancel() immediately followed by speak() can drop rate/pitch.
|
|
108
|
+
// A short delay lets the engine reset before the new utterance starts.
|
|
109
|
+
setTimeout(() => {
|
|
110
|
+
synth.speak(utterance);
|
|
111
|
+
// `utterance.onstart` is unreliable across browsers — Chrome can leave it
|
|
112
|
+
// unfired even while actively speaking, which would strand the UI in the
|
|
113
|
+
// "loading" state. Browser TTS has no real async prepare phase, so treat
|
|
114
|
+
// scheduling as the start; onend/onerror still drive the return to idle.
|
|
115
|
+
callbacks.onStart?.();
|
|
116
|
+
}, 50);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
pause(): void {
|
|
120
|
+
if (BrowserSpeechEngine.isSupported()) window.speechSynthesis.pause();
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
resume(): void {
|
|
124
|
+
if (BrowserSpeechEngine.isSupported()) window.speechSynthesis.resume();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
stop(): void {
|
|
128
|
+
if (BrowserSpeechEngine.isSupported()) window.speechSynthesis.cancel();
|
|
129
|
+
}
|
|
130
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// Fallback Speech Engine
|
|
2
|
+
//
|
|
3
|
+
// Composes a primary `SpeechEngine` with a fallback one (typically a hosted
|
|
4
|
+
// engine + the browser Web Speech API). If the primary fails *before* any audio
|
|
5
|
+
// is audible — a missing/404 endpoint, a network error, an auth failure — the
|
|
6
|
+
// fallback transparently takes over for that utterance, so the "Read aloud"
|
|
7
|
+
// button is never left broken. A failure that happens *after* playback has
|
|
8
|
+
// started (a mid-stream drop) is surfaced as a real error instead of restarting
|
|
9
|
+
// from the top.
|
|
10
|
+
//
|
|
11
|
+
// This is what makes `textToSpeech: { provider: 'runtype' }` safe to ship before
|
|
12
|
+
// (or independently of) the Runtype TTS endpoint: it speaks with the browser
|
|
13
|
+
// voice today and auto-upgrades to Runtype voices the moment the endpoint
|
|
14
|
+
// answers. Set `browserFallback: false` to opt out and surface Runtype errors.
|
|
15
|
+
|
|
16
|
+
import type { SpeechCallbacks, SpeechEngine, SpeechRequest } from "../types";
|
|
17
|
+
|
|
18
|
+
export interface FallbackSpeechEngineOptions {
|
|
19
|
+
/**
|
|
20
|
+
* Called once when the primary engine fails before audio starts and the
|
|
21
|
+
* fallback takes over — so a silent downgrade is still observable in dev/
|
|
22
|
+
* telemetry even though the user keeps hearing speech.
|
|
23
|
+
*/
|
|
24
|
+
onFallback?: (error: Error) => void;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** A `SpeechEngine` that falls back from `primary` to `fallback` per utterance. */
|
|
28
|
+
export class FallbackSpeechEngine implements SpeechEngine {
|
|
29
|
+
readonly id = "fallback";
|
|
30
|
+
|
|
31
|
+
// Whichever engine is currently driving playback, so pause/resume/stop route
|
|
32
|
+
// to the right one after a fallback has (or hasn't) kicked in.
|
|
33
|
+
private active: SpeechEngine;
|
|
34
|
+
|
|
35
|
+
constructor(
|
|
36
|
+
private readonly primary: SpeechEngine,
|
|
37
|
+
private readonly fallback: SpeechEngine,
|
|
38
|
+
private readonly options: FallbackSpeechEngineOptions = {},
|
|
39
|
+
) {
|
|
40
|
+
this.active = primary;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Pause/resume only matters once something is playing, and both built-in
|
|
44
|
+
// engines support it; report the active engine's capability.
|
|
45
|
+
get supportsPause(): boolean {
|
|
46
|
+
return this.active.supportsPause;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
speak(request: SpeechRequest, callbacks: SpeechCallbacks): void {
|
|
50
|
+
this.active = this.primary;
|
|
51
|
+
let started = false;
|
|
52
|
+
|
|
53
|
+
this.primary.speak(request, {
|
|
54
|
+
onStart: () => {
|
|
55
|
+
started = true;
|
|
56
|
+
callbacks.onStart?.();
|
|
57
|
+
},
|
|
58
|
+
onEnd: () => callbacks.onEnd?.(),
|
|
59
|
+
onError: (error) => {
|
|
60
|
+
// A failure once audio is playing is a genuine error — don't restart.
|
|
61
|
+
if (started) {
|
|
62
|
+
callbacks.onError?.(error);
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
// Pre-start failure: silently hand the utterance to the fallback.
|
|
66
|
+
this.options.onFallback?.(error);
|
|
67
|
+
this.active = this.fallback;
|
|
68
|
+
this.fallback.speak(request, callbacks);
|
|
69
|
+
},
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
pause(): void {
|
|
74
|
+
this.active.pause();
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
resume(): void {
|
|
78
|
+
this.active.resume();
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
stop(): void {
|
|
82
|
+
this.active.stop();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
destroy(): void {
|
|
86
|
+
this.primary.destroy?.();
|
|
87
|
+
this.fallback.destroy?.();
|
|
88
|
+
}
|
|
89
|
+
}
|
package/src/voice/index.ts
CHANGED
|
@@ -14,3 +14,24 @@ export {
|
|
|
14
14
|
createBestAvailableVoiceProvider,
|
|
15
15
|
isVoiceSupported
|
|
16
16
|
} from './voice-factory';
|
|
17
|
+
|
|
18
|
+
export {
|
|
19
|
+
BrowserSpeechEngine,
|
|
20
|
+
pickBestVoice,
|
|
21
|
+
type BrowserSpeechEngineOptions
|
|
22
|
+
} from './browser-speech-engine';
|
|
23
|
+
|
|
24
|
+
// NOTE: RuntypeSpeechEngine is intentionally NOT re-exported here. It pairs with
|
|
25
|
+
// the AudioWorklet player (`worklet-playback-engine`) and ships from the
|
|
26
|
+
// `@runtypelabs/persona/voice-worklet-player` subpath; `session.ts` lazy-imports
|
|
27
|
+
// it so its module only evaluates when `provider: 'runtype'` actually plays audio.
|
|
28
|
+
// FallbackSpeechEngine's value export lives on the
|
|
29
|
+
// `@runtypelabs/persona/voice-worklet-player` subpath so the hosted read-aloud
|
|
30
|
+
// path can be deferred to the `runtype-tts.js` chunk; only the (erased) type is
|
|
31
|
+
// re-exported here for `session.ts`.
|
|
32
|
+
export type { FallbackSpeechEngineOptions } from './fallback-speech-engine';
|
|
33
|
+
|
|
34
|
+
export {
|
|
35
|
+
ReadAloudController,
|
|
36
|
+
type ReadAloudListener
|
|
37
|
+
} from './read-aloud-controller';
|