@runtypelabs/persona 4.24.0 → 4.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/animations/glyph-cycle.d.cts +1 -1
- package/dist/animations/glyph-cycle.d.ts +1 -1
- package/dist/animations/{types-Db61aI9J.d.cts → types-BwKTZhg9.d.cts} +14 -0
- package/dist/animations/{types-Db61aI9J.d.ts → types-BwKTZhg9.d.ts} +14 -0
- package/dist/animations/wipe.d.cts +1 -1
- package/dist/animations/wipe.d.ts +1 -1
- package/dist/animations-extra.d.cts +14 -0
- package/dist/animations-extra.d.ts +14 -0
- package/dist/approval-ui-QAKWNFNM.js +1 -0
- package/dist/approval-ui.cjs +2 -2
- package/dist/approval-ui.d.cts +243 -8
- package/dist/approval-ui.d.ts +243 -8
- package/dist/approval-ui.js +2 -2
- package/dist/{artifacts-ui-IGB5YKLO.js → artifacts-ui-NJGMM27I.js} +1 -1
- package/dist/artifacts-ui.cjs +4 -4
- package/dist/artifacts-ui.d.cts +243 -8
- package/dist/artifacts-ui.d.ts +243 -8
- package/dist/artifacts-ui.js +5 -5
- package/dist/chunk-2ANTRKTV.js +1 -0
- package/dist/chunk-4HPTPUC6.js +1 -0
- package/dist/chunk-6V3MNEKB.js +1 -0
- package/dist/chunk-HGUCGM3K.js +9 -0
- package/dist/chunk-IO5VVUKP.js +3 -0
- package/dist/chunk-J4J3Y2P2.js +1 -0
- package/dist/chunk-K6HKIP5K.js +1 -0
- package/dist/chunk-K7NA7G7J.js +1 -0
- package/dist/chunk-MXUTS3LB.js +1 -0
- package/dist/chunk-PDPVORUF.js +1 -0
- package/dist/chunk-QPPJUSZH.js +1 -0
- package/dist/chunk-QRKGQPU7.js +1 -0
- package/dist/chunk-S6PBWE75.js +1 -0
- package/dist/chunk-T7QM7YMF.js +1 -0
- package/dist/chunk-TR2H475E.js +1 -0
- package/dist/chunk-UPO4GUFC.js +1 -0
- package/dist/chunk-ZC66NAIR.js +1 -0
- package/dist/client-history-YPEG45Q4.js +1 -0
- package/dist/client-history.js +1 -0
- package/dist/client-stream-42FOI3EC.js +4 -0
- package/dist/client-stream.js +5 -0
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/context-mentions-E7HJYWO6.js +161 -0
- package/dist/{context-mentions-inline-I2RDEBAK.js → context-mentions-inline-QVESLW55.js} +1 -1
- package/dist/context-mentions.cjs +7 -7
- package/dist/context-mentions.d.cts +243 -8
- package/dist/context-mentions.d.ts +243 -8
- package/dist/context-mentions.js +8 -8
- package/dist/event-stream-view-FOIJHFQO.js +1 -0
- package/dist/event-stream-view.cjs +1 -1
- package/dist/event-stream-view.d.cts +246 -31
- package/dist/event-stream-view.d.ts +246 -31
- package/dist/event-stream-view.js +1 -1
- package/dist/forms-ui.d.cts +295 -667
- package/dist/forms-ui.d.ts +295 -667
- package/dist/history-shell-QDQ44PSE.js +3 -0
- package/dist/history-shell.js +3 -0
- package/dist/{history-view-BD6OXLNA.js → history-view-NCFBI5GR.js} +2 -2
- package/dist/history-view.cjs +2 -2
- package/dist/history-view.d.cts +35 -1
- package/dist/history-view.d.ts +35 -1
- package/dist/history-view.js +2 -2
- package/dist/index.cjs +80 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +301 -738
- package/dist/index.d.ts +301 -738
- package/dist/index.global.js +3 -215
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +81 -75
- package/dist/index.js.map +1 -1
- package/dist/install.global.js +1 -1
- package/dist/install.global.js.map +1 -1
- package/dist/launcher.global.js +3 -67
- package/dist/launcher.global.js.map +1 -1
- package/dist/markdown-parsers.js +22 -22
- package/dist/runtype-tts-entry-NFBVWR7N.js +1 -0
- package/dist/runtype-tts.js +1 -1
- package/dist/session-actions-ISSKSI66.js +1 -0
- package/dist/session-actions.js +1 -0
- package/dist/session-reconnect.d.cts +243 -8
- package/dist/session-reconnect.d.ts +243 -8
- package/dist/smart-dom-reader.cjs +13 -13
- package/dist/smart-dom-reader.d.cts +243 -8
- package/dist/smart-dom-reader.d.ts +243 -8
- package/dist/smart-dom-reader.js +11 -11
- package/dist/theme-editor-preview.cjs +116 -98
- package/dist/theme-editor-preview.d.cts +245 -8
- package/dist/theme-editor-preview.d.ts +245 -8
- package/dist/theme-editor-preview.js +49 -54
- package/dist/theme-editor.cjs +2 -2
- package/dist/theme-editor.d.cts +244 -23
- package/dist/theme-editor.d.ts +244 -23
- package/dist/theme-editor.js +2 -2
- package/dist/theme-reference.d.cts +14 -0
- package/dist/theme-reference.d.ts +14 -0
- package/dist/ui-extras-entry-WAYFKVNW.js +3 -0
- package/dist/ui-extras.js +72 -0
- package/dist/voice-runtime-UOQOFAFY.js +17 -0
- package/dist/voice-runtime.cjs +17 -1
- package/dist/voice-runtime.d.cts +11643 -192
- package/dist/voice-runtime.d.ts +11643 -192
- package/dist/voice-runtime.js +17 -1
- package/dist/voice-worklet-player.cjs +19 -4
- package/dist/voice-worklet-player.d.cts +9 -10
- package/dist/voice-worklet-player.d.ts +9 -10
- package/dist/voice-worklet-player.js +19 -4
- package/dist/webmcp-polyfill.js +1 -1
- package/dist/webmcp-runtime-V6ZWW3PI.js +6 -0
- package/dist/webmcp-runtime.cjs +2 -2
- package/dist/webmcp-runtime.d.cts +1 -33
- package/dist/webmcp-runtime.d.ts +1 -33
- package/dist/webmcp-runtime.js +2 -2
- package/dist/widget.css +1 -1
- package/package.json +11 -5
- package/src/approval-paused-tool-settle.test.ts +221 -0
- package/src/client-approve.test.ts +311 -0
- package/src/client-history-eager.ts +9 -0
- package/src/client-history-loader.ts +23 -0
- package/src/client-history.ts +579 -0
- package/src/client-resume.ts +177 -0
- package/src/client-stream-eager.ts +12 -0
- package/src/client-stream-loader.ts +25 -0
- package/src/client-stream.ts +1880 -0
- package/src/client.test.ts +31 -1
- package/src/client.ts +293 -2607
- package/src/components/approval-actions.test.ts +27 -0
- package/src/components/approval-actions.ts +27 -3
- package/src/components/message-bubble.test.ts +19 -0
- package/src/components/message-bubble.ts +2 -0
- package/src/components/registry.ts +6 -6
- package/src/components/tool-bubble.test.ts +9 -0
- package/src/components/tool-bubble.ts +1 -0
- package/src/history-shell-bundle.test.ts +54 -0
- package/src/history-shell-eager.ts +12 -0
- package/src/history-shell-loader.ts +27 -0
- package/src/history-shell.ts +2362 -0
- package/src/history-view-loader.ts +2 -0
- package/src/history-view.ts +3 -0
- package/src/index-core.ts +8 -0
- package/src/index-global.ts +90 -248
- package/src/index.ts +13 -0
- package/src/install.ts +6 -4
- package/src/internal/runtype-history-provider.test.ts +1 -0
- package/src/internal/runtype-history-provider.ts +28 -13
- package/src/plugins/registry.ts +8 -8
- package/src/session-actions-chunk.test.ts +55 -0
- package/src/session-actions-eager.ts +11 -0
- package/src/session-actions-loader.ts +37 -0
- package/src/session-actions.ts +909 -0
- package/src/session-history.test.ts +3 -1
- package/src/session.ts +875 -1530
- package/src/session.voice.test.ts +1241 -5
- package/src/styles/widget.css +58 -0
- package/src/theme-editor/state.ts +91 -91
- package/src/types.ts +269 -5
- package/src/ui-ask-user-sheet.ts +382 -0
- package/src/ui-extras-eager.ts +10 -0
- package/src/ui-extras-entry.ts +9 -0
- package/src/ui-extras-loader.ts +26 -0
- package/src/ui.artifact-rail-resize.test.ts +2 -1
- package/src/ui.ask-user-chunk.test.ts +93 -0
- package/src/ui.history-chunk.test.ts +2 -1
- package/src/ui.history-config-provider.test.ts +2 -1
- package/src/ui.history-confirm-fallback.test.ts +142 -0
- package/src/ui.history-controller.test.ts +2 -1
- package/src/ui.history-lifecycle.test.ts +2 -1
- package/src/ui.history-render-hooks.test.ts +2 -1
- package/src/ui.history-shell.test.ts +2 -1
- package/src/ui.rail-overlay.test.ts +2 -1
- package/src/ui.rail-resize.test.ts +2 -1
- package/src/ui.rail-trigger-pending.test.ts +2 -1
- package/src/ui.ts +399 -2770
- package/src/ui.voice-prewarm.test.ts +222 -0
- package/src/utils/attachment-manager.ts +67 -67
- package/src/utils/constants.ts +3 -0
- package/src/utils/context-mention-controller.ts +323 -323
- package/src/utils/context-mention-manager.ts +40 -40
- package/src/utils/context-mention-orchestrator.ts +28 -0
- package/src/utils/event-stream-buffer.ts +50 -50
- package/src/utils/event-stream-store.ts +43 -43
- package/src/utils/throughput-tracker.ts +34 -34
- package/src/utils/virtual-scroller.ts +67 -67
- package/src/utils/visitor-store.test.ts +50 -0
- package/src/utils/visitor-store.ts +23 -5
- package/src/voice/__fixtures__/voice-wire/README.md +1 -0
- package/src/voice/__fixtures__/voice-wire/client/cancel.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/context.json +4 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_result.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_update.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/ping.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/playback_progress.json +5 -0
- package/src/voice/__fixtures__/voice-wire/client/start.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/artifact.json +11 -0
- package/src/voice/__fixtures__/voice-wire/server/attached.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_clear.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_end.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_cancelled.json +5 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_completed.json +8 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_started.json +15 -0
- package/src/voice/__fixtures__/voice-wire/server/error.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/metrics.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/pong.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/session_config.json +16 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_final.json +7 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_interim.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_partial.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_update.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/warning.json +5 -0
- package/src/voice/audio-playback-manager.test.ts +57 -0
- package/src/voice/audio-playback-manager.ts +120 -98
- package/src/voice/browser-voice-provider.ts +37 -37
- package/src/voice/fallback-speech-engine.ts +22 -16
- package/src/voice/keyed-voice-transcript.ts +314 -0
- package/src/voice/mic-state-styles.ts +182 -0
- package/src/voice/read-aloud-controller.ts +46 -44
- package/src/voice/runtype-speech-engine.ts +35 -33
- package/src/voice/runtype-voice-provider.ts +856 -164
- package/src/voice/session-voice-wiring.ts +266 -0
- package/src/voice/voice-delegation.test.ts +71 -0
- package/src/voice/voice-delegation.ts +364 -0
- package/src/voice/voice-turn-tracker.ts +6 -6
- package/src/voice/voice.test.ts +1250 -7
- package/src/voice/worklet-playback-engine.test.ts +81 -0
- package/src/voice/worklet-playback-engine.ts +21 -3
- package/src/voice-runtime-loader.ts +4 -1
- package/src/voice-runtime.ts +7 -0
- package/src/webmcp-runtime-entry.ts +41 -39
- package/dist/approval-ui-3NMYNXBD.js +0 -1
- package/dist/chunk-5EIIHQLQ.js +0 -1
- package/dist/chunk-7CO4T5KB.js +0 -1
- package/dist/chunk-EFJBUH5F.js +0 -1
- package/dist/chunk-R2TMIMDI.js +0 -3
- package/dist/chunk-UTVLOTWB.js +0 -9
- package/dist/context-mentions-45MODWOY.js +0 -161
- package/dist/event-stream-view-GL3ADRY2.js +0 -1
- package/dist/runtype-tts-entry-UJAEF7NZ.js +0 -1
- package/dist/voice-runtime-EFR3R5ES.js +0 -1
- package/dist/webmcp-runtime-JI7T4QYF.js +0 -6
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "session_config",
|
|
3
|
+
"interruptionMode": "barge-in",
|
|
4
|
+
"speechMode": "speech_to_speech",
|
|
5
|
+
"protocolVersion": "runtype-browser-v1",
|
|
6
|
+
"capabilities": ["client_delegation", "context", "delegation_update"],
|
|
7
|
+
"audio": {
|
|
8
|
+
"inputSampleRate": 16000,
|
|
9
|
+
"outputSampleRate": 24000,
|
|
10
|
+
"encoding": "pcm16"
|
|
11
|
+
},
|
|
12
|
+
"clientDelegation": true,
|
|
13
|
+
"contextFrames": true,
|
|
14
|
+
"followUpFrames": true,
|
|
15
|
+
"callId": "call_01JABCDEF"
|
|
16
|
+
}
|
|
@@ -91,6 +91,40 @@ describe("AudioPlaybackManager (PcmStreamPlayer surface)", () => {
|
|
|
91
91
|
expect(started).toHaveBeenCalledTimes(1);
|
|
92
92
|
});
|
|
93
93
|
|
|
94
|
+
it("continuous mode releases a held tail after a prebuffer of quiet, and stays usable", () => {
|
|
95
|
+
vi.useFakeTimers();
|
|
96
|
+
try {
|
|
97
|
+
const m = new AudioPlaybackManager(24000, { prebufferMs: 200 });
|
|
98
|
+
m.setContinuousMode(true);
|
|
99
|
+
m.enqueue(pcm(500)); // below the waterline and no markStreamEnd will ever come
|
|
100
|
+
vi.advanceTimersByTime(150);
|
|
101
|
+
m.enqueue(pcm(500)); // more input restarts the quiet window
|
|
102
|
+
vi.advanceTimersByTime(150);
|
|
103
|
+
expect(MockAudioContext.instances).toHaveLength(0);
|
|
104
|
+
vi.advanceTimersByTime(50);
|
|
105
|
+
const ctx = MockAudioContext.instances[0];
|
|
106
|
+
expect(ctx.sources).toHaveLength(2);
|
|
107
|
+
|
|
108
|
+
// The next reply still plays through the same, never-ended stream.
|
|
109
|
+
m.enqueue(pcm(100));
|
|
110
|
+
expect(ctx.sources).toHaveLength(3);
|
|
111
|
+
} finally {
|
|
112
|
+
vi.useRealTimers();
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it("without continuous mode a short held tail waits for markStreamEnd", () => {
|
|
117
|
+
vi.useFakeTimers();
|
|
118
|
+
try {
|
|
119
|
+
const m = new AudioPlaybackManager(24000, { prebufferMs: 200 });
|
|
120
|
+
m.enqueue(pcm(500));
|
|
121
|
+
vi.advanceTimersByTime(1000);
|
|
122
|
+
expect(MockAudioContext.instances).toHaveLength(0);
|
|
123
|
+
} finally {
|
|
124
|
+
vi.useRealTimers();
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
|
|
94
128
|
it("plays a reply shorter than the prebuffer (markStreamEnd flushes the gate)", () => {
|
|
95
129
|
const m = new AudioPlaybackManager(24000, { prebufferMs: 200 });
|
|
96
130
|
const finished = vi.fn();
|
|
@@ -156,4 +190,27 @@ describe("AudioPlaybackManager (PcmStreamPlayer surface)", () => {
|
|
|
156
190
|
m.enqueue(pcm(100));
|
|
157
191
|
expect(startedAgain).toHaveBeenCalledTimes(1);
|
|
158
192
|
});
|
|
193
|
+
|
|
194
|
+
it("ignores onended from sources a flush stopped, so the next reply drains on its own audio", () => {
|
|
195
|
+
const m = new AudioPlaybackManager(24000);
|
|
196
|
+
m.enqueue(pcm(100));
|
|
197
|
+
m.enqueue(pcm(100));
|
|
198
|
+
const ctx = MockAudioContext.instances[0];
|
|
199
|
+
// Browsers fire onended for stopped sources after flush(); capture the
|
|
200
|
+
// handlers as the browser would have them queued.
|
|
201
|
+
const staleEnded = ctx.sources.map((s) => s.onended);
|
|
202
|
+
m.flush();
|
|
203
|
+
expect(ctx.sources.every((s) => s.onended === null)).toBe(true);
|
|
204
|
+
|
|
205
|
+
const finished = vi.fn();
|
|
206
|
+
m.onFinished(finished);
|
|
207
|
+
m.enqueue(pcm(100)); // the next reply
|
|
208
|
+
m.markStreamEnd();
|
|
209
|
+
for (const ended of staleEnded) ended?.();
|
|
210
|
+
expect(finished).not.toHaveBeenCalled();
|
|
211
|
+
expect(m.isPlaying()).toBe(true);
|
|
212
|
+
|
|
213
|
+
ctx.sources[2].onended?.();
|
|
214
|
+
expect(finished).toHaveBeenCalledTimes(1);
|
|
215
|
+
});
|
|
159
216
|
});
|
|
@@ -27,59 +27,63 @@ import type { PcmStreamPlayer } from "../types";
|
|
|
27
27
|
* approximation of the worklet's audio-thread silence, not parity.
|
|
28
28
|
*/
|
|
29
29
|
export class AudioPlaybackManager implements PcmStreamPlayer {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
30
|
+
#ctx: AudioContext | null = null;
|
|
31
|
+
#nextStartTime = 0;
|
|
32
|
+
#activeSources: AudioBufferSourceNode[] = [];
|
|
33
|
+
#finishedCallbacks: (() => void)[] = [];
|
|
34
|
+
#startedCallbacks: (() => void)[] = [];
|
|
35
|
+
#playing = false;
|
|
36
|
+
#streamEnded = false;
|
|
37
|
+
#pendingCount = 0;
|
|
38
38
|
// Fires once per playback session when the first sample is actually scheduled
|
|
39
39
|
// (loading→playing). Cleared by flush(); a mid-reply underrun does not re-fire.
|
|
40
|
-
|
|
40
|
+
#started = false;
|
|
41
41
|
// Explicit user pause via pause(); kept separate from the AudioContext's
|
|
42
42
|
// autoplay-policy suspension so ensureContext() doesn't auto-resume over it.
|
|
43
|
-
|
|
43
|
+
#userPaused = false;
|
|
44
44
|
|
|
45
45
|
// Prebuffer gate: while `buffering`, incoming samples accumulate in
|
|
46
46
|
// `pendingBuffers` until they reach `waterlineSamples`, then release into the
|
|
47
47
|
// scheduler. With `waterlineSamples === 0` the gate is off (realtime default).
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
#buffering: boolean;
|
|
49
|
+
#pendingBuffers: Float32Array[] = [];
|
|
50
|
+
#pendingSamples = 0;
|
|
51
|
+
// Continuous streams never call markStreamEnd: a held tail is released after one
|
|
52
|
+
// prebuffer's worth of quiet instead.
|
|
53
|
+
#continuous = false;
|
|
54
|
+
#tailTimer: ReturnType<typeof setTimeout> | undefined;
|
|
51
55
|
|
|
52
56
|
// PCM format constants
|
|
53
|
-
|
|
54
|
-
|
|
57
|
+
readonly #sampleRate: number;
|
|
58
|
+
readonly #waterlineSamples: number;
|
|
55
59
|
|
|
56
60
|
// Remainder byte from a previous chunk when the chunk had an odd byte count.
|
|
57
61
|
// Network chunks don't respect 2-byte sample boundaries, so we carry over
|
|
58
62
|
// the orphaned byte and prepend it to the next chunk.
|
|
59
|
-
|
|
63
|
+
#remainder: Uint8Array | null = null;
|
|
60
64
|
|
|
61
65
|
constructor(sampleRate = 24000, options: { prebufferMs?: number } = {}) {
|
|
62
|
-
this
|
|
66
|
+
this.#sampleRate = sampleRate;
|
|
63
67
|
const prebufferMs = Math.max(0, options.prebufferMs ?? 0);
|
|
64
|
-
this
|
|
65
|
-
this
|
|
68
|
+
this.#waterlineSamples = Math.round((sampleRate * prebufferMs) / 1000);
|
|
69
|
+
this.#buffering = this.#waterlineSamples > 0;
|
|
66
70
|
}
|
|
67
71
|
|
|
68
72
|
/**
|
|
69
73
|
* Ensure AudioContext is created and running.
|
|
70
74
|
* Must be called after a user gesture on iOS Safari.
|
|
71
75
|
*/
|
|
72
|
-
|
|
73
|
-
if (!this
|
|
76
|
+
#ensureContext(): AudioContext {
|
|
77
|
+
if (!this.#ctx) {
|
|
74
78
|
const w = typeof window !== "undefined" ? (window as any) : undefined;
|
|
75
79
|
if (!w) throw new Error("AudioPlaybackManager requires a browser environment");
|
|
76
80
|
const AudioCtx = w.AudioContext || w.webkitAudioContext;
|
|
77
|
-
this
|
|
81
|
+
this.#ctx = new AudioCtx({ sampleRate: this.#sampleRate }) as AudioContext;
|
|
78
82
|
}
|
|
79
|
-
const ctx = this
|
|
83
|
+
const ctx = this.#ctx!;
|
|
80
84
|
// Resume if suspended (autoplay policy) — but never override an explicit
|
|
81
85
|
// user pause(): more audio may still stream in while paused.
|
|
82
|
-
if (ctx.state === "suspended" && !this
|
|
86
|
+
if (ctx.state === "suspended" && !this.#userPaused) {
|
|
83
87
|
ctx.resume();
|
|
84
88
|
}
|
|
85
89
|
return ctx;
|
|
@@ -94,32 +98,39 @@ export class AudioPlaybackManager implements PcmStreamPlayer {
|
|
|
94
98
|
|
|
95
99
|
// Prepend any remainder byte from the previous chunk
|
|
96
100
|
let data = pcmData;
|
|
97
|
-
if (this
|
|
98
|
-
const merged = new Uint8Array(this
|
|
99
|
-
merged.set(this
|
|
100
|
-
merged.set(pcmData, this
|
|
101
|
+
if (this.#remainder) {
|
|
102
|
+
const merged = new Uint8Array(this.#remainder.length + pcmData.length);
|
|
103
|
+
merged.set(this.#remainder);
|
|
104
|
+
merged.set(pcmData, this.#remainder.length);
|
|
101
105
|
data = merged;
|
|
102
|
-
this
|
|
106
|
+
this.#remainder = null;
|
|
103
107
|
}
|
|
104
108
|
|
|
105
109
|
// If odd byte count, save the trailing byte for next chunk
|
|
106
110
|
if (data.length % 2 !== 0) {
|
|
107
|
-
this
|
|
111
|
+
this.#remainder = new Uint8Array([data[data.length - 1]]);
|
|
108
112
|
data = data.subarray(0, data.length - 1);
|
|
109
113
|
}
|
|
110
114
|
|
|
111
115
|
if (data.length === 0) return;
|
|
112
116
|
|
|
113
|
-
const float32 = this
|
|
117
|
+
const float32 = this.#pcmToFloat32(data);
|
|
114
118
|
if (float32.length === 0) return;
|
|
115
119
|
|
|
116
|
-
if (this
|
|
120
|
+
if (this.#buffering) {
|
|
117
121
|
// Hold until the prebuffer waterline fills, then release as a batch.
|
|
118
|
-
this
|
|
119
|
-
this
|
|
120
|
-
|
|
122
|
+
this.#pendingBuffers.push(float32);
|
|
123
|
+
this.#pendingSamples += float32.length;
|
|
124
|
+
clearTimeout(this.#tailTimer);
|
|
125
|
+
if (this.#pendingSamples >= this.#waterlineSamples) this.#releaseBuffer();
|
|
126
|
+
else if (this.#continuous) {
|
|
127
|
+
this.#tailTimer = setTimeout(
|
|
128
|
+
() => this.#releaseBuffer(),
|
|
129
|
+
(this.#waterlineSamples / this.#sampleRate) * 1000,
|
|
130
|
+
);
|
|
131
|
+
}
|
|
121
132
|
} else {
|
|
122
|
-
this
|
|
133
|
+
this.#scheduleSamples(float32);
|
|
123
134
|
}
|
|
124
135
|
}
|
|
125
136
|
|
|
@@ -130,16 +141,19 @@ export class AudioPlaybackManager implements PcmStreamPlayer {
|
|
|
130
141
|
markStreamEnd(): void {
|
|
131
142
|
// A reply shorter than the prebuffer never reaches the waterline; release
|
|
132
143
|
// whatever we held so it still plays.
|
|
133
|
-
if (this
|
|
134
|
-
this
|
|
135
|
-
this
|
|
144
|
+
if (this.#pendingBuffers.length > 0) this.#releaseBuffer();
|
|
145
|
+
this.#streamEnded = true;
|
|
146
|
+
this.#checkFinished();
|
|
136
147
|
}
|
|
137
148
|
|
|
138
149
|
/**
|
|
139
150
|
* Immediately stop all playback and discard queued audio.
|
|
140
151
|
*/
|
|
141
152
|
flush(): void {
|
|
142
|
-
for (const source of this
|
|
153
|
+
for (const source of this.#activeSources) {
|
|
154
|
+
// A stopped source still fires `onended` later; detach it so a flushed
|
|
155
|
+
// batch can't decrement the next reply's pendingCount and drain it early.
|
|
156
|
+
source.onended = null;
|
|
143
157
|
try {
|
|
144
158
|
source.stop();
|
|
145
159
|
source.disconnect();
|
|
@@ -147,33 +161,39 @@ export class AudioPlaybackManager implements PcmStreamPlayer {
|
|
|
147
161
|
// Ignore errors from already-stopped sources
|
|
148
162
|
}
|
|
149
163
|
}
|
|
150
|
-
this
|
|
151
|
-
this
|
|
152
|
-
this
|
|
153
|
-
this
|
|
154
|
-
this
|
|
155
|
-
this
|
|
156
|
-
this
|
|
157
|
-
this
|
|
164
|
+
this.#activeSources = [];
|
|
165
|
+
this.#pendingCount = 0;
|
|
166
|
+
this.#nextStartTime = 0;
|
|
167
|
+
this.#playing = false;
|
|
168
|
+
this.#streamEnded = false;
|
|
169
|
+
this.#finishedCallbacks = [];
|
|
170
|
+
this.#startedCallbacks = [];
|
|
171
|
+
this.#remainder = null;
|
|
158
172
|
// Reset the prebuffer gate and the started latch for the next reply.
|
|
159
|
-
this
|
|
160
|
-
this
|
|
161
|
-
this
|
|
162
|
-
this
|
|
173
|
+
this.#pendingBuffers = [];
|
|
174
|
+
this.#pendingSamples = 0;
|
|
175
|
+
this.#buffering = this.#waterlineSamples > 0;
|
|
176
|
+
this.#started = false;
|
|
177
|
+
clearTimeout(this.#tailTimer);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** See {@link VoicePlaybackEngine.setContinuousMode}. */
|
|
181
|
+
setContinuousMode(enabled: boolean): void {
|
|
182
|
+
this.#continuous = enabled;
|
|
163
183
|
}
|
|
164
184
|
|
|
165
185
|
/**
|
|
166
186
|
* Whether audio is currently playing or queued.
|
|
167
187
|
*/
|
|
168
188
|
isPlaying(): boolean {
|
|
169
|
-
return this
|
|
189
|
+
return this.#playing;
|
|
170
190
|
}
|
|
171
191
|
|
|
172
192
|
/**
|
|
173
193
|
* Register a callback for when all queued audio finishes playing.
|
|
174
194
|
*/
|
|
175
195
|
onFinished(callback: () => void): void {
|
|
176
|
-
this
|
|
196
|
+
this.#finishedCallbacks.push(callback);
|
|
177
197
|
}
|
|
178
198
|
|
|
179
199
|
/**
|
|
@@ -182,7 +202,7 @@ export class AudioPlaybackManager implements PcmStreamPlayer {
|
|
|
182
202
|
* mid-reply underrun re-buffer does not re-fire it.
|
|
183
203
|
*/
|
|
184
204
|
onStarted(callback: () => void): void {
|
|
185
|
-
this
|
|
205
|
+
this.#startedCallbacks.push(callback);
|
|
186
206
|
}
|
|
187
207
|
|
|
188
208
|
/**
|
|
@@ -190,14 +210,14 @@ export class AudioPlaybackManager implements PcmStreamPlayer {
|
|
|
190
210
|
* freezes in place and {@link resume} continues exactly where it left off.
|
|
191
211
|
*/
|
|
192
212
|
pause(): void {
|
|
193
|
-
this
|
|
194
|
-
if (this
|
|
213
|
+
this.#userPaused = true;
|
|
214
|
+
if (this.#ctx && this.#ctx.state === "running") void this.#ctx.suspend();
|
|
195
215
|
}
|
|
196
216
|
|
|
197
217
|
/** Resume playback after {@link pause}. */
|
|
198
218
|
resume(): void {
|
|
199
|
-
this
|
|
200
|
-
if (this
|
|
219
|
+
this.#userPaused = false;
|
|
220
|
+
if (this.#ctx && this.#ctx.state === "suspended") void this.#ctx.resume();
|
|
201
221
|
}
|
|
202
222
|
|
|
203
223
|
/**
|
|
@@ -205,27 +225,27 @@ export class AudioPlaybackManager implements PcmStreamPlayer {
|
|
|
205
225
|
*/
|
|
206
226
|
async destroy(): Promise<void> {
|
|
207
227
|
this.flush();
|
|
208
|
-
if (this
|
|
209
|
-
await this
|
|
210
|
-
this
|
|
228
|
+
if (this.#ctx) {
|
|
229
|
+
await this.#ctx.close();
|
|
230
|
+
this.#ctx = null;
|
|
211
231
|
}
|
|
212
232
|
}
|
|
213
233
|
|
|
214
234
|
/** Release held prebuffer samples into the scheduler in arrival order. */
|
|
215
|
-
|
|
216
|
-
this
|
|
217
|
-
const held = this
|
|
218
|
-
this
|
|
219
|
-
this
|
|
220
|
-
for (const samples of held) this
|
|
235
|
+
#releaseBuffer(): void {
|
|
236
|
+
this.#buffering = false;
|
|
237
|
+
const held = this.#pendingBuffers;
|
|
238
|
+
this.#pendingBuffers = [];
|
|
239
|
+
this.#pendingSamples = 0;
|
|
240
|
+
for (const samples of held) this.#scheduleSamples(samples);
|
|
221
241
|
}
|
|
222
242
|
|
|
223
243
|
/** Schedule one Float32 sample block for gap-free playback. */
|
|
224
|
-
|
|
244
|
+
#scheduleSamples(float32: Float32Array): void {
|
|
225
245
|
if (float32.length === 0) return;
|
|
226
|
-
const ctx = this
|
|
246
|
+
const ctx = this.#ensureContext();
|
|
227
247
|
|
|
228
|
-
const buffer = ctx.createBuffer(1, float32.length, this
|
|
248
|
+
const buffer = ctx.createBuffer(1, float32.length, this.#sampleRate);
|
|
229
249
|
buffer.getChannelData(0).set(float32);
|
|
230
250
|
|
|
231
251
|
const source = ctx.createBufferSource();
|
|
@@ -233,52 +253,54 @@ export class AudioPlaybackManager implements PcmStreamPlayer {
|
|
|
233
253
|
source.connect(ctx.destination);
|
|
234
254
|
|
|
235
255
|
const now = ctx.currentTime;
|
|
236
|
-
if (this
|
|
256
|
+
if (this.#nextStartTime === 0) {
|
|
237
257
|
// Fresh start (first audio of this session, or after flush).
|
|
238
|
-
this
|
|
239
|
-
} else if (this
|
|
258
|
+
this.#nextStartTime = now;
|
|
259
|
+
} else if (this.#nextStartTime < now) {
|
|
240
260
|
// Underrun: the playhead caught up to the queue. Snap to now (a small,
|
|
241
261
|
// unavoidable gap) and, when a prebuffer is configured, re-enter buffering
|
|
242
262
|
// so subsequent chunks re-accumulate before scheduling — collapsing a
|
|
243
263
|
// train of clicks into a single rebuffer.
|
|
244
|
-
this
|
|
245
|
-
if (this
|
|
264
|
+
this.#nextStartTime = now;
|
|
265
|
+
if (this.#waterlineSamples > 0) this.#buffering = true;
|
|
246
266
|
}
|
|
247
|
-
source.start(this
|
|
248
|
-
this
|
|
267
|
+
source.start(this.#nextStartTime);
|
|
268
|
+
this.#nextStartTime += buffer.duration;
|
|
249
269
|
|
|
250
|
-
this
|
|
251
|
-
this
|
|
252
|
-
this
|
|
270
|
+
this.#activeSources.push(source);
|
|
271
|
+
this.#pendingCount++;
|
|
272
|
+
this.#playing = true;
|
|
253
273
|
|
|
254
|
-
if (!this
|
|
255
|
-
this
|
|
256
|
-
const cbs = this
|
|
257
|
-
this
|
|
274
|
+
if (!this.#started) {
|
|
275
|
+
this.#started = true;
|
|
276
|
+
const cbs = this.#startedCallbacks.slice();
|
|
277
|
+
this.#startedCallbacks = [];
|
|
258
278
|
for (const cb of cbs) cb();
|
|
259
279
|
}
|
|
260
280
|
|
|
261
281
|
source.onended = () => {
|
|
262
|
-
const idx = this
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
this.
|
|
282
|
+
const idx = this.#activeSources.indexOf(source);
|
|
283
|
+
// Not active: removed by flush(), so it no longer counts toward a drain.
|
|
284
|
+
if (idx === -1) return;
|
|
285
|
+
this.#activeSources.splice(idx, 1);
|
|
286
|
+
this.#pendingCount--;
|
|
287
|
+
this.#checkFinished();
|
|
266
288
|
};
|
|
267
289
|
}
|
|
268
290
|
|
|
269
|
-
|
|
291
|
+
#checkFinished(): void {
|
|
270
292
|
// Fire once the stream has ended and nothing is scheduled or held. No
|
|
271
293
|
// `playing` precondition: an empty reply (markStreamEnd with no audio) must
|
|
272
294
|
// still resolve to idle, matching the worklet's immediate 'drained'.
|
|
273
295
|
if (
|
|
274
|
-
this
|
|
275
|
-
this
|
|
276
|
-
this
|
|
296
|
+
this.#streamEnded &&
|
|
297
|
+
this.#pendingCount <= 0 &&
|
|
298
|
+
this.#pendingBuffers.length === 0
|
|
277
299
|
) {
|
|
278
|
-
this
|
|
279
|
-
this
|
|
280
|
-
const cbs = this
|
|
281
|
-
this
|
|
300
|
+
this.#playing = false;
|
|
301
|
+
this.#streamEnded = false;
|
|
302
|
+
const cbs = this.#finishedCallbacks.slice();
|
|
303
|
+
this.#finishedCallbacks = [];
|
|
282
304
|
for (const cb of cbs) cb();
|
|
283
305
|
}
|
|
284
306
|
}
|
|
@@ -286,7 +308,7 @@ export class AudioPlaybackManager implements PcmStreamPlayer {
|
|
|
286
308
|
/**
|
|
287
309
|
* Convert 16-bit signed LE PCM to Float32 samples in [-1, 1].
|
|
288
310
|
*/
|
|
289
|
-
|
|
311
|
+
#pcmToFloat32(pcmData: Uint8Array): Float32Array {
|
|
290
312
|
// 2 bytes per sample (16-bit)
|
|
291
313
|
const numSamples = Math.floor(pcmData.length / 2);
|
|
292
314
|
const float32 = new Float32Array(numSamples);
|
|
@@ -5,43 +5,43 @@ import type { VoiceProvider, VoiceResult, VoiceStatus, VoiceConfig } from '../ty
|
|
|
5
5
|
|
|
6
6
|
export class BrowserVoiceProvider implements VoiceProvider {
|
|
7
7
|
type: 'browser' = 'browser';
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
8
|
+
#recognition: any = null;
|
|
9
|
+
#resultCallbacks: ((result: VoiceResult) => void)[] = [];
|
|
10
|
+
#errorCallbacks: ((error: Error) => void)[] = [];
|
|
11
|
+
#statusCallbacks: ((status: VoiceStatus) => void)[] = [];
|
|
12
|
+
#isListening = false;
|
|
13
|
+
#w: any = typeof window !== 'undefined' ? window : undefined;
|
|
14
14
|
|
|
15
15
|
constructor(private config: VoiceConfig['browser'] = {}) {}
|
|
16
16
|
|
|
17
17
|
async connect() {
|
|
18
18
|
// Browser provider doesn't need connection
|
|
19
|
-
this
|
|
19
|
+
this.#statusCallbacks.forEach(cb => cb('connected'));
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
async startListening() {
|
|
23
23
|
try {
|
|
24
|
-
if (this
|
|
24
|
+
if (this.#isListening) {
|
|
25
25
|
throw new Error('Already listening');
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
if (!this
|
|
28
|
+
if (!this.#w) {
|
|
29
29
|
throw new Error('Window object not available');
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
// @ts-ignore - Browser SpeechRecognition API
|
|
33
|
-
const SpeechRecognition = this
|
|
33
|
+
const SpeechRecognition = this.#w!.SpeechRecognition || this.#w!.webkitSpeechRecognition;
|
|
34
34
|
|
|
35
35
|
if (!SpeechRecognition) {
|
|
36
36
|
throw new Error('Browser speech recognition not supported');
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
this
|
|
40
|
-
this
|
|
41
|
-
this
|
|
42
|
-
this
|
|
39
|
+
this.#recognition = new SpeechRecognition();
|
|
40
|
+
this.#recognition.lang = this.config?.language || 'en-US';
|
|
41
|
+
this.#recognition.continuous = this.config?.continuous || false;
|
|
42
|
+
this.#recognition.interimResults = true;
|
|
43
43
|
|
|
44
|
-
this
|
|
44
|
+
this.#recognition.onresult = (event: any) => {
|
|
45
45
|
const transcript = Array.from(event.results)
|
|
46
46
|
.map((result: any) => result[0])
|
|
47
47
|
.map((result: any) => result.transcript)
|
|
@@ -49,7 +49,7 @@ export class BrowserVoiceProvider implements VoiceProvider {
|
|
|
49
49
|
|
|
50
50
|
const isFinal = event.results[event.results.length - 1].isFinal;
|
|
51
51
|
|
|
52
|
-
this
|
|
52
|
+
this.#resultCallbacks.forEach(cb => cb({
|
|
53
53
|
text: transcript,
|
|
54
54
|
confidence: isFinal ? 0.8 : 0.5,
|
|
55
55
|
provider: 'browser'
|
|
@@ -60,55 +60,55 @@ export class BrowserVoiceProvider implements VoiceProvider {
|
|
|
60
60
|
}
|
|
61
61
|
};
|
|
62
62
|
|
|
63
|
-
this
|
|
64
|
-
this
|
|
65
|
-
this
|
|
63
|
+
this.#recognition.onerror = (event: any) => {
|
|
64
|
+
this.#errorCallbacks.forEach(cb => cb(new Error(event.error)));
|
|
65
|
+
this.#statusCallbacks.forEach(cb => cb('error'));
|
|
66
66
|
};
|
|
67
67
|
|
|
68
|
-
this
|
|
69
|
-
this
|
|
70
|
-
this
|
|
68
|
+
this.#recognition.onstart = () => {
|
|
69
|
+
this.#isListening = true;
|
|
70
|
+
this.#statusCallbacks.forEach(cb => cb('listening'));
|
|
71
71
|
};
|
|
72
72
|
|
|
73
|
-
this
|
|
74
|
-
this
|
|
75
|
-
this
|
|
73
|
+
this.#recognition.onend = () => {
|
|
74
|
+
this.#isListening = false;
|
|
75
|
+
this.#statusCallbacks.forEach(cb => cb('idle'));
|
|
76
76
|
};
|
|
77
77
|
|
|
78
|
-
this
|
|
78
|
+
this.#recognition.start();
|
|
79
79
|
|
|
80
80
|
} catch (error) {
|
|
81
|
-
this
|
|
82
|
-
this
|
|
81
|
+
this.#errorCallbacks.forEach(cb => cb(error as Error));
|
|
82
|
+
this.#statusCallbacks.forEach(cb => cb('error'));
|
|
83
83
|
throw error;
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
86
|
|
|
87
87
|
async stopListening() {
|
|
88
|
-
if (this
|
|
89
|
-
this
|
|
90
|
-
this
|
|
88
|
+
if (this.#recognition) {
|
|
89
|
+
this.#recognition.stop();
|
|
90
|
+
this.#recognition = null;
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
-
this
|
|
94
|
-
this
|
|
93
|
+
this.#isListening = false;
|
|
94
|
+
this.#statusCallbacks.forEach(cb => cb('idle'));
|
|
95
95
|
}
|
|
96
96
|
|
|
97
97
|
onResult(callback: (result: VoiceResult) => void): void {
|
|
98
|
-
this
|
|
98
|
+
this.#resultCallbacks.push(callback);
|
|
99
99
|
}
|
|
100
100
|
|
|
101
101
|
onError(callback: (error: Error) => void): void {
|
|
102
|
-
this
|
|
102
|
+
this.#errorCallbacks.push(callback);
|
|
103
103
|
}
|
|
104
104
|
|
|
105
105
|
onStatusChange(callback: (status: VoiceStatus) => void): void {
|
|
106
|
-
this
|
|
106
|
+
this.#statusCallbacks.push(callback);
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
async disconnect(): Promise<void> {
|
|
110
110
|
await this.stopListening();
|
|
111
|
-
this
|
|
111
|
+
this.#statusCallbacks.forEach(cb => cb('disconnected'));
|
|
112
112
|
}
|
|
113
113
|
|
|
114
114
|
// Check if browser supports speech recognition
|