@charivo/tts 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -75,6 +75,95 @@ var WebSpeechLipSyncSimulator = class {
75
75
  }
76
76
  };
77
77
 
78
+ // src/pcm-playback.ts
79
+ function createPcmPlaybackGraph(context) {
80
+ const output = context.createGain();
81
+ output.connect(context.destination);
82
+ const tap = context.createMediaStreamDestination();
83
+ output.connect(tap);
84
+ return { output, lipSyncStream: tap.stream };
85
+ }
86
+ var PcmPlaybackScheduler = class {
87
+ constructor(context, destination, format, callbacks) {
88
+ this.context = context;
89
+ this.destination = destination;
90
+ this.format = format;
91
+ this.callbacks = callbacks;
92
+ }
93
+ nextStartTime = 0;
94
+ /**
95
+ * Bumped by `flush()`. Every source captures the value it was scheduled
96
+ * under, so a discarded utterance's late `onended` cannot report against the
97
+ * one that replaced it.
98
+ */
99
+ generation = 0;
100
+ active = /* @__PURE__ */ new Set();
101
+ /**
102
+ * Whether the scheduler has nothing left to play — nothing sounding, and
103
+ * nothing queued behind it. Derived from the live source set on
104
+ * every call rather than remembered: `enqueue()` clears idleness by adding to
105
+ * that set, so a caller cannot be holding a "drained" answer that the next
106
+ * chunk already invalidated.
107
+ */
108
+ isIdle() {
109
+ return this.active.size === 0;
110
+ }
111
+ /** Schedule one chunk of the declared format after everything already queued. */
112
+ enqueue(pcm) {
113
+ const sampleCount = Math.floor(pcm.byteLength / 2);
114
+ if (sampleCount === 0) {
115
+ return;
116
+ }
117
+ const generation = this.generation;
118
+ const view = new DataView(pcm.buffer, pcm.byteOffset, pcm.byteLength);
119
+ const buffer = this.context.createBuffer(
120
+ 1,
121
+ sampleCount,
122
+ this.format.sampleRate
123
+ );
124
+ const channel = buffer.getChannelData(0);
125
+ for (let index = 0; index < sampleCount; index += 1) {
126
+ channel[index] = view.getInt16(index * 2, true) / 32768;
127
+ }
128
+ const source = this.context.createBufferSource();
129
+ source.buffer = buffer;
130
+ source.connect(this.destination);
131
+ source.onended = () => {
132
+ this.active.delete(source);
133
+ if (generation !== this.generation || this.active.size > 0) {
134
+ return;
135
+ }
136
+ this.callbacks.onPlayingChange(false);
137
+ this.callbacks.onDrain();
138
+ };
139
+ const wasIdle = this.isIdle();
140
+ const startAt = Math.max(this.context.currentTime, this.nextStartTime);
141
+ this.active.add(source);
142
+ source.start(startAt);
143
+ this.nextStartTime = startAt + buffer.duration;
144
+ if (wasIdle) {
145
+ this.callbacks.onPlayingChange(true);
146
+ }
147
+ }
148
+ /** Discard everything scheduled, audible or merely queued. */
149
+ flush() {
150
+ this.generation += 1;
151
+ const wasPlaying = !this.isIdle();
152
+ for (const source of this.active) {
153
+ source.onended = null;
154
+ try {
155
+ source.stop();
156
+ } catch {
157
+ }
158
+ }
159
+ this.active.clear();
160
+ this.nextStartTime = 0;
161
+ if (wasPlaying) {
162
+ this.callbacks.onPlayingChange(false);
163
+ }
164
+ }
165
+ };
166
+
78
167
  // src/tts-utils.ts
79
168
  function getTTSPlaybackMode(player) {
80
169
  return player.playbackMode || "audio";
@@ -85,10 +174,19 @@ function getTTSAudioMimeType(player) {
85
174
  function supportsGenerateAudio(player) {
86
175
  return typeof player.generateAudio === "function";
87
176
  }
177
+ function supportsGenerateAudioStream(player) {
178
+ return typeof player.generateAudioStream === "function";
179
+ }
88
180
 
89
181
  // src/tts-manager.ts
90
182
  var STARTUP_CANCELLED = Symbol("charivo.tts.startupCancelled");
91
183
  var noop = () => void 0;
184
+ function joinChunks(carry, chunk) {
185
+ const joined = new Uint8Array(carry.byteLength + chunk.byteLength);
186
+ joined.set(carry);
187
+ joined.set(chunk, carry.byteLength);
188
+ return joined;
189
+ }
92
190
  var TTSManagerImpl = class {
93
191
  ttsPlayer;
94
192
  eventEmitter;
@@ -97,12 +195,14 @@ var TTSManagerImpl = class {
97
195
  playbackMode;
98
196
  isAudioSessionActive = false;
99
197
  teardownBrowserLifecycle;
100
- // Settles the stateless-audio Promise that handleStatelessAudio() is
101
- // currently awaiting, if any. stop() clears currentAudio's onended/onerror
102
- // handlers so they can never fire on their own; without this, stopping a
103
- // still-playing stateless-audio utterance would strand that speak() call's
104
- // promise forever.
105
- pendingStatelessStop = null;
198
+ // Settles the Promise whichever audio path is currently awaiting -- the
199
+ // buffered one in handleStatelessAudio(), or the streamed one in
200
+ // handleStreamingAudio(). Neither can settle itself once stop() has torn its
201
+ // playback down: stop() clears currentAudio's onended/onerror handlers so
202
+ // they can never fire, and it flushes the scheduler so no source's onended
203
+ // is left to report a drain. Without this nudge, stopping a still-playing
204
+ // utterance would strand that speak() call's promise forever.
205
+ pendingAudioStop = null;
106
206
  // Settles the web-speech Promise handleWebSpeech() is currently awaiting,
107
207
  // if any. Unlike the stateless path, the manager can't sever the
108
208
  // TTSPlayer's own onend/onerror handlers directly -- its completion is
@@ -136,6 +236,16 @@ var TTSManagerImpl = class {
136
236
  // requests share one player stop and one cleanup, and no new utterance can
137
237
  // dispatch until it has fully settled.
138
238
  pendingStopPlayback = null;
239
+ // The Web Audio side, built only for streamed playback -- the buffered path
240
+ // plays through an <audio> element and never touches these. Both outlive a
241
+ // single utterance: one context per manager, one graph per context.
242
+ playbackContext = null;
243
+ playbackGraph = null;
244
+ // The streamed utterance in flight, if any: the three handles needed to
245
+ // reach it -- the queue to discard, the request to abort, and the body to
246
+ // cancel through (never through the body itself, which is locked to this
247
+ // reader).
248
+ currentStream = null;
139
249
  // Only the Web Speech lip-sync simulation is needed
140
250
  webSimulator;
141
251
  // Reads the emitter at emit time, so a later setEventEmitter() still applies.
@@ -146,9 +256,9 @@ var TTSManagerImpl = class {
146
256
  constructor(ttsPlayer) {
147
257
  this.ttsPlayer = ttsPlayer;
148
258
  this.playbackMode = getTTSPlaybackMode(ttsPlayer);
149
- if (this.playbackMode === "audio" && !supportsGenerateAudio(ttsPlayer)) {
259
+ if (this.playbackMode === "audio" && !supportsGenerateAudio(ttsPlayer) && !supportsGenerateAudioStream(ttsPlayer)) {
150
260
  throw new CharivoStateError(
151
- 'TTS playback mode "audio" requires the player to implement generateAudio() so the manager can create and analyze playback for lip-sync. Implement generateAudio() or set playbackMode: "web-speech".'
261
+ 'TTS playback mode "audio" requires the player to implement generateAudio() or generateAudioStream() so the manager can create and analyze playback for lip-sync. Implement one of them or set playbackMode: "web-speech".'
152
262
  );
153
263
  }
154
264
  this.webSimulator = new WebSpeechLipSyncSimulator();
@@ -165,10 +275,16 @@ var TTSManagerImpl = class {
165
275
  *
166
276
  * Resolves when the utterance finishes -- or silently, without speaking, if
167
277
  * stop() or a newer speak() lands while this call is still starting up (the
168
- * pre-speech stop, or synthesis). A cancelled call never begins playback,
169
- * whether or not it had already opened an audio session (a reentrant stop
170
- * from the tts:audio:start listener still sees the session close), so a
171
- * resolved speak() is not proof audio played.
278
+ * pre-speech stop, or synthesis), so a resolved speak() is not proof audio
279
+ * played.
280
+ *
281
+ * On the buffered path a cancelled call never begins playback, whether or
282
+ * not it had already opened an audio session: a reentrant stop from the
283
+ * tts:audio:start listener still sees the session close before anything is
284
+ * handed to the speakers. Streamed playback cannot promise that much,
285
+ * because there tts:audio:start follows the first scheduled sample -- a stop
286
+ * from that listener can land after a few milliseconds of sound. It still
287
+ * settles the call and emits exactly one tts:audio:end.
172
288
  */
173
289
  async speak(text, options) {
174
290
  const generation = ++this.speakGeneration;
@@ -188,6 +304,8 @@ var TTSManagerImpl = class {
188
304
  }
189
305
  if (this.playbackMode === "web-speech") {
190
306
  return await this.handleWebSpeech(text, options, startup);
307
+ } else if (supportsGenerateAudioStream(this.ttsPlayer)) {
308
+ return await this.handleStreamingAudio(text, options, startup);
191
309
  } else {
192
310
  return await this.handleStatelessAudio(text, options, startup);
193
311
  }
@@ -251,9 +369,14 @@ var TTSManagerImpl = class {
251
369
  URL.revokeObjectURL(this.currentAudioUrl);
252
370
  this.currentAudioUrl = null;
253
371
  }
372
+ const streaming = this.currentStream;
373
+ if (streaming) {
374
+ streaming.scheduler.flush();
375
+ this.abandonStream(streaming.abort, streaming.reader);
376
+ }
254
377
  this.lipSync.stop();
255
378
  this.endAudioSession();
256
- this.pendingStatelessStop?.();
379
+ this.pendingAudioStop?.();
257
380
  const settleWebSpeech = this.pendingWebSpeechStop;
258
381
  this.pendingWebSpeechStop = null;
259
382
  settleWebSpeech?.();
@@ -272,22 +395,35 @@ var TTSManagerImpl = class {
272
395
  return this.ttsPlayer.isSupported();
273
396
  }
274
397
  /**
275
- * Create the audio analysis context up front, typically from a user gesture
398
+ * Create the audio contexts up front -- the analyzer's, plus a playback
399
+ * context for a player that streams -- typically from a user gesture
276
400
  * handler so browsers allow playback later. Throws on unsupported browsers.
277
401
  */
278
402
  async prepareAudio() {
279
403
  this.ensureLifecycleBound();
404
+ if (this.playbackMode !== "web-speech" && supportsGenerateAudioStream(this.ttsPlayer)) {
405
+ this.ensurePlaybackContext();
406
+ }
280
407
  await this.lipSync.prepare();
281
408
  }
282
409
  /**
283
- * Release audio resources. Call stop() first: dispose() does not stop playback.
410
+ * Release audio resources. Call stop() first: dispose() does not stop
411
+ * buffered playback -- an <audio> element outlives this manager and settles
412
+ * its own speak() on its own onended. A streamed utterance cannot survive
413
+ * this call, so it is torn down here.
284
414
  */
285
415
  async dispose() {
286
416
  this.webSimulator.dispose();
287
417
  this.teardownBrowserLifecycle?.();
288
418
  this.teardownBrowserLifecycle = void 0;
419
+ if (this.currentStream) {
420
+ this.pendingAudioStop?.();
421
+ }
422
+ const playbackContext = this.playbackContext;
423
+ this.playbackContext = null;
424
+ this.playbackGraph = null;
289
425
  try {
290
- await this.lipSync.cleanup();
426
+ await Promise.all([this.lipSync.cleanup(), playbackContext?.close()]);
291
427
  } catch (error) {
292
428
  throw toCharivoError(
293
429
  "dispose",
@@ -428,7 +564,7 @@ var TTSManagerImpl = class {
428
564
  const finalize = (next) => {
429
565
  if (isFinalized) return;
430
566
  isFinalized = true;
431
- this.pendingStatelessStop = null;
567
+ this.pendingAudioStop = null;
432
568
  if (this.currentAudioUrl) {
433
569
  URL.revokeObjectURL(this.currentAudioUrl);
434
570
  this.currentAudioUrl = null;
@@ -445,7 +581,7 @@ var TTSManagerImpl = class {
445
581
  finalize(() => reject(new Error("Audio playback failed")));
446
582
  };
447
583
  const dispatch = this.beginUtterance(() => {
448
- this.pendingStatelessStop = () => finalize(resolve);
584
+ this.pendingAudioStop = () => finalize(resolve);
449
585
  }, startup);
450
586
  if (!dispatch) {
451
587
  finalize(resolve);
@@ -460,6 +596,164 @@ var TTSManagerImpl = class {
460
596
  });
461
597
  });
462
598
  }
599
+ /**
600
+ * Streamed audio handling: PCM is scheduled into Web Audio as it arrives,
601
+ * instead of waiting for one finished buffer.
602
+ */
603
+ async handleStreamingAudio(text, options, startup) {
604
+ const context = this.ensurePlaybackContext();
605
+ const abort = new AbortController();
606
+ const opening = this.ttsPlayer.generateAudioStream(
607
+ text,
608
+ options,
609
+ abort.signal
610
+ ).catch(
611
+ (error) => Promise.reject(
612
+ toCharivoError("provider", error, "Failed to generate TTS audio")
613
+ )
614
+ );
615
+ const stream = await this.raceStartup(
616
+ opening,
617
+ startup.cancelled,
618
+ startup.isCancelled
619
+ );
620
+ if (stream === STARTUP_CANCELLED) {
621
+ abort.abort();
622
+ void opening.then((late) => void late.body.cancel().catch(noop), noop);
623
+ return;
624
+ }
625
+ if (stream.format.channels !== 1) {
626
+ void stream.body.cancel().catch(noop);
627
+ abort.abort();
628
+ throw new CharivoStateError(
629
+ `Streamed TTS audio must be mono, but the player answered ${stream.format.channels} channels.`
630
+ );
631
+ }
632
+ const graph = this.ensurePlaybackGraph(context);
633
+ graph.output.gain.value = options?.volume !== void 0 ? Math.max(0, Math.min(1, options.volume)) : 1;
634
+ this.ensureLifecycleBound();
635
+ this.lipSync.attachMediaStream(graph.lipSyncStream);
636
+ const reader = stream.body.getReader();
637
+ return new Promise((resolve, reject) => {
638
+ let bodyEnded = false;
639
+ let isFinalized = false;
640
+ const scheduler = new PcmPlaybackScheduler(
641
+ context,
642
+ graph.output,
643
+ stream.format,
644
+ {
645
+ onDrain: () => {
646
+ if (bodyEnded) {
647
+ finalize(resolve);
648
+ }
649
+ },
650
+ onPlayingChange: (playing) => {
651
+ if (playing) {
652
+ this.startAudioSession();
653
+ }
654
+ }
655
+ }
656
+ );
657
+ const finalize = (next) => {
658
+ if (isFinalized) return;
659
+ isFinalized = true;
660
+ this.pendingAudioStop = null;
661
+ this.currentStream = null;
662
+ scheduler.flush();
663
+ this.abandonStream(abort, reader);
664
+ this.lipSync.stop();
665
+ this.endAudioSession();
666
+ next();
667
+ };
668
+ this.currentStream = { scheduler, abort, reader };
669
+ this.pendingAudioStop = () => finalize(resolve);
670
+ if (startup.isCancelled()) {
671
+ finalize(resolve);
672
+ return;
673
+ }
674
+ const pump = async () => {
675
+ let carry = null;
676
+ let hasScheduled = false;
677
+ for (; ; ) {
678
+ const { done, value } = await reader.read();
679
+ if (done) {
680
+ bodyEnded = true;
681
+ if (scheduler.isIdle()) {
682
+ finalize(resolve);
683
+ }
684
+ return;
685
+ }
686
+ if (isFinalized) {
687
+ return;
688
+ }
689
+ const chunk = carry ? joinChunks(carry, value) : value;
690
+ const whole = chunk.byteLength - chunk.byteLength % 2;
691
+ carry = whole < chunk.byteLength ? chunk.slice(whole) : null;
692
+ if (whole > 0) {
693
+ if (!hasScheduled && context.state !== "running") {
694
+ finalize(
695
+ () => reject(
696
+ new CharivoStateError(
697
+ "The TTS playback audio context is blocked, so streamed audio cannot start. Call prepareAudio() from a user gesture handler before speak()."
698
+ )
699
+ )
700
+ );
701
+ return;
702
+ }
703
+ hasScheduled = true;
704
+ scheduler.enqueue(chunk.subarray(0, whole));
705
+ }
706
+ }
707
+ };
708
+ void pump().catch((error) => {
709
+ finalize(() => reject(error));
710
+ });
711
+ });
712
+ }
713
+ /**
714
+ * Stop the route synthesizing and throw the body away.
715
+ *
716
+ * Through the reader, never the body: past getReader() the body is locked
717
+ * and cancelling it throws. An aborted stream errors, so its own cancel()
718
+ * rejects with the reason raised here -- ours, and already handled.
719
+ */
720
+ abandonStream(abort, reader) {
721
+ abort.abort();
722
+ void reader.cancel().catch(noop);
723
+ }
724
+ /**
725
+ * The AudioContext streamed PCM is scheduled into, created on first use and
726
+ * then only nudged back towards running: a context created inside a user
727
+ * gesture is the one browsers let play, so a replacement built later would
728
+ * be refused. It does not promise a running context on return.
729
+ */
730
+ ensurePlaybackContext() {
731
+ const existing = this.playbackContext;
732
+ if (existing) {
733
+ if (existing.state !== "running") {
734
+ void existing.resume().catch(
735
+ (error) => console.error(
736
+ "TTS Manager: failed to resume the playback audio context:",
737
+ error
738
+ )
739
+ );
740
+ }
741
+ return existing;
742
+ }
743
+ const constructor = window.AudioContext || window.webkitAudioContext;
744
+ if (!constructor) {
745
+ throw new CharivoStateError(
746
+ "AudioContext is not supported in this browser, so streamed TTS audio cannot be played."
747
+ );
748
+ }
749
+ const context = new constructor();
750
+ this.playbackContext = context;
751
+ return context;
752
+ }
753
+ ensurePlaybackGraph(context) {
754
+ this.playbackGraph ??= createPcmPlaybackGraph(context);
755
+ return this.playbackGraph;
756
+ }
463
757
  /**
464
758
  * Subscribes to browser lifecycle events once; the teardown is released in
465
759
  * dispose(). Pausing analysis while the tab is hidden closes the mouth
@@ -106,7 +106,11 @@ async function withTimeout(promise, timeoutMessage) {
106
106
  // src/openai/index.ts
107
107
  var OpenAITTSPlayer = class {
108
108
  playbackMode = "audio";
109
- audioMimeType = "audio/wav";
109
+ // The API answers with its mp3 default: `OpenAITTSProvider` sends
110
+ // `format: "wav"`, which is not the parameter the API reads -- that is
111
+ // `response_format` -- so it is carried in the body and ignored.
112
+ // `tests/live-tts` pins the container this actually yields.
113
+ audioMimeType = "audio/mpeg";
110
114
  provider;
111
115
  constructor(config) {
112
116
  this.provider = createOpenAITTSProvider({
@@ -125,7 +129,7 @@ var OpenAITTSPlayer = class {
125
129
  */
126
130
  async speak(text, options) {
127
131
  const audioBuffer = await this.generateAudio(text, options);
128
- const blob = new Blob([audioBuffer], { type: "audio/mp3" });
132
+ const blob = new Blob([audioBuffer], { type: this.audioMimeType });
129
133
  const audioUrl = URL.createObjectURL(blob);
130
134
  return new Promise((resolve, reject) => {
131
135
  const audio = new Audio(audioUrl);
@@ -72,7 +72,11 @@ async function withTimeout(promise, timeoutMessage) {
72
72
  // src/openai/index.ts
73
73
  var OpenAITTSPlayer = class {
74
74
  playbackMode = "audio";
75
- audioMimeType = "audio/wav";
75
+ // The API answers with its mp3 default: `OpenAITTSProvider` sends
76
+ // `format: "wav"`, which is not the parameter the API reads -- that is
77
+ // `response_format` -- so it is carried in the body and ignored.
78
+ // `tests/live-tts` pins the container this actually yields.
79
+ audioMimeType = "audio/mpeg";
76
80
  provider;
77
81
  constructor(config) {
78
82
  this.provider = createOpenAITTSProvider({
@@ -91,7 +95,7 @@ var OpenAITTSPlayer = class {
91
95
  */
92
96
  async speak(text, options) {
93
97
  const audioBuffer = await this.generateAudio(text, options);
94
- const blob = new Blob([audioBuffer], { type: "audio/mp3" });
98
+ const blob = new Blob([audioBuffer], { type: this.audioMimeType });
95
99
  const audioUrl = URL.createObjectURL(blob);
96
100
  return new Promise((resolve, reject) => {
97
101
  const audio = new Audio(audioUrl);
@@ -3,6 +3,13 @@ import { TTSPlayer } from '@charivo/core';
3
3
  interface RemoteTTSConfig {
4
4
  apiEndpoint?: string;
5
5
  defaultVoice?: string;
6
+ /**
7
+ * Opt in to `generateAudioStream`. Off by default: an unconditionally
8
+ * streaming player would break every route that answers a container today,
9
+ * and this player cannot fall back to the buffered path after a non-PCM
10
+ * answer without paying for a second synthesis.
11
+ */
12
+ streaming?: boolean;
6
13
  }
7
14
  declare function createRemoteTTSPlayer(config?: RemoteTTSConfig): TTSPlayer;
8
15
 
@@ -3,6 +3,13 @@ import { TTSPlayer } from '@charivo/core';
3
3
  interface RemoteTTSConfig {
4
4
  apiEndpoint?: string;
5
5
  defaultVoice?: string;
6
+ /**
7
+ * Opt in to `generateAudioStream`. Off by default: an unconditionally
8
+ * streaming player would break every route that answers a container today,
9
+ * and this player cannot fall back to the buffered path after a non-PCM
10
+ * answer without paying for a second synthesis.
11
+ */
12
+ streaming?: boolean;
6
13
  }
7
14
  declare function createRemoteTTSPlayer(config?: RemoteTTSConfig): TTSPlayer;
8
15