@pie-players/tts-client-server 0.3.75 → 1.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -56
- package/dist/ServerTTSProvider.d.ts +21 -21
- package/dist/ServerTTSProvider.js +80 -117
- package/package.json +5 -6
package/README.md
CHANGED
|
@@ -1,23 +1,21 @@
|
|
|
1
1
|
# @pie-players/tts-client-server
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
This package provides a browser-side TTS provider that offloads synthesis to a server API. The server handles provider selection (AWS Polly, Google Cloud TTS, etc.) and credential management, while the client plays audio and coordinates word highlighting.
|
|
3
|
+
`ServerTTSProvider`, the browser-side text-to-speech (TTS) provider that sends
|
|
4
|
+
synthesis to a server API the host runs. The server picks the vendor (AWS Polly,
|
|
5
|
+
Google Cloud TTS, or any provider built on `@pie-players/tts-server-core`) and
|
|
6
|
+
holds its credentials; the client plays the returned audio and reports word
|
|
7
|
+
boundaries from the server's speech marks. This README covers the provider's API
|
|
8
|
+
for host integrators adding server-backed read-aloud. For the package layering,
|
|
9
|
+
see [TTS Architecture](../../docs/accessibility/tts-architecture.md); for the
|
|
10
|
+
runtime flow, the [TTS deep dive](../../docs/accessibility/tts-deep-dive.md).
|
|
12
11
|
|
|
13
12
|
## Features
|
|
14
13
|
|
|
15
|
-
-
|
|
16
|
-
-
|
|
17
|
-
-
|
|
18
|
-
-
|
|
19
|
-
|
|
20
|
-
- ✅ **Blob URLs** - Efficient memory management
|
|
14
|
+
- **Server-side synthesis**: vendor credentials stay on the server
|
|
15
|
+
- **Speech marks**: word timings from the server drive highlighting
|
|
16
|
+
- **Word highlighting**: the audio clock is polled every 50ms
|
|
17
|
+
- **Audio playback**: an `HTMLAudioElement` playing a Blob URL, with pause and
|
|
18
|
+
resume
|
|
21
19
|
|
|
22
20
|
## Installation
|
|
23
21
|
|
|
@@ -38,7 +36,7 @@ import { TTSService } from '@pie-players/pie-assessment-toolkit';
|
|
|
38
36
|
|
|
39
37
|
const provider = new ServerTTSProvider();
|
|
40
38
|
const config: ServerTTSProviderConfig = {
|
|
41
|
-
apiEndpoint: '/api/tts', //
|
|
39
|
+
apiEndpoint: '/api/tts', // the host's TTS API route
|
|
42
40
|
provider: 'polly', // Server-side provider to use
|
|
43
41
|
voice: 'Joanna',
|
|
44
42
|
language: 'en-US',
|
|
@@ -52,7 +50,8 @@ await ttsService.initialize(provider, config);
|
|
|
52
50
|
provider reads none of them from `providerOptions`. `providerOptions` carries
|
|
53
51
|
request extras: the PIE transport forwards its `engine` (when no top-level
|
|
54
52
|
`engine` is set), `sampleRate`, `format` and `speechMarkTypes`, and the custom
|
|
55
|
-
transport reads its `speedRate
|
|
53
|
+
transport reads its `speedRate` (else a bucket from `rate`), `lang_id` (else the
|
|
54
|
+
content language, else `language`, else `en-US`) and `cache` (default `true`).
|
|
56
55
|
|
|
57
56
|
`apiEndpoint` is required when using `ServerTTSProvider` directly, and
|
|
58
57
|
`initialize` throws without it. Toolkit-level `tools.providers.textToSpeech`
|
|
@@ -77,11 +76,21 @@ await ttsService.speak(document.getElementById('content'));
|
|
|
77
76
|
|
|
78
77
|
## Transport Modes
|
|
79
78
|
|
|
80
|
-
`ServerTTSProvider`
|
|
79
|
+
`ServerTTSProvider` supports two transport modes:
|
|
81
80
|
|
|
82
|
-
- `pie
|
|
81
|
+
- `pie`: POST `${apiEndpoint}/synthesize`, inline base64 audio + inline speech marks
|
|
83
82
|
- `custom`: POST to root endpoint, then fetch `audioContent` and JSONL marks URLs
|
|
84
83
|
|
|
84
|
+
With `transportMode` unset, the transport is `custom` when `provider` is
|
|
85
|
+
`"custom"` and `pie` otherwise. `endpointMode` defaults to `synthesizePath` for
|
|
86
|
+
`pie` and `rootPost` for `custom`. The toolkit's `tools.providers.textToSpeech`
|
|
87
|
+
applies the same default from `serverProvider`.
|
|
88
|
+
|
|
89
|
+
The provider reports SSML support only on the `pie` transport with `provider`
|
|
90
|
+
`polly` (the default) or `google`. There the toolkit voices generated math as
|
|
91
|
+
SSML; everywhere else, the custom transport included, generated speech is plain
|
|
92
|
+
text.
|
|
93
|
+
|
|
85
94
|
### PIE mode request
|
|
86
95
|
|
|
87
96
|
**Request:**
|
|
@@ -115,18 +124,6 @@ await ttsService.speak(document.getElementById('content'));
|
|
|
115
124
|
}
|
|
116
125
|
```
|
|
117
126
|
|
|
118
|
-
### PIE mode response
|
|
119
|
-
|
|
120
|
-
```json
|
|
121
|
-
{
|
|
122
|
-
"audio": "base64-encoded-audio",
|
|
123
|
-
"contentType": "audio/mpeg",
|
|
124
|
-
"speechMarks": [
|
|
125
|
-
{ "time": 0, "type": "word", "start": 0, "end": 5, "value": "Hello" }
|
|
126
|
-
]
|
|
127
|
-
}
|
|
128
|
-
```
|
|
129
|
-
|
|
130
127
|
### Custom mode request
|
|
131
128
|
|
|
132
129
|
```json
|
|
@@ -162,7 +159,7 @@ Speech marks are fetched from the `word` URL and parsed as JSONL.
|
|
|
162
159
|
|
|
163
160
|
## SvelteKit Implementation Example
|
|
164
161
|
|
|
165
|
-
|
|
162
|
+
The [`@pie-players/tts-server-polly` integration guide](../tts-server-polly/examples/INTEGRATION-GUIDE.md) builds these routes for Polly, and the [`@pie-players/tts-server-google` integration guide](../tts-server-google/examples/INTEGRATION-GUIDE.md) for Google.
|
|
166
163
|
|
|
167
164
|
Example route structure:
|
|
168
165
|
```
|
|
@@ -181,8 +178,7 @@ interface ServerTTSProviderConfig {
|
|
|
181
178
|
provider?: string; // Server provider ('polly', 'google', etc.)
|
|
182
179
|
transportMode?: 'pie' | 'custom';
|
|
183
180
|
endpointMode?: 'synthesizePath' | 'rootPost';
|
|
184
|
-
endpointValidationMode?: 'voices' | 'endpoint' | 'none';
|
|
185
|
-
validateEndpoint?: boolean; // Probe the endpoint during initialize()
|
|
181
|
+
endpointValidationMode?: 'voices' | 'endpoint' | 'none'; // Probe run by initialize(); default 'none'
|
|
186
182
|
authToken?: string; // JWT or API key
|
|
187
183
|
includeAuthOnAssetFetch?: boolean;
|
|
188
184
|
assetOrigins?: string[]; // Trusted origins for Authorization header
|
|
@@ -236,33 +232,35 @@ for the host-wide contract this provider fits into.
|
|
|
236
232
|
|
|
237
233
|
## How It Works
|
|
238
234
|
|
|
239
|
-
1. **
|
|
235
|
+
1. **TTSService calls** `speak(text)` with the text it resolved from the element or range
|
|
240
236
|
2. **Adapter builds** backend-specific request payload
|
|
241
237
|
3. **Provider POSTs** to resolved synthesis endpoint (`/synthesize` or root POST)
|
|
242
238
|
4. **Adapter normalizes** response into audio + speech marks
|
|
243
239
|
5. **Client loads** audio as Blob URL
|
|
244
240
|
6. **Client plays** audio via HTMLAudioElement
|
|
245
241
|
7. **Client polls** audio time every 50ms
|
|
246
|
-
8. **Client
|
|
242
|
+
8. **Client reports** the word at the current audio time through the word-boundary callback
|
|
247
243
|
9. **TTSService** highlights words in DOM
|
|
248
244
|
|
|
249
245
|
## Word Highlighting Synchronization
|
|
250
246
|
|
|
251
|
-
|
|
247
|
+
Every 50ms the provider reads the audio's current time and reports the last
|
|
248
|
+
word whose start time has arrived, once:
|
|
252
249
|
|
|
253
250
|
```typescript
|
|
254
|
-
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
onWordBoundary('', timing.charIndex, timing.length);
|
|
261
|
-
}
|
|
251
|
+
const currentTimeMs = audio.currentTime * 1000;
|
|
252
|
+
const index = lastTimingAtOrBefore(wordTimings, currentTimeMs);
|
|
253
|
+
if (index >= 0 && index !== lastReported) {
|
|
254
|
+
lastReported = index;
|
|
255
|
+
const timing = wordTimings[index];
|
|
256
|
+
onWordBoundary(timing.word, timing.charIndex, timing.length);
|
|
262
257
|
}
|
|
263
258
|
```
|
|
264
259
|
|
|
265
|
-
|
|
260
|
+
Words a tick crossed are skipped: the callback names the word being spoken, so
|
|
261
|
+
replaying them would paint highlights the learner never sees and leave the
|
|
262
|
+
highlight behind the audio. A background tab's clamped timer resyncs on its next
|
|
263
|
+
tick for the same reason.
|
|
266
264
|
|
|
267
265
|
## Memory Management
|
|
268
266
|
|
|
@@ -280,16 +278,17 @@ try {
|
|
|
280
278
|
await ttsService.speak(document.getElementById('content'));
|
|
281
279
|
} catch (error) {
|
|
282
280
|
console.error('TTS failed:', error.message);
|
|
283
|
-
//
|
|
281
|
+
// A failed read rejects without switching provider; the toolkit coordinator
|
|
282
|
+
// falls back to browser TTS only when a provider fails to start.
|
|
284
283
|
}
|
|
285
284
|
```
|
|
286
285
|
|
|
287
286
|
## Browser Compatibility
|
|
288
287
|
|
|
289
|
-
-
|
|
290
|
-
-
|
|
291
|
-
-
|
|
292
|
-
-
|
|
288
|
+
- Chrome/Edge (latest)
|
|
289
|
+
- Firefox (latest)
|
|
290
|
+
- Safari (latest)
|
|
291
|
+
- Mobile browsers
|
|
293
292
|
|
|
294
293
|
Requires:
|
|
295
294
|
|
|
@@ -298,12 +297,11 @@ Requires:
|
|
|
298
297
|
- `URL.createObjectURL`
|
|
299
298
|
- `atob` for base64 decoding
|
|
300
299
|
|
|
301
|
-
##
|
|
300
|
+
## Caching
|
|
302
301
|
|
|
303
|
-
|
|
304
|
-
-
|
|
305
|
-
|
|
306
|
-
- **Parallel requests:** Audio + marks fetched together
|
|
302
|
+
The client keeps no audio cache. Caching is server-side, through the host's
|
|
303
|
+
`ITTSCache` (`@pie-players/tts-server-core`); the custom transport also sends a
|
|
304
|
+
`cache` flag, `true` unless `providerOptions.cache` is `false`.
|
|
307
305
|
|
|
308
306
|
## License
|
|
309
307
|
|
|
@@ -4,8 +4,15 @@
|
|
|
4
4
|
* Provides high-quality TTS by calling a server-side API that uses
|
|
5
5
|
* providers like AWS Polly, Google Cloud TTS, etc.
|
|
6
6
|
* Returns audio with precise word-level timing (speech marks).
|
|
7
|
+
*
|
|
8
|
+
* Layout: config types; the asset-URL and auth helpers that guard fetches of
|
|
9
|
+
* server-returned URLs; wire types; mode resolvers; the `pie` and `custom`
|
|
10
|
+
* transport adapters (`ADAPTERS`); `ServerTTSProviderImpl`, which owns one
|
|
11
|
+
* playback (synthesis, audio element, word highlighting); and
|
|
12
|
+
* `ServerTTSProvider`, which picks the adapter, runs the optional endpoint probe
|
|
13
|
+
* and hands out implementations.
|
|
7
14
|
*/
|
|
8
|
-
import type { ITTSProvider, ITTSProviderImplementation, TTSConfig, TTSProviderCapabilities, TTSProviderOptions } from "@pie-players/pie-tts";
|
|
15
|
+
import type { ITTSProvider, ITTSProviderImplementation, SpeedRateBucket, TTSConfig, TTSProviderCapabilities, TTSProviderOptions } from "@pie-players/pie-tts";
|
|
9
16
|
/**
|
|
10
17
|
* The provider options the server provider reads. `contentLanguage`, set per
|
|
11
18
|
* speak, names the language synthesized: it wins over `language`, and a
|
|
@@ -20,8 +27,11 @@ export interface ServerTTSProviderOptions extends TTSProviderOptions {
|
|
|
20
27
|
format?: "mp3" | "ogg" | "pcm";
|
|
21
28
|
/** Speech mark types requested, on the pie transport. */
|
|
22
29
|
speechMarkTypes?: Array<"word" | "sentence" | "ssml">;
|
|
23
|
-
/**
|
|
24
|
-
|
|
30
|
+
/**
|
|
31
|
+
* Speed bucket, on the custom transport; derived from `rate` when unset or
|
|
32
|
+
* not one of the three buckets.
|
|
33
|
+
*/
|
|
34
|
+
speedRate?: SpeedRateBucket;
|
|
25
35
|
/**
|
|
26
36
|
* Language id, on the custom transport. Without it, the language a speak
|
|
27
37
|
* names, then `language`, then en-US.
|
|
@@ -62,10 +72,12 @@ export interface ServerTTSProviderConfig extends TTSConfig {
|
|
|
62
72
|
*/
|
|
63
73
|
endpointMode?: "synthesizePath" | "rootPost";
|
|
64
74
|
/**
|
|
65
|
-
*
|
|
66
|
-
* - voices:
|
|
67
|
-
* - endpoint:
|
|
68
|
-
* - none:
|
|
75
|
+
* The probe `initialize()` runs before it resolves; a failed probe rejects it.
|
|
76
|
+
* - voices: GET {apiEndpoint}/voices, trying the provider's own route first
|
|
77
|
+
* - endpoint: OPTIONS on the resolved synthesis endpoint
|
|
78
|
+
* - none: no probe, so an unreachable server fails at the first synthesis
|
|
79
|
+
*
|
|
80
|
+
* @default "none"
|
|
69
81
|
*/
|
|
70
82
|
endpointValidationMode?: "voices" | "endpoint" | "none";
|
|
71
83
|
/**
|
|
@@ -93,14 +105,6 @@ export interface ServerTTSProviderConfig extends TTSConfig {
|
|
|
93
105
|
* fetch on the browser default, `"same-origin"`.
|
|
94
106
|
*/
|
|
95
107
|
credentials?: "omit" | "same-origin" | "include";
|
|
96
|
-
/**
|
|
97
|
-
* Validate API endpoint availability during initialization (slower but safer)
|
|
98
|
-
*
|
|
99
|
-
* @extension Performance vs safety tradeoff
|
|
100
|
-
* @default false (fast initialization, fail on first synthesis if unavailable)
|
|
101
|
-
* @note When true, adds 100-500ms to initialization time
|
|
102
|
-
*/
|
|
103
|
-
validateEndpoint?: boolean;
|
|
104
108
|
}
|
|
105
109
|
/**
|
|
106
110
|
* Server TTS Provider
|
|
@@ -110,8 +114,6 @@ export interface ServerTTSProviderConfig extends TTSConfig {
|
|
|
110
114
|
*/
|
|
111
115
|
export declare class ServerTTSProvider implements ITTSProvider {
|
|
112
116
|
readonly providerId = "server-tts";
|
|
113
|
-
readonly providerName = "Server TTS";
|
|
114
|
-
readonly version = "1.0.0";
|
|
115
117
|
private config;
|
|
116
118
|
private adapter;
|
|
117
119
|
private telemetryReporter;
|
|
@@ -119,10 +121,8 @@ export declare class ServerTTSProvider implements ITTSProvider {
|
|
|
119
121
|
/**
|
|
120
122
|
* Initialize the server TTS provider.
|
|
121
123
|
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
* @performance Default: <10ms, With validation: 100-500ms
|
|
124
|
+
* Makes no request unless `endpointValidationMode` names a probe, which
|
|
125
|
+
* adds 100-500ms.
|
|
126
126
|
*/
|
|
127
127
|
initialize(config: TTSConfig): Promise<ITTSProviderImplementation>;
|
|
128
128
|
/**
|
|
@@ -4,13 +4,27 @@
|
|
|
4
4
|
* Provides high-quality TTS by calling a server-side API that uses
|
|
5
5
|
* providers like AWS Polly, Google Cloud TTS, etc.
|
|
6
6
|
* Returns audio with precise word-level timing (speech marks).
|
|
7
|
+
*
|
|
8
|
+
* Layout: config types; the asset-URL and auth helpers that guard fetches of
|
|
9
|
+
* server-returned URLs; wire types; mode resolvers; the `pie` and `custom`
|
|
10
|
+
* transport adapters (`ADAPTERS`); `ServerTTSProviderImpl`, which owns one
|
|
11
|
+
* playback (synthesis, audio element, word highlighting); and
|
|
12
|
+
* `ServerTTSProvider`, which picks the adapter, runs the optional endpoint probe
|
|
13
|
+
* and hands out implementations.
|
|
7
14
|
*/
|
|
8
15
|
import { createPieLogger, isTtsDebugEnabled, } from "@pie-players/pie-players-shared/pie";
|
|
9
|
-
import { normalizeSpeechMarks, resolveSpeedRateBucket, } from "@pie-players/tts-server-core";
|
|
16
|
+
import { anchorSpeechMarks, normalizeSpeechMarks, resolveSpeedRateBucket, } from "@pie-players/tts-server-core";
|
|
10
17
|
// Debug lines need PIE_TTS_DEBUG=1 or globalThis.__PIE_TTS_DEBUG__ = true.
|
|
11
18
|
const logger = createPieLogger("server-tts-provider", isTtsDebugEnabled);
|
|
12
19
|
// Telemetry names the tool by its canonical id, as the toolkit does.
|
|
13
20
|
const TOOL_ID = "textToSpeech";
|
|
21
|
+
// Word-highlight polling period; `startWordHighlighting` gives the reasons.
|
|
22
|
+
const HIGHLIGHT_INTERVAL_MS = 50;
|
|
23
|
+
// How long the endpoint probe waits before reporting the server unreachable.
|
|
24
|
+
const ENDPOINT_PROBE_TIMEOUT_MS = 5000;
|
|
25
|
+
// Debug-log previews of an error body and of the text under synthesis.
|
|
26
|
+
const RAW_BODY_PREVIEW_LENGTH = 500;
|
|
27
|
+
const TEXT_PREVIEW_LENGTH = 120;
|
|
14
28
|
/**
|
|
15
29
|
* Parse a TTS asset URL (speech marks or audio). Rejects anything that is
|
|
16
30
|
* not an absolute http(s) URL; relative URLs are resolved against
|
|
@@ -119,6 +133,13 @@ const getTelemetryReporter = (config) => {
|
|
|
119
133
|
const reporter = config.providerOptions?.__pieTelemetry;
|
|
120
134
|
return typeof reporter === "function" ? reporter : undefined;
|
|
121
135
|
};
|
|
136
|
+
/** The payload of a `pie-tool-backend-call-*` telemetry event. */
|
|
137
|
+
const backendTelemetry = (config, operation, extra) => ({
|
|
138
|
+
toolId: TOOL_ID,
|
|
139
|
+
backend: config.provider || "server",
|
|
140
|
+
operation,
|
|
141
|
+
...extra,
|
|
142
|
+
});
|
|
122
143
|
const MAX_TEXT_LENGTH_BY_MODE = {
|
|
123
144
|
pie: 3000,
|
|
124
145
|
custom: 3000,
|
|
@@ -164,14 +185,18 @@ const resolveEndpointMode = (config, mode) => {
|
|
|
164
185
|
return config.endpointMode;
|
|
165
186
|
return mode === "custom" ? "rootPost" : "synthesizePath";
|
|
166
187
|
};
|
|
167
|
-
const resolveValidationMode = (config
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
188
|
+
const resolveValidationMode = (config) => config.endpointValidationMode === "voices" ||
|
|
189
|
+
config.endpointValidationMode === "endpoint"
|
|
190
|
+
? config.endpointValidationMode
|
|
191
|
+
: "none";
|
|
192
|
+
const SPEED_RATE_BUCKETS = new Set([
|
|
193
|
+
"slow",
|
|
194
|
+
"medium",
|
|
195
|
+
"fast",
|
|
196
|
+
]);
|
|
172
197
|
const resolveSpeedRate = (config) => {
|
|
173
198
|
const speedRate = config.providerOptions?.speedRate;
|
|
174
|
-
if (
|
|
199
|
+
if (SPEED_RATE_BUCKETS.has(speedRate))
|
|
175
200
|
return speedRate;
|
|
176
201
|
return resolveSpeedRateBucket(config.rate);
|
|
177
202
|
};
|
|
@@ -253,7 +278,7 @@ const pieAdapter = {
|
|
|
253
278
|
includeSpeechMarks: true,
|
|
254
279
|
};
|
|
255
280
|
},
|
|
256
|
-
parseResponse: async (response, _config, _headers, _signal,
|
|
281
|
+
parseResponse: async (response, _config, _headers, _signal, text) => {
|
|
257
282
|
const data = await response.json();
|
|
258
283
|
return {
|
|
259
284
|
audio: {
|
|
@@ -261,7 +286,9 @@ const pieAdapter = {
|
|
|
261
286
|
data: data.audio,
|
|
262
287
|
contentType: data.contentType,
|
|
263
288
|
},
|
|
264
|
-
speechMarks: Array.isArray(data.speechMarks)
|
|
289
|
+
speechMarks: Array.isArray(data.speechMarks)
|
|
290
|
+
? anchorSpeechMarks(data.speechMarks, text)
|
|
291
|
+
: [],
|
|
265
292
|
};
|
|
266
293
|
},
|
|
267
294
|
};
|
|
@@ -378,14 +405,11 @@ class ServerTTSProviderImpl {
|
|
|
378
405
|
}
|
|
379
406
|
}
|
|
380
407
|
async speak(text) {
|
|
381
|
-
// Stop any current playback
|
|
382
408
|
this.stop();
|
|
383
|
-
// Reset intentionally stopped flag for new playback
|
|
384
409
|
this.intentionallyStopped = false;
|
|
385
410
|
const runId = ++this.synthesisRunId;
|
|
386
411
|
const synthesisController = new AbortController();
|
|
387
412
|
this.activeSynthesisController = synthesisController;
|
|
388
|
-
// Call server API to synthesize speech
|
|
389
413
|
let synthesized;
|
|
390
414
|
try {
|
|
391
415
|
synthesized = await this.synthesizeSpeech(text, synthesisController.signal, runId);
|
|
@@ -410,10 +434,8 @@ class ServerTTSProviderImpl {
|
|
|
410
434
|
this.wordTimings = wordTimings;
|
|
411
435
|
this.highlightCursor = -1;
|
|
412
436
|
return new Promise((resolve, reject) => {
|
|
413
|
-
// Create audio element
|
|
414
437
|
const audio = new Audio(audioUrl);
|
|
415
438
|
this.currentAudio = audio;
|
|
416
|
-
// Apply volume from config
|
|
417
439
|
if (this.config.volume !== undefined) {
|
|
418
440
|
audio.volume = Math.max(0, Math.min(1, this.config.volume));
|
|
419
441
|
}
|
|
@@ -427,12 +449,13 @@ class ServerTTSProviderImpl {
|
|
|
427
449
|
return;
|
|
428
450
|
this.pausedState = false;
|
|
429
451
|
try {
|
|
430
|
-
this.onPlaybackStart?.(
|
|
452
|
+
this.onPlaybackStart?.({
|
|
453
|
+
wordBoundaries: this.wordTimings.length > 0,
|
|
454
|
+
});
|
|
431
455
|
}
|
|
432
456
|
catch (error) {
|
|
433
457
|
logger.warn("playback-start callback failed:", error);
|
|
434
458
|
}
|
|
435
|
-
// Start word highlighting
|
|
436
459
|
if (this.onWordBoundary && this.wordTimings.length > 0) {
|
|
437
460
|
this.startWordHighlighting();
|
|
438
461
|
}
|
|
@@ -479,16 +502,11 @@ class ServerTTSProviderImpl {
|
|
|
479
502
|
*/
|
|
480
503
|
async synthesizeSpeech(text, signal, runId) {
|
|
481
504
|
const synthStartedAt = Date.now();
|
|
482
|
-
await this.emitTelemetry("pie-tool-backend-call-start",
|
|
483
|
-
toolId: TOOL_ID,
|
|
484
|
-
backend: this.config.provider || "server",
|
|
485
|
-
operation: "synthesize-speech",
|
|
486
|
-
});
|
|
505
|
+
await this.emitTelemetry("pie-tool-backend-call-start", backendTelemetry(this.config, "synthesize-speech"));
|
|
487
506
|
const headers = {
|
|
488
507
|
"Content-Type": "application/json",
|
|
489
508
|
...this.config.headers,
|
|
490
509
|
};
|
|
491
|
-
// Add authentication if provided
|
|
492
510
|
if (this.config.authToken) {
|
|
493
511
|
headers["Authorization"] = `Bearer ${this.config.authToken}`;
|
|
494
512
|
}
|
|
@@ -507,14 +525,11 @@ class ServerTTSProviderImpl {
|
|
|
507
525
|
catch (error) {
|
|
508
526
|
if (signal.aborted)
|
|
509
527
|
throw error;
|
|
510
|
-
await this.emitTelemetry("pie-tool-backend-call-error", {
|
|
511
|
-
toolId: TOOL_ID,
|
|
512
|
-
backend: this.config.provider || "server",
|
|
513
|
-
operation: "synthesize-speech",
|
|
528
|
+
await this.emitTelemetry("pie-tool-backend-call-error", backendTelemetry(this.config, "synthesize-speech", {
|
|
514
529
|
duration: Date.now() - synthStartedAt,
|
|
515
530
|
errorType: "TTSBackendNetworkError",
|
|
516
531
|
message: error instanceof Error ? error.message : String(error),
|
|
517
|
-
});
|
|
532
|
+
}));
|
|
518
533
|
throw error;
|
|
519
534
|
}
|
|
520
535
|
})();
|
|
@@ -538,7 +553,7 @@ class ServerTTSProviderImpl {
|
|
|
538
553
|
// itself stays behind the debug gate; the length above is what says
|
|
539
554
|
// a body arrived and was not JSON.
|
|
540
555
|
logger.debug("non-OK response body", {
|
|
541
|
-
rawBodyPreview: rawBody.slice(0,
|
|
556
|
+
rawBodyPreview: rawBody.slice(0, RAW_BODY_PREVIEW_LENGTH),
|
|
542
557
|
});
|
|
543
558
|
}
|
|
544
559
|
}
|
|
@@ -547,15 +562,12 @@ class ServerTTSProviderImpl {
|
|
|
547
562
|
? errorData.error.message
|
|
548
563
|
: undefined) ||
|
|
549
564
|
`Server returned ${response.status}`;
|
|
550
|
-
await this.emitTelemetry("pie-tool-backend-call-error", {
|
|
551
|
-
toolId: TOOL_ID,
|
|
552
|
-
backend: this.config.provider || "server",
|
|
553
|
-
operation: "synthesize-speech",
|
|
565
|
+
await this.emitTelemetry("pie-tool-backend-call-error", backendTelemetry(this.config, "synthesize-speech", {
|
|
554
566
|
duration: Date.now() - synthStartedAt,
|
|
555
567
|
statusCode: response.status,
|
|
556
568
|
errorType: "TTSBackendRequestError",
|
|
557
569
|
message: errorMessage,
|
|
558
|
-
});
|
|
570
|
+
}));
|
|
559
571
|
logger.error("synthesize request failed", {
|
|
560
572
|
status: response.status,
|
|
561
573
|
url: synthUrl,
|
|
@@ -567,7 +579,10 @@ class ServerTTSProviderImpl {
|
|
|
567
579
|
// can echo it, so both stay behind the debug gate. `errorMessage` above is
|
|
568
580
|
// thrown to the caller regardless, and `textLength` is what correlates a
|
|
569
581
|
// failure with a payload without publishing what the learner is reading.
|
|
570
|
-
const textPreview = text
|
|
582
|
+
const textPreview = text
|
|
583
|
+
.replace(/\s+/g, " ")
|
|
584
|
+
.trim()
|
|
585
|
+
.slice(0, TEXT_PREVIEW_LENGTH);
|
|
571
586
|
logger.debug("synthesize request failed for text", {
|
|
572
587
|
responseBody: errorData,
|
|
573
588
|
textPreview: textPreview.length < text.length ? `${textPreview}…` : textPreview,
|
|
@@ -585,22 +600,15 @@ class ServerTTSProviderImpl {
|
|
|
585
600
|
else {
|
|
586
601
|
const audioAssetUrl = normalized.audio.url;
|
|
587
602
|
const assetFetchStartedAt = Date.now();
|
|
588
|
-
await this.emitTelemetry("pie-tool-backend-call-start",
|
|
589
|
-
toolId: TOOL_ID,
|
|
590
|
-
backend: this.config.provider || "server",
|
|
591
|
-
operation: "fetch-synthesized-audio-asset",
|
|
592
|
-
});
|
|
603
|
+
await this.emitTelemetry("pie-tool-backend-call-start", backendTelemetry(this.config, "fetch-synthesized-audio-asset"));
|
|
593
604
|
const assetHeaders = assetFetchHeaders(headers, this.config);
|
|
594
605
|
const parsedAssetUrl = parseAssetUrl(audioAssetUrl, this.config);
|
|
595
606
|
if (parsedAssetUrl === null) {
|
|
596
|
-
await this.emitTelemetry("pie-tool-backend-call-error", {
|
|
597
|
-
toolId: TOOL_ID,
|
|
598
|
-
backend: this.config.provider || "server",
|
|
599
|
-
operation: "fetch-synthesized-audio-asset",
|
|
607
|
+
await this.emitTelemetry("pie-tool-backend-call-error", backendTelemetry(this.config, "fetch-synthesized-audio-asset", {
|
|
600
608
|
duration: Date.now() - assetFetchStartedAt,
|
|
601
609
|
errorType: "TTSAssetInvalidUrl",
|
|
602
610
|
message: "TTS asset URL rejected (non-http(s) or malformed)",
|
|
603
|
-
});
|
|
611
|
+
}));
|
|
604
612
|
throw new Error("TTS asset URL rejected (non-http(s) or malformed)");
|
|
605
613
|
}
|
|
606
614
|
const assetTrusted = isOriginTrustedForAuth(parsedAssetUrl, this.config);
|
|
@@ -619,46 +627,33 @@ class ServerTTSProviderImpl {
|
|
|
619
627
|
catch (error) {
|
|
620
628
|
if (signal.aborted)
|
|
621
629
|
throw error;
|
|
622
|
-
await this.emitTelemetry("pie-tool-backend-call-error", {
|
|
623
|
-
toolId: TOOL_ID,
|
|
624
|
-
backend: this.config.provider || "server",
|
|
625
|
-
operation: "fetch-synthesized-audio-asset",
|
|
630
|
+
await this.emitTelemetry("pie-tool-backend-call-error", backendTelemetry(this.config, "fetch-synthesized-audio-asset", {
|
|
626
631
|
duration: Date.now() - assetFetchStartedAt,
|
|
627
632
|
errorType: "TTSAssetNetworkError",
|
|
628
633
|
message: error instanceof Error ? error.message : String(error),
|
|
629
|
-
});
|
|
634
|
+
}));
|
|
630
635
|
throw error;
|
|
631
636
|
}
|
|
632
637
|
})();
|
|
633
638
|
if (!audioResponse.ok) {
|
|
634
|
-
await this.emitTelemetry("pie-tool-backend-call-error", {
|
|
635
|
-
toolId: TOOL_ID,
|
|
636
|
-
backend: this.config.provider || "server",
|
|
637
|
-
operation: "fetch-synthesized-audio-asset",
|
|
639
|
+
await this.emitTelemetry("pie-tool-backend-call-error", backendTelemetry(this.config, "fetch-synthesized-audio-asset", {
|
|
638
640
|
duration: Date.now() - assetFetchStartedAt,
|
|
639
641
|
statusCode: audioResponse.status,
|
|
640
642
|
errorType: "TTSAssetFetchError",
|
|
641
643
|
message: `Failed to download synthesized audio (${audioResponse.status})`,
|
|
642
|
-
});
|
|
644
|
+
}));
|
|
643
645
|
throw new Error(`Failed to download synthesized audio (${audioResponse.status})`);
|
|
644
646
|
}
|
|
645
647
|
audioBlob = await audioResponse.blob();
|
|
646
|
-
await this.emitTelemetry("pie-tool-backend-call-success", {
|
|
647
|
-
toolId: TOOL_ID,
|
|
648
|
-
backend: this.config.provider || "server",
|
|
649
|
-
operation: "fetch-synthesized-audio-asset",
|
|
648
|
+
await this.emitTelemetry("pie-tool-backend-call-success", backendTelemetry(this.config, "fetch-synthesized-audio-asset", {
|
|
650
649
|
duration: Date.now() - assetFetchStartedAt,
|
|
651
|
-
});
|
|
650
|
+
}));
|
|
652
651
|
}
|
|
653
652
|
const audioUrl = URL.createObjectURL(audioBlob);
|
|
654
|
-
// Convert speech marks to word timings
|
|
655
653
|
const wordTimings = this.parseSpeechMarks(normalized.speechMarks);
|
|
656
|
-
await this.emitTelemetry("pie-tool-backend-call-success", {
|
|
657
|
-
toolId: TOOL_ID,
|
|
658
|
-
backend: this.config.provider || "server",
|
|
659
|
-
operation: "synthesize-speech",
|
|
654
|
+
await this.emitTelemetry("pie-tool-backend-call-success", backendTelemetry(this.config, "synthesize-speech", {
|
|
660
655
|
duration: Date.now() - synthStartedAt,
|
|
661
|
-
});
|
|
656
|
+
}));
|
|
662
657
|
return { audioUrl, wordTimings };
|
|
663
658
|
}
|
|
664
659
|
/**
|
|
@@ -742,7 +737,7 @@ class ServerTTSProviderImpl {
|
|
|
742
737
|
currentTimeMs,
|
|
743
738
|
});
|
|
744
739
|
this.onWordBoundary?.(timing.word, timing.charIndex, timing.length);
|
|
745
|
-
},
|
|
740
|
+
}, HIGHLIGHT_INTERVAL_MS);
|
|
746
741
|
}
|
|
747
742
|
/**
|
|
748
743
|
* Index of the last timing whose start time has arrived at `currentTimeMs`,
|
|
@@ -826,20 +821,13 @@ class ServerTTSProviderImpl {
|
|
|
826
821
|
this.wordTimings = [];
|
|
827
822
|
this.highlightCursor = -1;
|
|
828
823
|
}
|
|
829
|
-
isPlaying() {
|
|
830
|
-
return this.currentAudio !== null && !this.pausedState;
|
|
831
|
-
}
|
|
832
|
-
isPaused() {
|
|
833
|
-
return this.pausedState;
|
|
834
|
-
}
|
|
835
824
|
/**
|
|
836
|
-
*
|
|
837
|
-
*
|
|
838
|
-
*
|
|
839
|
-
*
|
|
825
|
+
* Store rate, pitch, voice and the per-speak language for the next
|
|
826
|
+
* synthesis. Server-side synthesis bakes them into the audio and speech
|
|
827
|
+
* marks, so an active speed change is TTSService's job: it replays from the
|
|
828
|
+
* current segment.
|
|
840
829
|
*/
|
|
841
830
|
updateSettings(settings) {
|
|
842
|
-
// Update config
|
|
843
831
|
if (settings.rate !== undefined) {
|
|
844
832
|
this.config.rate = settings.rate;
|
|
845
833
|
this.config.providerOptions = {
|
|
@@ -848,11 +836,9 @@ class ServerTTSProviderImpl {
|
|
|
848
836
|
};
|
|
849
837
|
}
|
|
850
838
|
if (settings.pitch !== undefined) {
|
|
851
|
-
// Server-side pitch is baked into audio, so this only affects next speak()
|
|
852
839
|
this.config.pitch = settings.pitch;
|
|
853
840
|
}
|
|
854
841
|
if (settings.voice !== undefined) {
|
|
855
|
-
// Voice change requires resynthesis, affects next speak()
|
|
856
842
|
this.config.voice = settings.voice;
|
|
857
843
|
}
|
|
858
844
|
// Only the per-speak language: the toolkit's other options would replace
|
|
@@ -874,8 +860,6 @@ class ServerTTSProviderImpl {
|
|
|
874
860
|
*/
|
|
875
861
|
export class ServerTTSProvider {
|
|
876
862
|
providerId = "server-tts";
|
|
877
|
-
providerName = "Server TTS";
|
|
878
|
-
version = "1.0.0";
|
|
879
863
|
config = null;
|
|
880
864
|
adapter = null;
|
|
881
865
|
telemetryReporter;
|
|
@@ -890,10 +874,8 @@ export class ServerTTSProvider {
|
|
|
890
874
|
/**
|
|
891
875
|
* Initialize the server TTS provider.
|
|
892
876
|
*
|
|
893
|
-
*
|
|
894
|
-
*
|
|
895
|
-
*
|
|
896
|
-
* @performance Default: <10ms, With validation: 100-500ms
|
|
877
|
+
* Makes no request unless `endpointValidationMode` names a probe, which
|
|
878
|
+
* adds 100-500ms.
|
|
897
879
|
*/
|
|
898
880
|
async initialize(config) {
|
|
899
881
|
const serverConfig = config;
|
|
@@ -904,32 +886,22 @@ export class ServerTTSProvider {
|
|
|
904
886
|
this.telemetryReporter = getTelemetryReporter(serverConfig);
|
|
905
887
|
const transportMode = resolveTransportMode(serverConfig);
|
|
906
888
|
this.adapter = ADAPTERS[transportMode];
|
|
907
|
-
|
|
908
|
-
if (
|
|
889
|
+
const validationMode = resolveValidationMode(serverConfig);
|
|
890
|
+
if (validationMode !== "none") {
|
|
909
891
|
const validationStartedAt = Date.now();
|
|
910
|
-
await this.emitTelemetry("pie-tool-backend-call-start",
|
|
911
|
-
|
|
912
|
-
backend: serverConfig.provider || "server",
|
|
913
|
-
operation: "validate-endpoint",
|
|
914
|
-
});
|
|
915
|
-
const available = await this.testAPIAvailability();
|
|
892
|
+
await this.emitTelemetry("pie-tool-backend-call-start", backendTelemetry(serverConfig, "validate-endpoint"));
|
|
893
|
+
const available = await this.testAPIAvailability(validationMode);
|
|
916
894
|
if (!available) {
|
|
917
|
-
await this.emitTelemetry("pie-tool-backend-call-error", {
|
|
918
|
-
toolId: TOOL_ID,
|
|
919
|
-
backend: serverConfig.provider || "server",
|
|
920
|
-
operation: "validate-endpoint",
|
|
895
|
+
await this.emitTelemetry("pie-tool-backend-call-error", backendTelemetry(serverConfig, "validate-endpoint", {
|
|
921
896
|
duration: Date.now() - validationStartedAt,
|
|
922
897
|
errorType: "TTSEndpointValidationError",
|
|
923
898
|
message: `Server TTS API not available at ${serverConfig.apiEndpoint}`,
|
|
924
|
-
});
|
|
899
|
+
}));
|
|
925
900
|
throw new Error(`Server TTS API not available at ${serverConfig.apiEndpoint}`);
|
|
926
901
|
}
|
|
927
|
-
await this.emitTelemetry("pie-tool-backend-call-success", {
|
|
928
|
-
toolId: TOOL_ID,
|
|
929
|
-
backend: serverConfig.provider || "server",
|
|
930
|
-
operation: "validate-endpoint",
|
|
902
|
+
await this.emitTelemetry("pie-tool-backend-call-success", backendTelemetry(serverConfig, "validate-endpoint", {
|
|
931
903
|
duration: Date.now() - validationStartedAt,
|
|
932
|
-
});
|
|
904
|
+
}));
|
|
933
905
|
}
|
|
934
906
|
return new ServerTTSProviderImpl(serverConfig, this.adapter);
|
|
935
907
|
}
|
|
@@ -938,7 +910,7 @@ export class ServerTTSProvider {
|
|
|
938
910
|
*
|
|
939
911
|
* @performance 100-500ms depending on network
|
|
940
912
|
*/
|
|
941
|
-
async testAPIAvailability() {
|
|
913
|
+
async testAPIAvailability(mode) {
|
|
942
914
|
if (!this.config || !this.adapter)
|
|
943
915
|
return false;
|
|
944
916
|
try {
|
|
@@ -946,14 +918,8 @@ export class ServerTTSProvider {
|
|
|
946
918
|
if (this.config.authToken) {
|
|
947
919
|
headers["Authorization"] = `Bearer ${this.config.authToken}`;
|
|
948
920
|
}
|
|
949
|
-
// Create abort controller for timeout
|
|
950
921
|
const controller = new AbortController();
|
|
951
|
-
const timeoutId = setTimeout(() => controller.abort(),
|
|
952
|
-
const mode = resolveValidationMode(this.config, this.adapter.id);
|
|
953
|
-
if (mode === "none") {
|
|
954
|
-
clearTimeout(timeoutId);
|
|
955
|
-
return true;
|
|
956
|
-
}
|
|
922
|
+
const timeoutId = setTimeout(() => controller.abort(), ENDPOINT_PROBE_TIMEOUT_MS);
|
|
957
923
|
const validationUrls = mode === "voices"
|
|
958
924
|
? resolveVoicesValidationUrls(this.config)
|
|
959
925
|
: [this.adapter.resolveSynthesisUrl(this.config)];
|
|
@@ -987,12 +953,9 @@ export class ServerTTSProvider {
|
|
|
987
953
|
getCapabilities() {
|
|
988
954
|
const mode = this.config ? resolveTransportMode(this.config) : "pie";
|
|
989
955
|
return {
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
supportsWordBoundary: true,
|
|
993
|
-
supportsVoiceSelection: true,
|
|
994
|
-
supportsRateControl: true,
|
|
995
|
-
supportsPitchControl: false, // Depends on server provider
|
|
956
|
+
// Word boundaries come from the server's speech marks; a response
|
|
957
|
+
// without them reports so at its playback start.
|
|
958
|
+
supportsWordBoundary: true,
|
|
996
959
|
supportsSSML: resolveSupportsSSML(this.config),
|
|
997
960
|
maxTextLength: MAX_TEXT_LENGTH_BY_MODE[mode],
|
|
998
961
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pie-players/tts-client-server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "1.0.0-beta.0",
|
|
4
4
|
"description": "Client-side TTS provider that calls server API for synthesis",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -17,8 +17,7 @@
|
|
|
17
17
|
"scripts": {
|
|
18
18
|
"build": "tsc",
|
|
19
19
|
"dev": "tsc --watch",
|
|
20
|
-
"test": "vitest run
|
|
21
|
-
"test:coverage": "vitest --coverage"
|
|
20
|
+
"test": "vitest run"
|
|
22
21
|
},
|
|
23
22
|
"keywords": [
|
|
24
23
|
"tts",
|
|
@@ -38,9 +37,9 @@
|
|
|
38
37
|
"access": "public"
|
|
39
38
|
},
|
|
40
39
|
"dependencies": {
|
|
41
|
-
"@pie-players/pie-players-shared": "0.
|
|
42
|
-
"@pie-players/pie-tts": "0.
|
|
43
|
-
"@pie-players/tts-server-core": "0.
|
|
40
|
+
"@pie-players/pie-players-shared": "1.0.0-beta.0",
|
|
41
|
+
"@pie-players/pie-tts": "1.0.0-beta.0",
|
|
42
|
+
"@pie-players/tts-server-core": "1.0.0-beta.0"
|
|
44
43
|
},
|
|
45
44
|
"devDependencies": {
|
|
46
45
|
"typescript": "^5.9.3",
|