@pie-players/pie-assessment-toolkit 0.3.75 → 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +451 -541
- package/custom-elements.json +1315 -0
- package/dist/attempt/TestSession.d.ts +0 -1
- package/dist/attempt/TestSession.js +10 -5
- package/dist/components/ItemScope.custom-element.js +1 -1
- package/dist/components/ItemToolBar.custom-element.js +1 -1
- package/dist/components/PieAssessmentToolkit.custom-element.js +8 -8
- package/dist/components/SectionToolBar.custom-element.js +1 -1
- package/dist/components/chunks/{ItemToolBar-7tt7v2xp.js → ItemToolBar-3zrc4p3d.js} +4 -4
- package/dist/components/chunks/{ItemToolBar-jx38w99s.js → ItemToolBar-66hcwmfw.js} +2 -2
- package/dist/components/chunks/ItemToolBar-asenkvx6.js +40 -0
- package/dist/components/chunks/ItemToolBar-hzc5wyjp.js +2 -0
- package/dist/components/chunks/{ItemToolBar-97hx60z4.js → ItemToolBar-kkj69ptr.js} +1 -1
- package/dist/components/chunks/sre-engine-bj997w4b.js +1 -0
- package/dist/context/assessment-toolkit-context.d.ts +6 -0
- package/dist/index.d.ts +7 -7
- package/dist/index.js +4 -10
- package/dist/policy/core/PolicySource.d.ts +1 -1
- package/dist/policy/core/ToolPolicyEngine.d.ts +46 -7
- package/dist/policy/core/ToolPolicyEngine.js +73 -16
- package/dist/policy/core/compose-decision.d.ts +3 -3
- package/dist/policy/core/compose-decision.js +32 -30
- package/dist/policy/core/decision-types.d.ts +51 -34
- package/dist/policy/core/feature-decision.d.ts +17 -15
- package/dist/policy/core/feature-decision.js +11 -13
- package/dist/policy/core/pnp-policy-inputs.d.ts +7 -3
- package/dist/policy/core/pnp-policy-inputs.js +17 -33
- package/dist/policy/core/provenance.d.ts +2 -2
- package/dist/policy/core/tool-parameters.d.ts +12 -0
- package/dist/policy/core/tool-parameters.js +27 -0
- package/dist/policy/engine.d.ts +1 -1
- package/dist/policy/sources/PnpPolicySource.d.ts +18 -24
- package/dist/policy/sources/PnpPolicySource.js +57 -29
- package/dist/runtime/SectionControllerBinding.d.ts +6 -2
- package/dist/runtime/SectionControllerBinding.js +21 -15
- package/dist/runtime/adapter/subscriber-fanout.d.ts +0 -1
- package/dist/runtime/adapter/subscriber-fanout.js +1 -4
- package/dist/runtime/composition-emit-scheduler.d.ts +3 -3
- package/dist/runtime/composition-emit-scheduler.js +1 -1
- package/dist/runtime/content-region.d.ts +0 -4
- package/dist/runtime/content-region.js +2 -2
- package/dist/runtime/core/SectionEngineCore.d.ts +0 -2
- package/dist/runtime/core/SectionEngineCore.js +0 -4
- package/dist/runtime/core/cohort.d.ts +0 -5
- package/dist/runtime/core/cohort.js +0 -9
- package/dist/runtime/core/engine-output.d.ts +3 -2
- package/dist/runtime/core/engine-transition.js +3 -2
- package/dist/runtime/registration-events.d.ts +11 -14
- package/dist/runtime/runtime-id.d.ts +1 -1
- package/dist/runtime/runtime-id.js +1 -1
- package/dist/runtime/shell-scope.d.ts +4 -4
- package/dist/runtime/shell-scope.js +15 -9
- package/dist/runtime/tool-host-contract.d.ts +0 -4
- package/dist/runtime/tool-host-contract.js +1 -1
- package/dist/services/AccessibilityCatalogResolver.d.ts +21 -31
- package/dist/services/AccessibilityCatalogResolver.js +29 -42
- package/dist/services/ElementToolStateStore.d.ts +32 -61
- package/dist/services/ElementToolStateStore.js +37 -67
- package/dist/services/HighlightCoordinator.d.ts +9 -21
- package/dist/services/HighlightCoordinator.js +52 -80
- package/dist/services/RangeSerializer.d.ts +4 -35
- package/dist/services/RangeSerializer.js +4 -48
- package/dist/services/SSMLExtractor.d.ts +8 -31
- package/dist/services/SSMLExtractor.js +18 -40
- package/dist/services/TTSService.d.ts +51 -57
- package/dist/services/TTSService.js +323 -284
- package/dist/services/ToolCoordinator.d.ts +6 -49
- package/dist/services/ToolCoordinator.js +25 -61
- package/dist/services/ToolRegistry.d.ts +83 -80
- package/dist/services/ToolRegistry.js +91 -67
- package/dist/services/ToolkitCoordinator.d.ts +83 -81
- package/dist/services/ToolkitCoordinator.js +186 -174
- package/dist/services/framework-error.d.ts +30 -7
- package/dist/services/framework-error.js +10 -13
- package/dist/services/interfaces.d.ts +51 -71
- package/dist/services/section-controller-types.d.ts +7 -7
- package/dist/services/section-session-assignment.js +6 -8
- package/dist/services/spoken-audio-cards.d.ts +2 -4
- package/dist/services/spoken-audio-cards.js +2 -4
- package/dist/services/tool-config-validation.d.ts +5 -7
- package/dist/services/tool-config-validation.js +11 -26
- package/dist/services/tool-context.d.ts +39 -24
- package/dist/services/tool-context.js +158 -92
- package/dist/services/tool-failure.d.ts +30 -0
- package/dist/services/tool-failure.js +11 -0
- package/dist/services/tool-icons.d.ts +1 -2
- package/dist/services/tool-icons.js +2 -2
- package/dist/services/tool-instance-id.d.ts +8 -6
- package/dist/services/tool-instance-id.js +12 -18
- package/dist/services/tool-providers/TTSToolProvider.d.ts +6 -43
- package/dist/services/tool-providers/TTSToolProvider.js +7 -63
- package/dist/services/tool-providers/ToolProviderApi.d.ts +0 -7
- package/dist/services/tool-providers/ToolProviderRegistry.d.ts +7 -42
- package/dist/services/tool-providers/ToolProviderRegistry.js +21 -63
- package/dist/services/tool-request.d.ts +4 -0
- package/dist/services/tool-request.js +7 -2
- package/dist/services/toolbar-decided-tools.d.ts +2 -0
- package/dist/services/toolbar-decided-tools.js +2 -0
- package/dist/services/toolbar-items.js +2 -5
- package/dist/services/tools-config-normalizer.d.ts +18 -9
- package/dist/services/tools-config-normalizer.js +25 -19
- package/dist/services/tts/browser-provider.d.ts +2 -5
- package/dist/services/tts/browser-provider.js +13 -22
- package/dist/services/tts/catalog-span-alignment.js +14 -7
- package/dist/services/tts/flat-tree.d.ts +0 -7
- package/dist/services/tts/flat-tree.js +2 -2
- package/dist/services/tts/generated-speech/dom/to-chunk-inputs.d.ts +2 -2
- package/dist/services/tts/generated-speech/index.d.ts +1 -1
- package/dist/services/tts/generated-speech/index.js +1 -1
- package/dist/services/tts/generated-speech/types.d.ts +1 -1
- package/dist/services/tts/generated-speech/types.js +1 -1
- package/dist/services/tts/highlight-pipeline/create-highlight-plan.js +10 -12
- package/dist/services/tts/highlight-target-resolver.d.ts +12 -0
- package/dist/services/tts/math-aware-text-processing.js +10 -29
- package/dist/services/tts/math-speech.d.ts +1 -1
- package/dist/services/tts/math-speech.js +3 -0
- package/dist/services/tts/mathml-sanitization.d.ts +7 -0
- package/dist/services/tts/mathml-sanitization.js +9 -1
- package/dist/services/tts/provider-options.d.ts +12 -1
- package/dist/services/tts/ssml/spoken-text.js +1 -1
- package/dist/services/tts/start-failure.d.ts +0 -7
- package/dist/services/tts/start-failure.js +1 -1
- package/dist/services/tts/text-segmentation.d.ts +5 -0
- package/dist/services/tts/text-segmentation.js +13 -10
- package/dist/services/tts-runtime-config.d.ts +32 -12
- package/dist/services/tts-runtime-config.js +14 -20
- package/dist/tools/content-capability-resolution.d.ts +2 -2
- package/dist/tools/registration.d.ts +7 -6
- package/dist/tools/registration.js +10 -9
- package/dist/tools/registrations/toolbar-registration-helpers.d.ts +6 -0
- package/dist/tools/registrations/toolbar-registration-helpers.js +6 -0
- package/dist/tools/tool-surface-host.js +1 -8
- package/package.json +8 -6
- package/dist/components/chunks/ItemToolBar-bxwyhr5p.js +0 -2
- package/dist/components/chunks/ItemToolBar-rd2bb4p6.js +0 -40
- package/dist/components/chunks/sre-engine-jd8cc08x.js +0 -1
- package/dist/services/ContextVariableStore.d.ts +0 -113
- package/dist/services/ContextVariableStore.js +0 -206
- package/dist/services/tts-control-events.d.ts +0 -13
- package/dist/services/tts-control-events.js +0 -7
- package/dist/utils/logger.d.ts +0 -6
- package/dist/utils/logger.js +0 -10
|
@@ -1,23 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* TTSService
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* and integrates with HighlightCoordinator for word highlighting.
|
|
4
|
+
* Owns one read-aloud session: the installed provider, playback state, and the
|
|
5
|
+
* word and sentence highlights painted through HighlightCoordinator.
|
|
7
6
|
*
|
|
8
|
-
*
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
11
|
-
* -
|
|
12
|
-
*
|
|
13
|
-
* -
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
7
|
+
* Layout:
|
|
8
|
+
* - Provider and host wiring: initialize, setCatalogResolver, the highlight
|
|
9
|
+
* target resolver and math-speech sources.
|
|
10
|
+
* - Content resolution: catalog spoken card, then catalog-composed chunks
|
|
11
|
+
* (script and/or recording with a plain fallback), then generated speech.
|
|
12
|
+
* - Speech plan: structural pauses, seek segments, sentence segments and the
|
|
13
|
+
* normalized-to-DOM position map.
|
|
14
|
+
* - Playback: speakContent → executeSpeakPlayback, the run guard (speakRunId,
|
|
15
|
+
* runOwner), the pause hold and the playback-start barrier.
|
|
16
|
+
* - Highlight painting: boundary handling, the host resolver remap, catalog
|
|
17
|
+
* highlight plans.
|
|
18
|
+
* - Controls and state: pause/resume/stop/seek/rate, setState and listeners.
|
|
17
19
|
*/
|
|
18
20
|
import { applyMediaFragment, enforceMediaFragment, } from "@pie-players/pie-players-shared/media";
|
|
19
21
|
import { createPieLogger, isTtsDebugEnabled, } from "@pie-players/pie-players-shared/pie";
|
|
20
|
-
import { HighlightType } from "./HighlightCoordinator.js";
|
|
21
22
|
import { resolveSpokenAudioMedia, } from "./spoken-audio-cards.js";
|
|
22
23
|
import { DEFAULT_CONTENT_LANGUAGE, findContentLanguage, findLangAttribute, } from "../runtime/content-language.js";
|
|
23
24
|
import { isSsmlDocument } from "./tts/ssml/spoken-text.js";
|
|
@@ -32,7 +33,6 @@ import { toTTSStartFailure } from "./tts/start-failure.js";
|
|
|
32
33
|
import { segmentSentences as segmentTextToSentences, splitTextToLength, } from "./tts/text-segmentation.js";
|
|
33
34
|
import { createTTSHighlightPlan, normalizeSpeechChunks, resolveReadableRegion, } from "./tts/highlight-pipeline/index.js";
|
|
34
35
|
import { createRangesFromVisibleMap } from "./tts/highlight-pipeline/visible-map-range.js";
|
|
35
|
-
import { PIE_TTS_CONTROL_HANDOFF_EVENT, } from "./tts-control-events.js";
|
|
36
36
|
// Debug lines need `PIE_TTS_DEBUG=1` or `globalThis.__PIE_TTS_DEBUG__ = true`.
|
|
37
37
|
const logger = createPieLogger("tts-service", isTtsDebugEnabled);
|
|
38
38
|
/**
|
|
@@ -46,6 +46,14 @@ export var PlaybackState;
|
|
|
46
46
|
PlaybackState["PAUSED"] = "paused";
|
|
47
47
|
PlaybackState["ERROR"] = "error";
|
|
48
48
|
})(PlaybackState || (PlaybackState = {}));
|
|
49
|
+
const DEFAULT_STRUCTURAL_PAUSE_PROFILE = {
|
|
50
|
+
baseMs: 280,
|
|
51
|
+
units: { minor: 0.65, section: 1.0, major: 1.35 },
|
|
52
|
+
minMs: 120,
|
|
53
|
+
maxMs: 900,
|
|
54
|
+
};
|
|
55
|
+
const MIN_PLAYBACK_RATE = 0.25;
|
|
56
|
+
const MAX_PLAYBACK_RATE = 4;
|
|
49
57
|
const optionalString = (value) => typeof value === "string" && value.trim() ? value.trim() : undefined;
|
|
50
58
|
const hostLocalesOf = (options, previous = {}) => {
|
|
51
59
|
if (!options)
|
|
@@ -89,12 +97,53 @@ const sameRenderableHighlightTarget = (left, right) => {
|
|
|
89
97
|
return false;
|
|
90
98
|
};
|
|
91
99
|
/**
|
|
92
|
-
*
|
|
93
|
-
*
|
|
100
|
+
* Where two texts first differ, with 20 characters of context each side;
|
|
101
|
+
* debug tracing only.
|
|
102
|
+
*/
|
|
103
|
+
const firstDifference = (spoken, normalized) => {
|
|
104
|
+
for (let i = 0; i < Math.min(spoken.length, normalized.length); i++) {
|
|
105
|
+
if (spoken[i] !== normalized[i]) {
|
|
106
|
+
return {
|
|
107
|
+
index: i,
|
|
108
|
+
spokenChar: spoken[i],
|
|
109
|
+
normalizedChar: normalized[i],
|
|
110
|
+
spokenContext: spoken.substring(Math.max(0, i - 20), i + 20),
|
|
111
|
+
normalizedContext: normalized.substring(Math.max(0, i - 20), i + 20),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return null;
|
|
116
|
+
};
|
|
117
|
+
/** The authored `data-tts-break-ms` pause, when it is a positive number. */
|
|
118
|
+
const explicitBreakMsOf = (element) => {
|
|
119
|
+
const ms = Number(element.getAttribute("data-tts-break-ms"));
|
|
120
|
+
return Number.isFinite(ms) && ms > 0 ? ms : null;
|
|
121
|
+
};
|
|
122
|
+
const isBlockDisplay = (element) => {
|
|
123
|
+
if (typeof window === "undefined")
|
|
124
|
+
return false;
|
|
125
|
+
const display = window.getComputedStyle(element).display;
|
|
126
|
+
return (display === "block" ||
|
|
127
|
+
display === "list-item" ||
|
|
128
|
+
display.startsWith("table") ||
|
|
129
|
+
display === "flex" ||
|
|
130
|
+
display === "grid");
|
|
131
|
+
};
|
|
132
|
+
/** `[start, end)` of `text` with its surrounding whitespace excluded. */
|
|
133
|
+
const trimSpan = (text, start, end) => {
|
|
134
|
+
const raw = text.substring(start, end);
|
|
135
|
+
return {
|
|
136
|
+
start: start + (raw.match(/^\s*/)?.[0].length || 0),
|
|
137
|
+
end: end - (raw.match(/\s*$/)?.[0].length || 0),
|
|
138
|
+
};
|
|
139
|
+
};
|
|
140
|
+
/**
|
|
94
141
|
* Instantiable service for text-to-speech functionality.
|
|
95
142
|
* Each instance manages its own playback state.
|
|
96
143
|
*/
|
|
97
144
|
export class TTSService {
|
|
145
|
+
// currentProvider is the provider the host installed; provider is the
|
|
146
|
+
// implementation its initialize() returned, which does the speaking.
|
|
98
147
|
currentProvider = null;
|
|
99
148
|
provider = null;
|
|
100
149
|
highlightCoordinator = null;
|
|
@@ -115,10 +164,18 @@ export class TTSService {
|
|
|
115
164
|
mathSpeechSource = null;
|
|
116
165
|
currentText = null;
|
|
117
166
|
currentContentElement = null;
|
|
167
|
+
// Offsets in the normalized visible text of the element being read, mapped
|
|
168
|
+
// to DOM text positions; word boundaries resolve through it.
|
|
118
169
|
normalizedToDOM = new Map();
|
|
119
170
|
listeners = new Set();
|
|
120
171
|
lastError = null;
|
|
172
|
+
// Bumped by every new run and by stop, restart and dispose; an async step
|
|
173
|
+
// that resumes under an older id has been superseded and returns.
|
|
121
174
|
speakRunId = 0;
|
|
175
|
+
runOwner = null;
|
|
176
|
+
// currentBoundaryOffset is the spoken-text offset the run has reached, which
|
|
177
|
+
// seek and word boundaries count from; activeWordBoundaryOffset is where the
|
|
178
|
+
// spoken text starts in the element's visible text, nonzero for a selection.
|
|
122
179
|
currentBoundaryOffset = 0;
|
|
123
180
|
activeWordBoundaryOffset = 0;
|
|
124
181
|
seekSegments = [];
|
|
@@ -142,20 +199,21 @@ export class TTSService {
|
|
|
142
199
|
pendingPlaybackRateRequest = null;
|
|
143
200
|
activePlaybackRate = null;
|
|
144
201
|
activeHighlightMode = "word";
|
|
202
|
+
// What the plan part being spoken paints in sentence mode.
|
|
203
|
+
wordlessPartHighlight = null;
|
|
145
204
|
lastRenderedRegionTarget = null;
|
|
146
205
|
telemetryReporter = null;
|
|
147
|
-
// Memoized SRE resolver for the runtime generated-speech path
|
|
206
|
+
// Memoized SRE resolver for the runtime generated-speech path.
|
|
148
207
|
// Caches DOM-free spoken text keyed by SRE settings and MathML source across
|
|
149
208
|
// utterances within this service instance.
|
|
150
209
|
generatedMathSpeechResolver = createMemoizedMathSpeechResolver();
|
|
151
|
-
constructor() { }
|
|
152
210
|
clearPlaybackStartBarrier(cleanup = this.playbackStartBarrierCleanup) {
|
|
153
211
|
cleanup?.();
|
|
154
212
|
if (this.playbackStartBarrierCleanup === cleanup) {
|
|
155
213
|
this.playbackStartBarrierCleanup = null;
|
|
156
214
|
}
|
|
157
215
|
}
|
|
158
|
-
notifyPlaybackStarted(runId) {
|
|
216
|
+
notifyPlaybackStarted(runId, playback) {
|
|
159
217
|
if (this.playbackStartDeferredRunId !== runId ||
|
|
160
218
|
runId !== this.speakRunId) {
|
|
161
219
|
return;
|
|
@@ -170,7 +228,27 @@ export class TTSService {
|
|
|
170
228
|
for (const applyHighlight of pendingHighlights) {
|
|
171
229
|
applyHighlight();
|
|
172
230
|
}
|
|
231
|
+
if (playback?.wordBoundaries === false)
|
|
232
|
+
this.highlightWordlessPlayback();
|
|
173
233
|
}
|
|
234
|
+
/**
|
|
235
|
+
* A word-mode part whose playback sends no boundaries, such as a server
|
|
236
|
+
* response without speech marks, highlights as a sentence-mode read does.
|
|
237
|
+
* Catalog chunks and a single speak highlight the same in both modes.
|
|
238
|
+
*/
|
|
239
|
+
highlightWordlessPlayback() {
|
|
240
|
+
const paint = this.wordlessPartHighlight;
|
|
241
|
+
if (!paint || this.activeHighlightMode !== "word")
|
|
242
|
+
return;
|
|
243
|
+
this.highlightCoordinator?.clearTTSWord();
|
|
244
|
+
paint();
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Defers marking the run playing, and the highlights queued through
|
|
248
|
+
* runWhenPlaybackStarts, until the provider or the recording signals that
|
|
249
|
+
* audio started. Returns the barrier's cleanup, or null when nothing will
|
|
250
|
+
* signal a start, in which case the caller marks the run playing itself.
|
|
251
|
+
*/
|
|
174
252
|
installPlaybackStartBarrier(runId, hasMediaStartSignal = false) {
|
|
175
253
|
this.clearPlaybackStartBarrier();
|
|
176
254
|
const provider = this.provider;
|
|
@@ -178,12 +256,12 @@ export class TTSService {
|
|
|
178
256
|
if (!startAwareProvider && !hasMediaStartSignal)
|
|
179
257
|
return null;
|
|
180
258
|
const previousOnPlaybackStart = startAwareProvider?.onPlaybackStart;
|
|
181
|
-
const onPlaybackStart = () => {
|
|
259
|
+
const onPlaybackStart = (playback) => {
|
|
182
260
|
try {
|
|
183
|
-
this.notifyPlaybackStarted(runId);
|
|
261
|
+
this.notifyPlaybackStarted(runId, playback);
|
|
184
262
|
}
|
|
185
263
|
finally {
|
|
186
|
-
previousOnPlaybackStart?.();
|
|
264
|
+
previousOnPlaybackStart?.(playback);
|
|
187
265
|
}
|
|
188
266
|
};
|
|
189
267
|
if (startAwareProvider) {
|
|
@@ -259,17 +337,19 @@ export class TTSService {
|
|
|
259
337
|
* run, whatever it is still waiting for: the provider, its content or the
|
|
260
338
|
* provider's audio.
|
|
261
339
|
*/
|
|
262
|
-
beginRun() {
|
|
340
|
+
beginRun(owner) {
|
|
263
341
|
// A recorded clip that starts the new run would otherwise play over the
|
|
264
342
|
// previous run's synthesis, which only the provider's next speak stops.
|
|
265
343
|
const supersedesPlayback = this.state === PlaybackState.LOADING ||
|
|
266
344
|
this.state === PlaybackState.PLAYING ||
|
|
267
345
|
this.state === PlaybackState.PAUSED;
|
|
268
346
|
const runId = ++this.speakRunId;
|
|
347
|
+
this.runOwner = owner;
|
|
269
348
|
this.abandonRun();
|
|
270
349
|
if (supersedesPlayback)
|
|
271
350
|
this.provider?.stop();
|
|
272
|
-
|
|
351
|
+
// Announced even from loading, so a listener sees the owner change.
|
|
352
|
+
this.setState(PlaybackState.LOADING, true);
|
|
273
353
|
return runId;
|
|
274
354
|
}
|
|
275
355
|
/** The run's audio is sounding; under a pause, it sounds again on resume. */
|
|
@@ -295,9 +375,6 @@ export class TTSService {
|
|
|
295
375
|
}
|
|
296
376
|
/**
|
|
297
377
|
* Initialize TTS service with a provider
|
|
298
|
-
*
|
|
299
|
-
* @param provider TTS provider instance
|
|
300
|
-
* @param config Provider configuration
|
|
301
378
|
*/
|
|
302
379
|
async initialize(provider, config = {}) {
|
|
303
380
|
if (this.disposed)
|
|
@@ -312,7 +389,8 @@ export class TTSService {
|
|
|
312
389
|
this.provider = await provider.initialize(config);
|
|
313
390
|
}
|
|
314
391
|
/**
|
|
315
|
-
* Update rate, pitch, voice or provider options without
|
|
392
|
+
* Update rate, pitch, volume, voice or provider options without
|
|
393
|
+
* reinitializing.
|
|
316
394
|
* `providerOptions` merge over the ones already configured.
|
|
317
395
|
*/
|
|
318
396
|
async updateSettings(settings) {
|
|
@@ -391,8 +469,6 @@ export class TTSService {
|
|
|
391
469
|
*
|
|
392
470
|
* When set, the speak() method will check for pre-authored spoken
|
|
393
471
|
* content in catalogs before falling back to generated TTS.
|
|
394
|
-
*
|
|
395
|
-
* @param resolver AccessibilityCatalogResolver instance
|
|
396
472
|
*/
|
|
397
473
|
setCatalogResolver(resolver) {
|
|
398
474
|
this.catalogResolver = resolver;
|
|
@@ -529,12 +605,6 @@ export class TTSService {
|
|
|
529
605
|
}
|
|
530
606
|
this.highlightCoordinator.highlightTTSSentence(paintedRanges);
|
|
531
607
|
}
|
|
532
|
-
/**
|
|
533
|
-
* Get provider capabilities
|
|
534
|
-
*/
|
|
535
|
-
getCapabilities() {
|
|
536
|
-
return this.currentProvider?.getCapabilities() || null;
|
|
537
|
-
}
|
|
538
608
|
providerOptions() {
|
|
539
609
|
const options = this.ttsConfig.providerOptions;
|
|
540
610
|
return options && typeof options === "object" ? options : {};
|
|
@@ -603,8 +673,8 @@ export class TTSService {
|
|
|
603
673
|
});
|
|
604
674
|
}
|
|
605
675
|
/**
|
|
606
|
-
*
|
|
607
|
-
*
|
|
676
|
+
* Text-collection options for `locale`, with the provider's boundary-spacing
|
|
677
|
+
* mode when it names a known one.
|
|
608
678
|
*/
|
|
609
679
|
getTextProcessingOptions(locale) {
|
|
610
680
|
const mode = this.providerOptions().textNormalization?.boundarySpacingMode;
|
|
@@ -640,6 +710,10 @@ export class TTSService {
|
|
|
640
710
|
}
|
|
641
711
|
return normalizeSREMathSpeechOptions(this.providerOptions().mathSpeech);
|
|
642
712
|
}
|
|
713
|
+
/**
|
|
714
|
+
* Rebuilds normalizedToDOM from `element`'s visible text; traced runs also
|
|
715
|
+
* log how it differs from `spokenText`.
|
|
716
|
+
*/
|
|
643
717
|
buildPositionMap(element, spokenText, language) {
|
|
644
718
|
this.normalizedToDOM.clear();
|
|
645
719
|
const { text: normalizedDomText, map } = collectVisibleTextAndMap(element, this.getTextProcessingOptions(language));
|
|
@@ -655,20 +729,7 @@ export class TTSService {
|
|
|
655
729
|
normalizedPreview: normalizedDomText.substring(0, 150),
|
|
656
730
|
differAtIndex: spokenText === normalizedDomText
|
|
657
731
|
? null
|
|
658
|
-
: (
|
|
659
|
-
for (let i = 0; i < Math.min(spokenText.length, normalizedDomText.length); i++) {
|
|
660
|
-
if (spokenText[i] !== normalizedDomText[i]) {
|
|
661
|
-
return {
|
|
662
|
-
index: i,
|
|
663
|
-
spokenChar: spokenText[i],
|
|
664
|
-
normalizedChar: normalizedDomText[i],
|
|
665
|
-
spokenContext: spokenText.substring(Math.max(0, i - 20), i + 20),
|
|
666
|
-
normalizedContext: normalizedDomText.substring(Math.max(0, i - 20), i + 20),
|
|
667
|
-
};
|
|
668
|
-
}
|
|
669
|
-
}
|
|
670
|
-
return null;
|
|
671
|
-
})(),
|
|
732
|
+
: firstDifference(spokenText, normalizedDomText),
|
|
672
733
|
});
|
|
673
734
|
logger.debug("Position map built:", {
|
|
674
735
|
entries: this.normalizedToDOM.size,
|
|
@@ -697,20 +758,21 @@ export class TTSService {
|
|
|
697
758
|
}
|
|
698
759
|
return fallback;
|
|
699
760
|
};
|
|
761
|
+
const defaults = DEFAULT_STRUCTURAL_PAUSE_PROFILE;
|
|
700
762
|
return {
|
|
701
|
-
baseMs: toNumber(custom.baseMs,
|
|
763
|
+
baseMs: toNumber(custom.baseMs, defaults.baseMs),
|
|
702
764
|
units: {
|
|
703
|
-
minor: toNumber(customUnits.minor,
|
|
704
|
-
section: toNumber(customUnits.section,
|
|
705
|
-
major: toNumber(customUnits.major,
|
|
765
|
+
minor: toNumber(customUnits.minor, defaults.units.minor),
|
|
766
|
+
section: toNumber(customUnits.section, defaults.units.section),
|
|
767
|
+
major: toNumber(customUnits.major, defaults.units.major),
|
|
706
768
|
},
|
|
707
|
-
minMs: toNumber(custom.minMs,
|
|
708
|
-
maxMs: toNumber(custom.maxMs,
|
|
769
|
+
minMs: toNumber(custom.minMs, defaults.minMs),
|
|
770
|
+
maxMs: toNumber(custom.maxMs, defaults.maxMs),
|
|
709
771
|
};
|
|
710
772
|
}
|
|
711
773
|
resolvePauseMsFromUnits(units) {
|
|
712
774
|
const profile = this.getStructuralPauseProfile();
|
|
713
|
-
const rate = Math.max(
|
|
775
|
+
const rate = Math.max(MIN_PLAYBACK_RATE, Math.min(MAX_PLAYBACK_RATE, Number(this.ttsConfig.rate || 1)));
|
|
714
776
|
const raw = (profile.baseMs * Math.max(0, units)) / rate;
|
|
715
777
|
return Math.max(profile.minMs, Math.min(profile.maxMs, Math.round(raw)));
|
|
716
778
|
}
|
|
@@ -730,12 +792,8 @@ export class TTSService {
|
|
|
730
792
|
return false;
|
|
731
793
|
return /<\s*break\b|<\s*speak\b|<\s*prosody\b|<\s*p\b|<\s*s\b/i.test(text);
|
|
732
794
|
}
|
|
733
|
-
isElementHidden(element) {
|
|
734
|
-
return isElementHiddenForTTS(element);
|
|
735
|
-
}
|
|
736
795
|
getBoundaryStrength(element) {
|
|
737
|
-
|
|
738
|
-
if (Number.isFinite(explicitBreakMs) && explicitBreakMs > 0)
|
|
796
|
+
if (explicitBreakMsOf(element) !== null)
|
|
739
797
|
return "major";
|
|
740
798
|
const tagName = element.tagName.toUpperCase();
|
|
741
799
|
if (tagName.match(/^H[1-6]$/))
|
|
@@ -750,26 +808,16 @@ export class TTSService {
|
|
|
750
808
|
tagName === "LI") {
|
|
751
809
|
return "section";
|
|
752
810
|
}
|
|
753
|
-
if (typeof window !== "undefined") {
|
|
754
|
-
const display = window.getComputedStyle(element).display;
|
|
755
|
-
if (display === "block" ||
|
|
756
|
-
display === "list-item" ||
|
|
757
|
-
display.startsWith("table") ||
|
|
758
|
-
display === "flex" ||
|
|
759
|
-
display === "grid") {
|
|
760
|
-
return "minor";
|
|
761
|
-
}
|
|
762
|
-
}
|
|
763
811
|
return "minor";
|
|
764
812
|
}
|
|
765
813
|
getBoundaryAnchor(textNode, root) {
|
|
766
814
|
let current = flatTreeParentElement(textNode);
|
|
767
815
|
let best = null;
|
|
768
816
|
while (current && current !== root) {
|
|
769
|
-
if (
|
|
817
|
+
if (isElementHiddenForTTS(current))
|
|
770
818
|
return null;
|
|
771
|
-
const explicitBreakMs =
|
|
772
|
-
if (
|
|
819
|
+
const explicitBreakMs = explicitBreakMsOf(current);
|
|
820
|
+
if (explicitBreakMs !== null) {
|
|
773
821
|
return {
|
|
774
822
|
anchor: current,
|
|
775
823
|
units: explicitBreakMs / this.getStructuralPauseProfile().baseMs,
|
|
@@ -784,18 +832,8 @@ export class TTSService {
|
|
|
784
832
|
best = current;
|
|
785
833
|
break;
|
|
786
834
|
}
|
|
787
|
-
if (!best)
|
|
788
|
-
|
|
789
|
-
const display = window.getComputedStyle(current).display;
|
|
790
|
-
if (display === "block" ||
|
|
791
|
-
display === "list-item" ||
|
|
792
|
-
display.startsWith("table") ||
|
|
793
|
-
display === "flex" ||
|
|
794
|
-
display === "grid") {
|
|
795
|
-
best = current;
|
|
796
|
-
}
|
|
797
|
-
}
|
|
798
|
-
}
|
|
835
|
+
if (!best && isBlockDisplay(current))
|
|
836
|
+
best = current;
|
|
799
837
|
current = flatTreeParentElement(current);
|
|
800
838
|
}
|
|
801
839
|
if (!best)
|
|
@@ -812,7 +850,7 @@ export class TTSService {
|
|
|
812
850
|
* part of it.
|
|
813
851
|
*/
|
|
814
852
|
createSpeechPlan(contentElement, normalizedText, textOffset, language) {
|
|
815
|
-
const boundaries = this.collectSpeechPlanBoundaries(contentElement,
|
|
853
|
+
const boundaries = this.collectSpeechPlanBoundaries(contentElement, language);
|
|
816
854
|
if (textOffset === 0) {
|
|
817
855
|
return this.createSpeechPlanSegments(normalizedText, boundaries);
|
|
818
856
|
}
|
|
@@ -822,7 +860,7 @@ export class TTSService {
|
|
|
822
860
|
}
|
|
823
861
|
return this.createSpeechPlanSegments(normalizedText, shifted);
|
|
824
862
|
}
|
|
825
|
-
collectSpeechPlanBoundaries(contentElement,
|
|
863
|
+
collectSpeechPlanBoundaries(contentElement, language) {
|
|
826
864
|
const boundaries = new Map();
|
|
827
865
|
// The same locale the position map used, so the boundary offsets index the
|
|
828
866
|
// text the map built.
|
|
@@ -832,7 +870,7 @@ export class TTSService {
|
|
|
832
870
|
for (const textNode of flatTreeTextNodes(contentElement)) {
|
|
833
871
|
const parent = flatTreeParentElement(textNode);
|
|
834
872
|
if (parent &&
|
|
835
|
-
!
|
|
873
|
+
!isElementHiddenForTTS(parent) &&
|
|
836
874
|
!isNodeSuppressedForTTS(textNode)) {
|
|
837
875
|
const boundary = this.getBoundaryAnchor(textNode, contentElement);
|
|
838
876
|
const boundaryPoint = nodeStartOffsets.get(textNode);
|
|
@@ -863,11 +901,7 @@ export class TTSService {
|
|
|
863
901
|
const segments = [];
|
|
864
902
|
let start = 0;
|
|
865
903
|
for (const point of points) {
|
|
866
|
-
const
|
|
867
|
-
const leading = raw.match(/^\s*/)?.[0].length || 0;
|
|
868
|
-
const trailing = raw.match(/\s*$/)?.[0].length || 0;
|
|
869
|
-
const segmentStart = start + leading;
|
|
870
|
-
const segmentEnd = point - trailing;
|
|
904
|
+
const { start: segmentStart, end: segmentEnd } = trimSpan(normalizedText, start, point);
|
|
871
905
|
if (segmentEnd > segmentStart) {
|
|
872
906
|
const text = normalizedText.substring(segmentStart, segmentEnd);
|
|
873
907
|
segments.push({
|
|
@@ -925,11 +959,7 @@ export class TTSService {
|
|
|
925
959
|
}
|
|
926
960
|
let cursor = segmentStart;
|
|
927
961
|
for (const point of [...splitPoints, segmentEnd]) {
|
|
928
|
-
const
|
|
929
|
-
const leadingWhitespace = raw.match(/^\s*/)?.[0].length || 0;
|
|
930
|
-
const trailingWhitespace = raw.match(/\s*$/)?.[0].length || 0;
|
|
931
|
-
const startOffset = cursor + leadingWhitespace;
|
|
932
|
-
const endOffset = point - trailingWhitespace;
|
|
962
|
+
const { start: startOffset, end: endOffset } = trimSpan(sourceText, cursor, point);
|
|
933
963
|
if (endOffset > startOffset) {
|
|
934
964
|
result.push({
|
|
935
965
|
text: sourceText.substring(startOffset, endOffset),
|
|
@@ -953,6 +983,9 @@ export class TTSService {
|
|
|
953
983
|
return this.splitSegmentsAtBoundaries(grammaticalSegments, structuralBoundaries, args.contentToSpeak);
|
|
954
984
|
}
|
|
955
985
|
createSeekSegmentsFromSpeechChunks(chunks) {
|
|
986
|
+
// One seek segment per chunk, at offsets into a virtual text that joins the
|
|
987
|
+
// chunks with one separator character. A chunk whose speech is SSML is
|
|
988
|
+
// measured by its plain fallback or visible text.
|
|
956
989
|
let offset = 0;
|
|
957
990
|
return chunks.map((chunk) => {
|
|
958
991
|
const speechText = chunk.plainFallback?.speechText ||
|
|
@@ -966,23 +999,12 @@ export class TTSService {
|
|
|
966
999
|
startOffset: offset,
|
|
967
1000
|
pauseMsAfter: 0,
|
|
968
1001
|
};
|
|
969
|
-
offset +=
|
|
1002
|
+
offset += text.length + 1;
|
|
970
1003
|
return segment;
|
|
971
1004
|
});
|
|
972
1005
|
}
|
|
973
1006
|
getCurrentSeekSegmentIndex() {
|
|
974
|
-
|
|
975
|
-
return 0;
|
|
976
|
-
let index = 0;
|
|
977
|
-
for (let i = 0; i < this.seekSegments.length; i++) {
|
|
978
|
-
if (this.seekSegments[i].startOffset <= this.currentBoundaryOffset) {
|
|
979
|
-
index = i;
|
|
980
|
-
}
|
|
981
|
-
else {
|
|
982
|
-
break;
|
|
983
|
-
}
|
|
984
|
-
}
|
|
985
|
-
return index;
|
|
1007
|
+
return Math.max(0, this.getSegmentIndexForOffset(this.seekSegments, this.currentBoundaryOffset));
|
|
986
1008
|
}
|
|
987
1009
|
async speakWithPlan(segments, runId, options) {
|
|
988
1010
|
if (!this.provider || segments.length === 0)
|
|
@@ -1022,12 +1044,21 @@ export class TTSService {
|
|
|
1022
1044
|
if (runId !== this.speakRunId)
|
|
1023
1045
|
return;
|
|
1024
1046
|
this.currentBoundaryOffset = segment.startOffset;
|
|
1047
|
+
const highlightSegment = () => {
|
|
1048
|
+
this.highlightSentenceSegment(segment.startOffset, segment.text);
|
|
1049
|
+
};
|
|
1025
1050
|
if (shouldTrackSentenceProgress) {
|
|
1026
|
-
this.runWhenPlaybackStarts(runId,
|
|
1027
|
-
|
|
1028
|
-
|
|
1051
|
+
this.runWhenPlaybackStarts(runId, highlightSegment);
|
|
1052
|
+
}
|
|
1053
|
+
this.wordlessPartHighlight = highlightSegment;
|
|
1054
|
+
try {
|
|
1055
|
+
await this.speakWithinLimit(segment.text, runId);
|
|
1056
|
+
}
|
|
1057
|
+
finally {
|
|
1058
|
+
if (this.wordlessPartHighlight === highlightSegment) {
|
|
1059
|
+
this.wordlessPartHighlight = null;
|
|
1060
|
+
}
|
|
1029
1061
|
}
|
|
1030
|
-
await this.speakWithinLimit(segment.text, runId);
|
|
1031
1062
|
const pauseMs = segment.pauseMsAfter ?? 0;
|
|
1032
1063
|
if (pauseMs > 0 && runId === this.speakRunId) {
|
|
1033
1064
|
await this.waitStructuralPause(pauseMs);
|
|
@@ -1100,7 +1131,8 @@ export class TTSService {
|
|
|
1100
1131
|
const pendingReadiness = this.pendingReadiness();
|
|
1101
1132
|
// A speak waiting on its provider is already the current run, so a pause
|
|
1102
1133
|
// or stop issued meanwhile applies to it.
|
|
1103
|
-
const
|
|
1134
|
+
const owner = options.ownerId ?? null;
|
|
1135
|
+
const waitingRunId = pendingReadiness ? this.beginRun(owner) : null;
|
|
1104
1136
|
if (pendingReadiness) {
|
|
1105
1137
|
try {
|
|
1106
1138
|
await pendingReadiness;
|
|
@@ -1128,7 +1160,38 @@ export class TTSService {
|
|
|
1128
1160
|
this.setState(PlaybackState.IDLE);
|
|
1129
1161
|
return;
|
|
1130
1162
|
}
|
|
1131
|
-
|
|
1163
|
+
const runId = waitingRunId ?? this.beginRun(owner);
|
|
1164
|
+
if (options.rate !== undefined ||
|
|
1165
|
+
options.pitch !== undefined ||
|
|
1166
|
+
options.volume !== undefined) {
|
|
1167
|
+
await this.applyRunSettings(options);
|
|
1168
|
+
if (runId !== this.speakRunId)
|
|
1169
|
+
return;
|
|
1170
|
+
}
|
|
1171
|
+
await this.speakContent(content.text, content.options, runId);
|
|
1172
|
+
}
|
|
1173
|
+
/**
|
|
1174
|
+
* Sets the rate, pitch and volume a run starts at, after any rate write in
|
|
1175
|
+
* flight. The run has no playback yet, so nothing restarts.
|
|
1176
|
+
*/
|
|
1177
|
+
async applyRunSettings(options) {
|
|
1178
|
+
await this.playbackRateWriteQueue;
|
|
1179
|
+
const settings = {};
|
|
1180
|
+
if (options.rate !== undefined) {
|
|
1181
|
+
const nextRate = this.normalizePlaybackRate(options.rate);
|
|
1182
|
+
const currentRate = this.normalizePlaybackRate(Number(this.ttsConfig.rate ?? 1));
|
|
1183
|
+
if (nextRate !== currentRate)
|
|
1184
|
+
settings.rate = nextRate;
|
|
1185
|
+
}
|
|
1186
|
+
if (options.pitch !== undefined && options.pitch !== this.ttsConfig.pitch) {
|
|
1187
|
+
settings.pitch = options.pitch;
|
|
1188
|
+
}
|
|
1189
|
+
if (options.volume !== undefined &&
|
|
1190
|
+
options.volume !== this.ttsConfig.volume) {
|
|
1191
|
+
settings.volume = options.volume;
|
|
1192
|
+
}
|
|
1193
|
+
if (Object.keys(settings).length > 0)
|
|
1194
|
+
await this.updateSettings(settings);
|
|
1132
1195
|
}
|
|
1133
1196
|
/**
|
|
1134
1197
|
* Suppression is read from the live ancestors of what is read, because the
|
|
@@ -1143,22 +1206,38 @@ export class TTSService {
|
|
|
1143
1206
|
resolveElementTarget(root, options, language) {
|
|
1144
1207
|
if (this.isSuppressedTarget(root))
|
|
1145
1208
|
return null;
|
|
1146
|
-
|
|
1209
|
+
const contentOptions = {
|
|
1210
|
+
catalogId: options.catalogId,
|
|
1211
|
+
catalogContext: options.catalogContext,
|
|
1212
|
+
language: language.locale,
|
|
1213
|
+
contentLanguage: language.contentLanguage,
|
|
1214
|
+
contentElement: root,
|
|
1215
|
+
};
|
|
1216
|
+
// Math-aware, so an element holding only an equation still reads; a spoken
|
|
1217
|
+
// card reads for content with no text, an image among them.
|
|
1147
1218
|
const text = collectMathAwareTextAndMap(root, this.getTextProcessingOptions(language.locale)).visibleText.trim();
|
|
1148
|
-
if (!text) {
|
|
1219
|
+
if (!text && !this.holdsSpokenCard(root, contentOptions)) {
|
|
1149
1220
|
console.warn("[tts] the content to read holds no speakable text; nothing was spoken.");
|
|
1150
1221
|
return null;
|
|
1151
1222
|
}
|
|
1152
|
-
return {
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1223
|
+
return { text, options: contentOptions };
|
|
1224
|
+
}
|
|
1225
|
+
/** Whether the root's card, or a card of a node inside it, would be read. */
|
|
1226
|
+
holdsSpokenCard(root, options) {
|
|
1227
|
+
if (!this.catalogResolver)
|
|
1228
|
+
return false;
|
|
1229
|
+
if (options.catalogId) {
|
|
1230
|
+
const card = this.catalogResolver.getAlternative(options.catalogId, {
|
|
1231
|
+
type: "spoken",
|
|
1232
|
+
language: options.language,
|
|
1233
|
+
useFallback: true,
|
|
1234
|
+
context: options.catalogContext,
|
|
1235
|
+
form: "content",
|
|
1236
|
+
});
|
|
1237
|
+
if (card?.content !== undefined)
|
|
1238
|
+
return true;
|
|
1239
|
+
}
|
|
1240
|
+
return this.collectCatalogSpeechChunks(root, options).some((chunk) => chunk.sourceElement);
|
|
1162
1241
|
}
|
|
1163
1242
|
resolveRangeTarget(range, options, language) {
|
|
1164
1243
|
if (this.isSuppressedTarget(range.commonAncestorContainer))
|
|
@@ -1237,27 +1316,35 @@ export class TTSService {
|
|
|
1237
1316
|
this.logResolvedSpeechContent({
|
|
1238
1317
|
contentToSpeak,
|
|
1239
1318
|
speechSource,
|
|
1240
|
-
catalogId: options
|
|
1319
|
+
catalogId: options.catalogId,
|
|
1241
1320
|
});
|
|
1242
1321
|
this.initializeSpeakTracking(highlightText, options);
|
|
1243
1322
|
const highlightMode = speechMatchesVisibleText
|
|
1244
1323
|
? this.resolveHighlightMode()
|
|
1245
1324
|
: "sentence";
|
|
1246
1325
|
this.activeHighlightMode = highlightMode;
|
|
1247
|
-
|
|
1326
|
+
const wordBoundaryOffset = options.wordBoundaryOffset || 0;
|
|
1327
|
+
this.activeWordBoundaryOffset = wordBoundaryOffset;
|
|
1248
1328
|
const hasExplicitBreaks = this.hasExplicitBreakSemantics(contentToSpeak);
|
|
1249
1329
|
const shouldUsePlan = !!this.currentContentElement &&
|
|
1250
1330
|
!usedCatalogSpoken &&
|
|
1251
1331
|
!hasExplicitBreaks &&
|
|
1252
1332
|
speechMatchesVisibleText;
|
|
1253
1333
|
this.playbackChunks = speechChunks?.length ? speechChunks : [];
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1334
|
+
if (this.playbackChunks.length) {
|
|
1335
|
+
this.seekSegments = this.createSeekSegmentsFromSpeechChunks(this.playbackChunks);
|
|
1336
|
+
}
|
|
1337
|
+
else if (hasExplicitBreaks || !speechMatchesVisibleText) {
|
|
1338
|
+
// Authored SSML breaks, and speech that differs from the visible text,
|
|
1339
|
+
// leave no visible-text offsets to seek to.
|
|
1340
|
+
this.seekSegments = [];
|
|
1341
|
+
}
|
|
1342
|
+
else if (shouldUsePlan && this.currentContentElement) {
|
|
1343
|
+
this.seekSegments = this.createSpeechPlan(this.currentContentElement, normalizedText, wordBoundaryOffset, options.language);
|
|
1344
|
+
}
|
|
1345
|
+
else {
|
|
1346
|
+
this.seekSegments = this.createSeekSegmentsFromText(highlightText);
|
|
1347
|
+
}
|
|
1261
1348
|
this.sentenceHighlightSegments = hasExplicitBreaks
|
|
1262
1349
|
? []
|
|
1263
1350
|
: this.createSentenceHighlightSegments({
|
|
@@ -1276,7 +1363,7 @@ export class TTSService {
|
|
|
1276
1363
|
if (speechMatchesVisibleText) {
|
|
1277
1364
|
this.configureWordBoundaryHighlighting({
|
|
1278
1365
|
highlightMode,
|
|
1279
|
-
wordBoundaryOffset
|
|
1366
|
+
wordBoundaryOffset,
|
|
1280
1367
|
});
|
|
1281
1368
|
}
|
|
1282
1369
|
else {
|
|
@@ -1421,6 +1508,9 @@ export class TTSService {
|
|
|
1421
1508
|
};
|
|
1422
1509
|
}
|
|
1423
1510
|
collectCatalogSpeechChunks(root, options) {
|
|
1511
|
+
const catalogResolver = this.catalogResolver;
|
|
1512
|
+
if (!catalogResolver)
|
|
1513
|
+
return [];
|
|
1424
1514
|
const chunks = [];
|
|
1425
1515
|
let textBuffer = "";
|
|
1426
1516
|
const flushTextBuffer = () => {
|
|
@@ -1457,14 +1547,14 @@ export class TTSService {
|
|
|
1457
1547
|
// Two lookups rather than one, because `form` is a preference: asking for
|
|
1458
1548
|
// the payload form happily returns a script card when no recording
|
|
1459
1549
|
// exists, so the answer has to be checked rather than assumed.
|
|
1460
|
-
const audioCard =
|
|
1550
|
+
const audioCard = catalogResolver.getAlternative(catalogIdRef, {
|
|
1461
1551
|
...lookup,
|
|
1462
1552
|
form: "payload",
|
|
1463
1553
|
});
|
|
1464
1554
|
const audio = audioCard?.payload
|
|
1465
1555
|
? (resolveSpokenAudioMedia({ payload: audioCard.payload }) ?? undefined)
|
|
1466
1556
|
: undefined;
|
|
1467
|
-
const scriptCard =
|
|
1557
|
+
const scriptCard = catalogResolver.getAlternative(catalogIdRef, {
|
|
1468
1558
|
...lookup,
|
|
1469
1559
|
form: "content",
|
|
1470
1560
|
});
|
|
@@ -1518,47 +1608,45 @@ export class TTSService {
|
|
|
1518
1608
|
const visibleText = collectedVisible.visibleText ||
|
|
1519
1609
|
normalizeTextForSpeech(flatTextContent(element));
|
|
1520
1610
|
const regionElement = resolveReadableRegion(element, root);
|
|
1521
|
-
// The script chunk
|
|
1522
|
-
//
|
|
1523
|
-
//
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1611
|
+
// The script chunk doubles as the recording's fallback: word-level
|
|
1612
|
+
// alignment is meaningful only on the synthesized path, which is the one
|
|
1613
|
+
// that runs if the audio cannot play.
|
|
1614
|
+
let scriptChunk = null;
|
|
1615
|
+
if (catalog.script !== undefined) {
|
|
1616
|
+
const script = catalog.script;
|
|
1617
|
+
const alignment = createCatalogSpanAlignment({
|
|
1618
|
+
speechText: script,
|
|
1619
|
+
visibleText,
|
|
1620
|
+
});
|
|
1621
|
+
const mathElement = getSingleMathElementForAlignment(element, visibleText);
|
|
1622
|
+
const mathAlignment = mathElement
|
|
1623
|
+
? createMathAwareAlignment({
|
|
1624
|
+
mathElement,
|
|
1529
1625
|
speechText: script,
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1626
|
+
})
|
|
1627
|
+
: undefined;
|
|
1628
|
+
const mathAlignments = mathAlignment
|
|
1629
|
+
? undefined
|
|
1630
|
+
: getMathElementsForAlignment(element).map((candidate) => ({
|
|
1631
|
+
element: candidate,
|
|
1632
|
+
alignment: createMathAwareAlignment({
|
|
1633
|
+
mathElement: candidate,
|
|
1536
1634
|
speechText: script,
|
|
1537
|
-
})
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
regionElement,
|
|
1553
|
-
speechMatchesVisibleText: normalizeTextForSpeech(script) === visibleText,
|
|
1554
|
-
playbackMode: alignment.playbackMode,
|
|
1555
|
-
alignment,
|
|
1556
|
-
mathAlignment,
|
|
1557
|
-
mathAlignments,
|
|
1558
|
-
visibleMap: collectedVisible.map,
|
|
1559
|
-
};
|
|
1560
|
-
})()
|
|
1561
|
-
: null;
|
|
1635
|
+
}),
|
|
1636
|
+
}));
|
|
1637
|
+
scriptChunk = {
|
|
1638
|
+
speechText: script,
|
|
1639
|
+
visibleText,
|
|
1640
|
+
sourceElement: element,
|
|
1641
|
+
regionElement,
|
|
1642
|
+
speechMatchesVisibleText: normalizeTextForSpeech(script) === visibleText,
|
|
1643
|
+
playbackMode: alignment.playbackMode,
|
|
1644
|
+
alignment,
|
|
1645
|
+
mathAlignment,
|
|
1646
|
+
mathAlignments,
|
|
1647
|
+
visibleMap: collectedVisible.map,
|
|
1648
|
+
};
|
|
1649
|
+
}
|
|
1562
1650
|
if (catalog.audio) {
|
|
1563
1651
|
chunks.push(scriptChunk
|
|
1564
1652
|
? {
|
|
@@ -1743,7 +1831,7 @@ export class TTSService {
|
|
|
1743
1831
|
this.lastRenderedRegionTarget = null;
|
|
1744
1832
|
if (chunk.regionRange) {
|
|
1745
1833
|
const ranges = createRangesFromVisibleMap(chunk.visibleMap, 0, chunk.visibleText.length);
|
|
1746
|
-
this.highlightCoordinator.
|
|
1834
|
+
this.highlightCoordinator.clearTTSWord();
|
|
1747
1835
|
this.paintTTSSentenceRanges(ranges.length > 1 ? ranges : [chunk.regionRange]);
|
|
1748
1836
|
return;
|
|
1749
1837
|
}
|
|
@@ -1751,7 +1839,7 @@ export class TTSService {
|
|
|
1751
1839
|
return;
|
|
1752
1840
|
const range = document.createRange();
|
|
1753
1841
|
range.selectNodeContents(element);
|
|
1754
|
-
this.highlightCoordinator.
|
|
1842
|
+
this.highlightCoordinator.clearTTSWord();
|
|
1755
1843
|
this.paintTTSSentenceRanges([range]);
|
|
1756
1844
|
}
|
|
1757
1845
|
catch {
|
|
@@ -1799,7 +1887,7 @@ export class TTSService {
|
|
|
1799
1887
|
// Clear the word layer so the previously spoken word does not stay
|
|
1800
1888
|
// highlighted. (The "hold last token" behavior returns a non-null
|
|
1801
1889
|
// target, so it does not reach here.)
|
|
1802
|
-
this.highlightCoordinator.
|
|
1890
|
+
this.highlightCoordinator.clearTTSWord();
|
|
1803
1891
|
return;
|
|
1804
1892
|
}
|
|
1805
1893
|
if (target.type === "text-range") {
|
|
@@ -1847,17 +1935,18 @@ export class TTSService {
|
|
|
1847
1935
|
await this.speakCatalogChunkOnce(chunk, runId);
|
|
1848
1936
|
}
|
|
1849
1937
|
catch (error) {
|
|
1850
|
-
//
|
|
1851
|
-
//
|
|
1852
|
-
//
|
|
1853
|
-
// fallback, so this cannot recurse.
|
|
1938
|
+
// A chunk with a plainFallback retries once with it when speaking fails:
|
|
1939
|
+
// a rejected SSML math chunk falls back to its plain-text variant, a
|
|
1940
|
+
// recording that cannot play to its script. The fallback carries no
|
|
1941
|
+
// fallback of its own, so this cannot recurse.
|
|
1854
1942
|
if (chunk.plainFallback && runId === this.speakRunId) {
|
|
1855
1943
|
logger.debug("SSML chunk speak failed; retrying plain text", {
|
|
1856
1944
|
message: error instanceof Error ? error.message : String(error),
|
|
1857
1945
|
});
|
|
1858
|
-
// A
|
|
1859
|
-
// provider
|
|
1860
|
-
//
|
|
1946
|
+
// A run that began with a recording holds the start barrier for the
|
|
1947
|
+
// recording's start signal. A provider with no start signal of its own
|
|
1948
|
+
// would never release it, so drop the recording's queued highlight and
|
|
1949
|
+
// mark the run playing now.
|
|
1861
1950
|
if (this.playbackStartDeferredRunId === runId &&
|
|
1862
1951
|
this.provider &&
|
|
1863
1952
|
!("onPlaybackStart" in this.provider)) {
|
|
@@ -2001,6 +2090,9 @@ export class TTSService {
|
|
|
2001
2090
|
mathTokenHighlighting,
|
|
2002
2091
|
})
|
|
2003
2092
|
: null;
|
|
2093
|
+
const providerOffsetSpace = !pipelineChunk || pipelineChunk.offsetSpace === "unsupported"
|
|
2094
|
+
? "unknown"
|
|
2095
|
+
: pipelineChunk.offsetSpace;
|
|
2004
2096
|
this.runWhenPlaybackStarts(runId, () => {
|
|
2005
2097
|
if (highlightPlan && pipelineChunk && chunk.mathAlignment) {
|
|
2006
2098
|
this.renderHighlightDecision(highlightPlan.resolveInitial(pipelineChunk.id));
|
|
@@ -2037,21 +2129,15 @@ export class TTSService {
|
|
|
2037
2129
|
const provider = this.provider;
|
|
2038
2130
|
const previousOnWordBoundary = provider.onWordBoundary;
|
|
2039
2131
|
const installedHandler = (word, position, length) => {
|
|
2040
|
-
if (runId !== this.speakRunId)
|
|
2132
|
+
if (runId !== this.speakRunId || !highlightPlan || !pipelineChunk)
|
|
2041
2133
|
return;
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
position,
|
|
2050
|
-
length,
|
|
2051
|
-
providerOffsetSpace,
|
|
2052
|
-
}));
|
|
2053
|
-
return;
|
|
2054
|
-
}
|
|
2134
|
+
this.renderHighlightDecision(highlightPlan.resolveBoundary({
|
|
2135
|
+
chunkId: pipelineChunk.id,
|
|
2136
|
+
word,
|
|
2137
|
+
position,
|
|
2138
|
+
length,
|
|
2139
|
+
providerOffsetSpace,
|
|
2140
|
+
}));
|
|
2055
2141
|
};
|
|
2056
2142
|
provider.onWordBoundary = installedHandler;
|
|
2057
2143
|
try {
|
|
@@ -2080,16 +2166,12 @@ export class TTSService {
|
|
|
2080
2166
|
}
|
|
2081
2167
|
return;
|
|
2082
2168
|
}
|
|
2083
|
-
if (args.shouldUsePlan &&
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
}
|
|
2090
|
-
else {
|
|
2091
|
-
await this.speakWithinLimit(args.contentToSpeak, args.runId);
|
|
2092
|
-
}
|
|
2169
|
+
if (args.shouldUsePlan &&
|
|
2170
|
+
this.currentContentElement &&
|
|
2171
|
+
this.seekSegments.length > 0) {
|
|
2172
|
+
await this.speakWithPlan(this.seekSegments, args.runId, {
|
|
2173
|
+
highlightMode: args.highlightMode,
|
|
2174
|
+
});
|
|
2093
2175
|
return;
|
|
2094
2176
|
}
|
|
2095
2177
|
await this.speakWithinLimit(args.contentToSpeak, args.runId);
|
|
@@ -2111,6 +2193,7 @@ export class TTSService {
|
|
|
2111
2193
|
this.playbackChunks = [];
|
|
2112
2194
|
this.sentenceHighlightSegments = [];
|
|
2113
2195
|
this.activeSentenceStartOffset = null;
|
|
2196
|
+
this.wordlessPartHighlight = null;
|
|
2114
2197
|
}
|
|
2115
2198
|
/**
|
|
2116
2199
|
* The selected part of `root`'s normalized visible text and where it starts, or
|
|
@@ -2171,7 +2254,7 @@ export class TTSService {
|
|
|
2171
2254
|
normalizePlaybackRate(rate) {
|
|
2172
2255
|
if (!Number.isFinite(rate))
|
|
2173
2256
|
return 1;
|
|
2174
|
-
return Math.max(
|
|
2257
|
+
return Math.max(MIN_PLAYBACK_RATE, Math.min(MAX_PLAYBACK_RATE, rate));
|
|
2175
2258
|
}
|
|
2176
2259
|
async restartFromSeekIndex(targetIndex) {
|
|
2177
2260
|
if (!this.provider || this.seekSegments.length === 0)
|
|
@@ -2233,8 +2316,11 @@ export class TTSService {
|
|
|
2233
2316
|
this.clearHighlightsAndTracking();
|
|
2234
2317
|
}
|
|
2235
2318
|
catch (error) {
|
|
2236
|
-
|
|
2319
|
+
// A superseded run's failure is the abort that superseded it.
|
|
2320
|
+
if (runId !== this.speakRunId) {
|
|
2321
|
+
logger.debug("Superseded seek restart failed:", error);
|
|
2237
2322
|
return;
|
|
2323
|
+
}
|
|
2238
2324
|
this.lastError = error instanceof Error ? error.message : String(error);
|
|
2239
2325
|
this.setState(PlaybackState.ERROR);
|
|
2240
2326
|
this.clearHighlightsAndTracking();
|
|
@@ -2332,32 +2418,19 @@ export class TTSService {
|
|
|
2332
2418
|
await this.seekBy(-step);
|
|
2333
2419
|
}
|
|
2334
2420
|
/**
|
|
2335
|
-
* Stop playback
|
|
2421
|
+
* Stop playback and release the run owner. A run that ends on its own keeps
|
|
2422
|
+
* its owner, so the owning control can replay it.
|
|
2336
2423
|
*/
|
|
2337
2424
|
stop() {
|
|
2338
2425
|
this.speakRunId += 1;
|
|
2339
2426
|
this.abandonRun();
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
this.provider.stop();
|
|
2343
|
-
}
|
|
2344
|
-
this.setState(PlaybackState.IDLE);
|
|
2427
|
+
this.provider?.stop();
|
|
2428
|
+
this.clearHighlightsAndTracking();
|
|
2345
2429
|
this.currentText = null;
|
|
2346
|
-
//
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
// Clear tracking
|
|
2351
|
-
this.lastRenderedRegionTarget = null;
|
|
2352
|
-
this.currentContentElement = null;
|
|
2353
|
-
this.normalizedToDOM.clear();
|
|
2354
|
-
this.currentBoundaryOffset = 0;
|
|
2355
|
-
this.activeWordBoundaryOffset = 0;
|
|
2356
|
-
this.seekSegments = [];
|
|
2357
|
-
this.playbackChunks = [];
|
|
2358
|
-
this.sentenceHighlightSegments = [];
|
|
2359
|
-
this.activeSentenceStartOffset = null;
|
|
2360
|
-
this.activePlaybackRate = null;
|
|
2430
|
+
// Announced even from idle, so a listener sees the owner released.
|
|
2431
|
+
const releasedOwner = this.runOwner !== null;
|
|
2432
|
+
this.runOwner = null;
|
|
2433
|
+
this.setState(PlaybackState.IDLE, releasedOwner);
|
|
2361
2434
|
}
|
|
2362
2435
|
/**
|
|
2363
2436
|
* Stop playback and drop the provider, whose owner destroys it. A later speak
|
|
@@ -2389,50 +2462,14 @@ export class TTSService {
|
|
|
2389
2462
|
this.catalogResolver = null;
|
|
2390
2463
|
this.telemetryReporter = null;
|
|
2391
2464
|
}
|
|
2392
|
-
/**
|
|
2393
|
-
* Request UI-level TTS controls to hand off/deactivate.
|
|
2394
|
-
*
|
|
2395
|
-
* This is intentionally separate from playback controls so hosts can orchestrate
|
|
2396
|
-
* control handoff explicitly (for example: stop playback, then dismiss controls).
|
|
2397
|
-
*/
|
|
2398
|
-
requestControlHandoff() {
|
|
2399
|
-
if (typeof window === "undefined")
|
|
2400
|
-
return;
|
|
2401
|
-
const detail = { source: "host" };
|
|
2402
|
-
window.dispatchEvent(new CustomEvent(PIE_TTS_CONTROL_HANDOFF_EVENT, {
|
|
2403
|
-
detail,
|
|
2404
|
-
}));
|
|
2405
|
-
}
|
|
2406
|
-
/**
|
|
2407
|
-
* Check if currently playing
|
|
2408
|
-
*/
|
|
2409
|
-
isPlaying() {
|
|
2410
|
-
return this.state === PlaybackState.PLAYING;
|
|
2411
|
-
}
|
|
2412
|
-
/**
|
|
2413
|
-
* Check if paused
|
|
2414
|
-
*/
|
|
2415
|
-
isPaused() {
|
|
2416
|
-
return this.state === PlaybackState.PAUSED;
|
|
2417
|
-
}
|
|
2418
2465
|
/**
|
|
2419
2466
|
* Get current state
|
|
2420
2467
|
*/
|
|
2421
2468
|
getState() {
|
|
2422
2469
|
return this.state;
|
|
2423
2470
|
}
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
*/
|
|
2427
|
-
getCurrentText() {
|
|
2428
|
-
return this.currentText;
|
|
2429
|
-
}
|
|
2430
|
-
/**
|
|
2431
|
-
* Get the last error message
|
|
2432
|
-
* Returns null if no error has occurred
|
|
2433
|
-
*/
|
|
2434
|
-
getLastError() {
|
|
2435
|
-
return this.lastError;
|
|
2471
|
+
getRunOwner() {
|
|
2472
|
+
return this.runOwner;
|
|
2436
2473
|
}
|
|
2437
2474
|
/**
|
|
2438
2475
|
* Subscribe to playback state changes. The returned function unsubscribes;
|
|
@@ -2446,17 +2483,19 @@ export class TTSService {
|
|
|
2446
2483
|
};
|
|
2447
2484
|
}
|
|
2448
2485
|
/**
|
|
2449
|
-
* Set state and notify listeners
|
|
2486
|
+
* Set state and notify listeners. `announceUnchanged` notifies listeners even
|
|
2487
|
+
* when the state does not change; telemetry fires only on a change.
|
|
2450
2488
|
*/
|
|
2451
|
-
setState(newState) {
|
|
2452
|
-
if (this.state === newState)
|
|
2453
|
-
return;
|
|
2489
|
+
setState(newState, announceUnchanged = false) {
|
|
2454
2490
|
const previousState = this.state;
|
|
2491
|
+
if (previousState === newState && !announceUnchanged)
|
|
2492
|
+
return;
|
|
2455
2493
|
this.state = newState;
|
|
2456
|
-
// Notify all listeners
|
|
2457
2494
|
for (const listener of [...this.listeners]) {
|
|
2458
2495
|
listener(newState);
|
|
2459
2496
|
}
|
|
2497
|
+
if (previousState === newState)
|
|
2498
|
+
return;
|
|
2460
2499
|
void this.emitTelemetry("pie-tool-playback-state-changed", {
|
|
2461
2500
|
toolId: "textToSpeech",
|
|
2462
2501
|
previousState,
|