@lovelace_lol/polymer 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +226 -0
- package/dist/index.d.ts +2603 -0
- package/dist/index.js +1031 -0
- package/dist/test.cjs +1519 -0
- package/dist/worker/agency-worker.js +972 -0
- package/dist/worker/manifest.edn +1 -0
- package/package.json +57 -0
- package/shadow-cljs.edn +173 -0
- package/src/polymer/animation/agency.cljs +84 -0
- package/src/polymer/animation/domain.cljs +110 -0
- package/src/polymer/animation/planner.cljs +88 -0
- package/src/polymer/animation/runtime.cljs +137 -0
- package/src/polymer/animation/scheduler.cljs +178 -0
- package/src/polymer/animation/service.cljs +981 -0
- package/src/polymer/animation/state.cljs +112 -0
- package/src/polymer/blink/agency.cljs +131 -0
- package/src/polymer/blink/planner.cljs +115 -0
- package/src/polymer/blink/scheduler.cljs +90 -0
- package/src/polymer/blink/snippet.cljs +50 -0
- package/src/polymer/blink/state.cljs +102 -0
- package/src/polymer/camera_context/agency.cljs +132 -0
- package/src/polymer/camera_context/domain.cljs +186 -0
- package/src/polymer/camera_context/planner.cljs +50 -0
- package/src/polymer/camera_context/scheduler.cljs +113 -0
- package/src/polymer/camera_context/state.cljs +121 -0
- package/src/polymer/character.cljs +471 -0
- package/src/polymer/conversation/agency.cljs +83 -0
- package/src/polymer/conversation/domain.cljs +35 -0
- package/src/polymer/conversation/effector.cljs +130 -0
- package/src/polymer/conversation/planner.cljs +178 -0
- package/src/polymer/conversation/scheduler.cljs +202 -0
- package/src/polymer/conversation/service.cljs +329 -0
- package/src/polymer/conversation/state.cljs +186 -0
- package/src/polymer/core.cljs +172 -0
- package/src/polymer/eye_head/agency.cljs +75 -0
- package/src/polymer/eye_head/domain.cljs +133 -0
- package/src/polymer/eye_head/goap.cljs +106 -0
- package/src/polymer/eye_head/scheduler.cljs +176 -0
- package/src/polymer/eye_head/service.cljs +259 -0
- package/src/polymer/eye_head/snippet.cljs +92 -0
- package/src/polymer/eye_head/state.cljs +167 -0
- package/src/polymer/gaze/agency.cljs +85 -0
- package/src/polymer/gaze/domain.cljs +211 -0
- package/src/polymer/gaze/goap.cljs +208 -0
- package/src/polymer/gaze/scheduler.cljs +184 -0
- package/src/polymer/gaze/state.cljs +202 -0
- package/src/polymer/gesture/agency.cljs +185 -0
- package/src/polymer/gesture/goap.cljs +461 -0
- package/src/polymer/gesture/scheduler.cljs +122 -0
- package/src/polymer/gesture/snippet.cljs +198 -0
- package/src/polymer/gesture/state.cljs +273 -0
- package/src/polymer/lipsync/agency.cljs +593 -0
- package/src/polymer/lipsync/articulation/jaw.cljs +258 -0
- package/src/polymer/lipsync/articulation/snippet.cljs +605 -0
- package/src/polymer/lipsync/articulation/tongue.cljs +418 -0
- package/src/polymer/lipsync/articulation/visemes.cljs +429 -0
- package/src/polymer/lipsync/goap.cljs +132 -0
- package/src/polymer/lipsync/pronunciation/effector.cljs +51 -0
- package/src/polymer/lipsync/scheduler.cljs +302 -0
- package/src/polymer/lipsync/state.cljs +202 -0
- package/src/polymer/nlp/phonemes.cljs +164 -0
- package/src/polymer/prosodic/agency.cljs +196 -0
- package/src/polymer/prosodic/goap.cljs +122 -0
- package/src/polymer/prosodic/scheduler.cljs +99 -0
- package/src/polymer/prosodic/snippet.cljs +106 -0
- package/src/polymer/prosodic/state.cljs +139 -0
- package/src/polymer/runtime/embody.cljs +117 -0
- package/src/polymer/runtime/embody_wasm.cljs +177 -0
- package/src/polymer/runtime/rust_embody_host.cljs +645 -0
- package/src/polymer/society.cljs +239 -0
- package/src/polymer/stream.cljs +130 -0
- package/src/polymer/transcription/agency.cljs +69 -0
- package/src/polymer/transcription/domain.cljs +68 -0
- package/src/polymer/transcription/effector.cljs +611 -0
- package/src/polymer/transcription/planner.cljs +203 -0
- package/src/polymer/transcription/scheduler.cljs +225 -0
- package/src/polymer/transcription/service.cljs +209 -0
- package/src/polymer/transcription/state.cljs +239 -0
- package/src/polymer/tts/agency.cljs +503 -0
- package/src/polymer/tts/azure.cljs +290 -0
- package/src/polymer/tts/planner.cljs +178 -0
- package/src/polymer/tts/runtime.cljs +624 -0
- package/src/polymer/tts/scheduler.cljs +162 -0
- package/src/polymer/tts/state.cljs +191 -0
- package/src/polymer/worker.cljs +70 -0
|
@@ -0,0 +1,624 @@
|
|
|
1
|
+
(ns polymer.tts.runtime
|
|
2
|
+
(:require [clojure.string :as str]
|
|
3
|
+
[polymer.tts.state :as state]))
|
|
4
|
+
|
|
5
|
+
;; TTS runtime functions are the provider/audio boundary for the TTS agency.
|
|
6
|
+
;;
|
|
7
|
+
;; This namespace is intentionally impure: it is the place that talks to browser
|
|
8
|
+
;; APIs, backend HTTP endpoints, injected test providers, HTMLAudio, Blob URLs,
|
|
9
|
+
;; and Web Speech utterance callbacks. Keeping those functions out of
|
|
10
|
+
;; `polymer.tts.agency` lets the agency stay focused on command planning,
|
|
11
|
+
;; session state, and plain event emission to LipSync.
|
|
12
|
+
|
|
13
|
+
(def azure-audio-unlock-data-url
|
|
14
|
+
;; This silent WAV primes HTMLAudio during a user gesture before real Azure audio.
|
|
15
|
+
"data:audio/wav;base64,UklGRsQAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
|
|
16
|
+
|
|
17
|
+
(defn now-ms
|
|
18
|
+
"Read the best available monotonic-ish time for public events."
|
|
19
|
+
[]
|
|
20
|
+
(if (and (exists? js/performance) (.-now js/performance))
|
|
21
|
+
(.now js/performance)
|
|
22
|
+
(.now js/Date)))
|
|
23
|
+
|
|
24
|
+
(defn window*
|
|
25
|
+
"Return browser window when the package is running in a browser."
|
|
26
|
+
[]
|
|
27
|
+
(when (exists? js/window)
|
|
28
|
+
js/window))
|
|
29
|
+
|
|
30
|
+
(defn speech-synthesis*
|
|
31
|
+
"Return the Web Speech synthesis object when available."
|
|
32
|
+
[]
|
|
33
|
+
(when-let [window (window*)]
|
|
34
|
+
(.-speechSynthesis window)))
|
|
35
|
+
|
|
36
|
+
(defn fetch*
|
|
37
|
+
"Return the host fetch function from globalThis/window when available."
|
|
38
|
+
[]
|
|
39
|
+
(or (when (exists? js/fetch) js/fetch)
|
|
40
|
+
(when-let [window (window*)]
|
|
41
|
+
(.-fetch window))))
|
|
42
|
+
|
|
43
|
+
(defn custom-provider
|
|
44
|
+
"Read an injected provider function from config for tests or alternate hosts."
|
|
45
|
+
[config key]
|
|
46
|
+
(get-in config [:providers key]))
|
|
47
|
+
|
|
48
|
+
(defn js-promise?
|
|
49
|
+
"Treat any thenable as a Promise for simple interop."
|
|
50
|
+
[value]
|
|
51
|
+
(and value (= "function" (goog/typeOf (aget value "then")))))
|
|
52
|
+
|
|
53
|
+
(defn js-callable?
|
|
54
|
+
"Return true when a JS interop value can be called."
|
|
55
|
+
[value]
|
|
56
|
+
(= "function" (goog/typeOf value)))
|
|
57
|
+
|
|
58
|
+
(defn promise-resolve
|
|
59
|
+
"Normalize raw values and promises into one async path."
|
|
60
|
+
[value]
|
|
61
|
+
(if (js-promise? value)
|
|
62
|
+
value
|
|
63
|
+
(js/Promise.resolve value)))
|
|
64
|
+
|
|
65
|
+
(defn js-error
|
|
66
|
+
"Create a JavaScript Error with a stable string message."
|
|
67
|
+
[message]
|
|
68
|
+
(js/Error. message))
|
|
69
|
+
|
|
70
|
+
(defn callback-event
|
|
71
|
+
"Normalize callback payloads from browser code or injected JS test providers."
|
|
72
|
+
[event]
|
|
73
|
+
(if (map? event)
|
|
74
|
+
event
|
|
75
|
+
(js->clj event :keywordize-keys true)))
|
|
76
|
+
|
|
77
|
+
(defn plan-failure-message
|
|
78
|
+
"Turn the first failed provider-plan step into a user/debug friendly error string."
|
|
79
|
+
[plan]
|
|
80
|
+
(let [step (first (:steps plan))
|
|
81
|
+
reason (:reason step)
|
|
82
|
+
engine (:engine step)]
|
|
83
|
+
(case reason
|
|
84
|
+
"missing-text" "TTS plan failed: missing text"
|
|
85
|
+
"provider-not-ready" (str "TTS plan failed: provider not ready"
|
|
86
|
+
(when engine (str " (" engine ")")))
|
|
87
|
+
"unsupported-engine" (str "TTS plan failed: unsupported engine"
|
|
88
|
+
(when engine (str " (" engine ")")))
|
|
89
|
+
(str "TTS plan failed: " reason))))
|
|
90
|
+
|
|
91
|
+
(defn scalar->azure-percent
|
|
92
|
+
"Convert UI scalar rate into Azure SSML percent strings."
|
|
93
|
+
[value]
|
|
94
|
+
(let [percent (js/Math.round (* (- value 1) 100))]
|
|
95
|
+
(if (pos? percent)
|
|
96
|
+
(str "+" percent "%")
|
|
97
|
+
(str percent "%"))))
|
|
98
|
+
|
|
99
|
+
(defn scalar->azure-pitch-percent
|
|
100
|
+
"Convert UI scalar pitch into Azure SSML percent strings.
|
|
101
|
+
|
|
102
|
+
Match Latticework's pitch mapping (* 50), not the rate mapping (* 100)."
|
|
103
|
+
[value]
|
|
104
|
+
(let [percent (js/Math.round (* (- value 1) 50))]
|
|
105
|
+
(if (pos? percent)
|
|
106
|
+
(str "+" percent "%")
|
|
107
|
+
(str percent "%"))))
|
|
108
|
+
|
|
109
|
+
(defn present-string?
|
|
110
|
+
"Return true for non-blank strings."
|
|
111
|
+
[value]
|
|
112
|
+
(and (string? value) (pos? (count (.trim value)))))
|
|
113
|
+
|
|
114
|
+
(defn azure-voice-name
|
|
115
|
+
"Resolve Azure voice with blank-safe fallbacks."
|
|
116
|
+
[config command]
|
|
117
|
+
(or (when (present-string? (:voiceName command)) (:voiceName command))
|
|
118
|
+
(when (present-string? (:azureVoiceName config)) (:azureVoiceName config))
|
|
119
|
+
"en-US-JennyNeural"))
|
|
120
|
+
|
|
121
|
+
(defn azure-style
|
|
122
|
+
"Resolve optional Azure style; blank becomes nil for the backend."
|
|
123
|
+
[config command]
|
|
124
|
+
(let [style (or (:style command) (:azureStyle config))]
|
|
125
|
+
(when (present-string? style) style)))
|
|
126
|
+
|
|
127
|
+
(defn backend-url
|
|
128
|
+
"Resolve backend URL from command first, then agency config."
|
|
129
|
+
[config command]
|
|
130
|
+
(state/string-or (:backendUrl command) (:backendUrl config)))
|
|
131
|
+
|
|
132
|
+
(defn http-error-detail
|
|
133
|
+
"Prefer FastAPI-style detail/body text over a bare statusText."
|
|
134
|
+
[status status-text body]
|
|
135
|
+
(let [detail (try
|
|
136
|
+
(let [parsed (js/JSON.parse body)
|
|
137
|
+
d (aget parsed "detail")]
|
|
138
|
+
(cond
|
|
139
|
+
(string? d) d
|
|
140
|
+
d (.stringify js/JSON d)
|
|
141
|
+
:else nil))
|
|
142
|
+
(catch :default _ nil))]
|
|
143
|
+
(str "HTTP " status " "
|
|
144
|
+
(or detail
|
|
145
|
+
(when (and (string? body) (pos? (count body))) body)
|
|
146
|
+
status-text
|
|
147
|
+
"request failed"))))
|
|
148
|
+
|
|
149
|
+
(defn json-request
|
|
150
|
+
"Make a JSON request through the browser fetch API."
|
|
151
|
+
[url options]
|
|
152
|
+
(if-let [fetch-fn (fetch*)]
|
|
153
|
+
(-> (.call fetch-fn js/globalThis url options)
|
|
154
|
+
(.then (fn [response]
|
|
155
|
+
(if (.-ok response)
|
|
156
|
+
(.json response)
|
|
157
|
+
(-> (.text response)
|
|
158
|
+
(.catch (fn [_] ""))
|
|
159
|
+
(.then (fn [body]
|
|
160
|
+
(throw (js-error (http-error-detail
|
|
161
|
+
(.-status response)
|
|
162
|
+
(.-statusText response)
|
|
163
|
+
body))))))))))
|
|
164
|
+
(js/Promise.reject (js-error "fetch is not available in this JavaScript runtime"))))
|
|
165
|
+
|
|
166
|
+
(defn post-json
|
|
167
|
+
"POST a JSON body and parse the JSON response."
|
|
168
|
+
[url body]
|
|
169
|
+
(json-request url
|
|
170
|
+
#js {:method "POST"
|
|
171
|
+
:headers #js {"Content-Type" "application/json"}
|
|
172
|
+
:body (.stringify js/JSON (clj->js body))}))
|
|
173
|
+
|
|
174
|
+
(defn get-json
|
|
175
|
+
"GET JSON from a backend endpoint."
|
|
176
|
+
[url]
|
|
177
|
+
(json-request url #js {:method "GET"}))
|
|
178
|
+
|
|
179
|
+
(defn audio-mime-type
|
|
180
|
+
"Infer an audio MIME type from Azure's response format string."
|
|
181
|
+
[format]
|
|
182
|
+
(let [raw (or format "")
|
|
183
|
+
lower (.toLowerCase raw)]
|
|
184
|
+
(cond
|
|
185
|
+
(.startsWith raw "audio/") raw
|
|
186
|
+
(or (.includes lower "mp3") (.includes lower "mpeg")) "audio/mpeg"
|
|
187
|
+
(.includes lower "wav") "audio/wav"
|
|
188
|
+
(.includes lower "ogg") "audio/ogg"
|
|
189
|
+
:else "audio/mpeg")))
|
|
190
|
+
|
|
191
|
+
(defn base64-payload
|
|
192
|
+
"Strip a data URL prefix so atob receives only base64 audio bytes."
|
|
193
|
+
[audio-base64]
|
|
194
|
+
(.replace (or audio-base64 "") #"^data:audio/[^;]+;base64," ""))
|
|
195
|
+
|
|
196
|
+
(defn base64->bytes
|
|
197
|
+
"Decode base64 audio into a Uint8Array for Blob/AudioContext playback."
|
|
198
|
+
[audio-base64]
|
|
199
|
+
(let [binary (js/atob (base64-payload audio-base64))
|
|
200
|
+
length (.-length binary)
|
|
201
|
+
bytes (js/Uint8Array. length)]
|
|
202
|
+
(doseq [index (range length)]
|
|
203
|
+
(aset bytes index (.charCodeAt binary index)))
|
|
204
|
+
bytes))
|
|
205
|
+
|
|
206
|
+
(defn base64->object-url
|
|
207
|
+
"Create an object URL for Azure audio playback."
|
|
208
|
+
[audio-base64 format]
|
|
209
|
+
(js/URL.createObjectURL
|
|
210
|
+
(js/Blob. #js [(base64->bytes audio-base64)]
|
|
211
|
+
#js {:type (audio-mime-type format)})))
|
|
212
|
+
|
|
213
|
+
(defn ensure-playback-audio-context!
|
|
214
|
+
"Create/resume the AudioContext used for Azure playback-reference capture."
|
|
215
|
+
[resources]
|
|
216
|
+
(when (exists? js/AudioContext)
|
|
217
|
+
(let [existing (:audioContext @resources)
|
|
218
|
+
ctx (if (or (nil? existing) (= "closed" (.-state existing)))
|
|
219
|
+
(js/AudioContext.)
|
|
220
|
+
existing)
|
|
221
|
+
destination (or (:playbackReferenceDestination @resources)
|
|
222
|
+
(.createMediaStreamDestination ctx))]
|
|
223
|
+
(swap! resources assoc
|
|
224
|
+
:audioContext ctx
|
|
225
|
+
:playbackReferenceDestination destination)
|
|
226
|
+
(when (= "suspended" (.-state ctx))
|
|
227
|
+
(let [resume (.resume ctx)]
|
|
228
|
+
(when (js-promise? resume)
|
|
229
|
+
(.catch resume (fn [_error] nil)))))
|
|
230
|
+
ctx)))
|
|
231
|
+
|
|
232
|
+
(defn ensure-html-audio-playback-graph!
|
|
233
|
+
"Route HTMLAudio through Web Audio so barge-in can hear agent playback.
|
|
234
|
+
|
|
235
|
+
Latticework Azure TTS taps a MediaStreamDestination on the same graph used
|
|
236
|
+
for speaker output. Polymer keeps HTMLAudio for autoplay unlock, then mirrors
|
|
237
|
+
that tap with MediaElementSource -> gain -> destination + reference stream."
|
|
238
|
+
[resources audio volume]
|
|
239
|
+
(when-let [ctx (ensure-playback-audio-context! resources)]
|
|
240
|
+
(when-not (:mediaElementSource @resources)
|
|
241
|
+
(try
|
|
242
|
+
(let [source (.createMediaElementSource ctx audio)
|
|
243
|
+
gain (.createGain ctx)
|
|
244
|
+
destination (:playbackReferenceDestination @resources)]
|
|
245
|
+
(set! (.-value (.-gain gain)) volume)
|
|
246
|
+
(.connect source gain)
|
|
247
|
+
(.connect gain (.-destination ctx))
|
|
248
|
+
(when destination
|
|
249
|
+
(.connect gain destination))
|
|
250
|
+
(swap! resources assoc
|
|
251
|
+
:mediaElementSource source
|
|
252
|
+
:playbackGain gain
|
|
253
|
+
:audio audio))
|
|
254
|
+
(catch :default error
|
|
255
|
+
(js/console.warn "[TTS] Failed to create MediaElementSource playback reference graph:" error))))
|
|
256
|
+
(when-let [gain (:playbackGain @resources)]
|
|
257
|
+
(set! (.-value (.-gain gain)) volume))
|
|
258
|
+
(aget (.getAudioTracks (.-stream (:playbackReferenceDestination @resources))) 0)))
|
|
259
|
+
|
|
260
|
+
(defn get-playback-reference-track
|
|
261
|
+
"Return the live agent playback reference track when available."
|
|
262
|
+
[resources]
|
|
263
|
+
(or (let [display (:displayMediaReferenceTrack @resources)]
|
|
264
|
+
(when (and display (= "live" (.-readyState display)))
|
|
265
|
+
display))
|
|
266
|
+
(when-let [destination (:playbackReferenceDestination @resources)]
|
|
267
|
+
(let [track (aget (.getAudioTracks (.-stream destination)) 0)]
|
|
268
|
+
(when (and track (= "live" (.-readyState track)))
|
|
269
|
+
track)))))
|
|
270
|
+
|
|
271
|
+
(defn clear-display-media-reference!
|
|
272
|
+
"Stop experimental Web Speech display-media capture."
|
|
273
|
+
[resources]
|
|
274
|
+
(when-let [stream (:displayMediaReferenceStream @resources)]
|
|
275
|
+
(doseq [track (array-seq (.getTracks stream))]
|
|
276
|
+
(set! (.-onended track) nil)
|
|
277
|
+
(when (= "live" (.-readyState track))
|
|
278
|
+
(.stop track))))
|
|
279
|
+
(swap! resources assoc
|
|
280
|
+
:displayMediaReferenceStream nil
|
|
281
|
+
:displayMediaReferenceTrack nil))
|
|
282
|
+
|
|
283
|
+
(defn ensure-web-speech-display-media-reference!
|
|
284
|
+
"Capture tab/system audio as an experimental Web Speech playback reference."
|
|
285
|
+
[resources]
|
|
286
|
+
(js/Promise.
|
|
287
|
+
(fn [resolve]
|
|
288
|
+
(let [existing (get-playback-reference-track resources)]
|
|
289
|
+
(if (and existing (:displayMediaReferenceTrack @resources))
|
|
290
|
+
(resolve "available")
|
|
291
|
+
(let [media (.-mediaDevices js/navigator)]
|
|
292
|
+
(if-not (and media (.-getDisplayMedia media))
|
|
293
|
+
(do
|
|
294
|
+
(js/console.warn "[TTS] Display media capture is not supported; Web Speech playback reference unavailable")
|
|
295
|
+
(resolve "unavailable"))
|
|
296
|
+
(do
|
|
297
|
+
(clear-display-media-reference! resources)
|
|
298
|
+
(-> (.getDisplayMedia media
|
|
299
|
+
(clj->js {:video true
|
|
300
|
+
:audio {:echoCancellation false
|
|
301
|
+
:noiseSuppression false
|
|
302
|
+
:autoGainControl false}
|
|
303
|
+
:preferCurrentTab true
|
|
304
|
+
:selfBrowserSurface "include"
|
|
305
|
+
:systemAudio "include"
|
|
306
|
+
:surfaceSwitching "exclude"}))
|
|
307
|
+
(.then (fn [stream]
|
|
308
|
+
(let [audio-track (aget (.getAudioTracks stream) 0)]
|
|
309
|
+
(if-not audio-track
|
|
310
|
+
(do
|
|
311
|
+
(doseq [track (array-seq (.getTracks stream))]
|
|
312
|
+
(.stop track))
|
|
313
|
+
(js/console.warn "[TTS] Display capture did not include audio; Web Speech playback reference unavailable")
|
|
314
|
+
(resolve "no-audio"))
|
|
315
|
+
(do
|
|
316
|
+
(swap! resources assoc
|
|
317
|
+
:displayMediaReferenceStream stream
|
|
318
|
+
:displayMediaReferenceTrack audio-track)
|
|
319
|
+
(doseq [track (array-seq (.getTracks stream))]
|
|
320
|
+
(set! (.-onended track)
|
|
321
|
+
(fn []
|
|
322
|
+
(clear-display-media-reference! resources))))
|
|
323
|
+
(js/console.info "[TTS] Using display media audio as experimental Web Speech playback reference")
|
|
324
|
+
(resolve "available"))))))
|
|
325
|
+
(.catch (fn [error]
|
|
326
|
+
(let [name (if (exists? (.-name error)) (.-name error) "")
|
|
327
|
+
status (if (= name "NotAllowedError") "denied" "failed")]
|
|
328
|
+
(js/console.warn "[TTS] Display media reference capture failed; continuing without Web Speech playback reference:" error)
|
|
329
|
+
(resolve status)))))))))))))
|
|
330
|
+
|
|
331
|
+
(defn prime-audio!
|
|
332
|
+
"Prime one HTMLAudio element while the user gesture is still fresh.
|
|
333
|
+
|
|
334
|
+
Intentionally does not attach the MediaElementSource graph yet. That graph is
|
|
335
|
+
created on the first real Azure play so the barge-in reference track carries
|
|
336
|
+
agent speech rather than the muted unlock clip."
|
|
337
|
+
[resources volume]
|
|
338
|
+
(when (exists? js/Audio)
|
|
339
|
+
(ensure-playback-audio-context! resources)
|
|
340
|
+
(let [audio (or (:audio @resources) (js/Audio.))]
|
|
341
|
+
(swap! resources assoc :audio audio)
|
|
342
|
+
(set! (.-muted audio) true)
|
|
343
|
+
(set! (.-loop audio) true)
|
|
344
|
+
(set! (.-volume audio) volume)
|
|
345
|
+
(set! (.-src audio) azure-audio-unlock-data-url)
|
|
346
|
+
(let [play-result (.play audio)]
|
|
347
|
+
(when (js-promise? play-result)
|
|
348
|
+
(.catch play-result (fn [_error] nil))))
|
|
349
|
+
audio)))
|
|
350
|
+
|
|
351
|
+
(defn reset-primed-audio!
|
|
352
|
+
"Reset the primed audio element before attaching real Azure audio."
|
|
353
|
+
[audio volume]
|
|
354
|
+
(.pause audio)
|
|
355
|
+
(set! (.-loop audio) false)
|
|
356
|
+
(set! (.-muted audio) false)
|
|
357
|
+
(set! (.-volume audio) volume)
|
|
358
|
+
(set! (.-onended audio) nil)
|
|
359
|
+
(set! (.-onerror audio) nil)
|
|
360
|
+
(.removeAttribute audio "src")
|
|
361
|
+
(.load audio))
|
|
362
|
+
|
|
363
|
+
(defn cleanup-audio!
|
|
364
|
+
"Stop and release the active audio element/object URL resources.
|
|
365
|
+
|
|
366
|
+
Keep the MediaElementSource graph and AudioContext alive across utterances so
|
|
367
|
+
the playback-reference track remains stable for interruption monitoring."
|
|
368
|
+
[resources]
|
|
369
|
+
(when-let [audio (:audio @resources)]
|
|
370
|
+
(.pause audio)
|
|
371
|
+
(set! (.-onended audio) nil)
|
|
372
|
+
(set! (.-onerror audio) nil)
|
|
373
|
+
(.removeAttribute audio "src"))
|
|
374
|
+
(when-let [source (:audioSource @resources)]
|
|
375
|
+
(try
|
|
376
|
+
(.stop source)
|
|
377
|
+
(catch :default _ nil)))
|
|
378
|
+
(when-let [url (:audioUrl @resources)]
|
|
379
|
+
(js/URL.revokeObjectURL url))
|
|
380
|
+
(swap! resources assoc
|
|
381
|
+
:audioUrl nil
|
|
382
|
+
:audioSource nil
|
|
383
|
+
:webSpeechFallbackTimer nil))
|
|
384
|
+
|
|
385
|
+
(defn dispose-playback-resources!
|
|
386
|
+
"Tear down playback-reference capture owned by the TTS agency."
|
|
387
|
+
[resources]
|
|
388
|
+
(cleanup-audio! resources)
|
|
389
|
+
(clear-display-media-reference! resources)
|
|
390
|
+
(when-let [source (:mediaElementSource @resources)]
|
|
391
|
+
(try (.disconnect source) (catch :default _ nil)))
|
|
392
|
+
(when-let [gain (:playbackGain @resources)]
|
|
393
|
+
(try (.disconnect gain) (catch :default _ nil)))
|
|
394
|
+
(when-let [ctx (:audioContext @resources)]
|
|
395
|
+
(try (.close ctx) (catch :default _ nil)))
|
|
396
|
+
(swap! resources assoc
|
|
397
|
+
:audio nil
|
|
398
|
+
:audioUrl nil
|
|
399
|
+
:audioSource nil
|
|
400
|
+
:mediaElementSource nil
|
|
401
|
+
:playbackGain nil
|
|
402
|
+
:audioContext nil
|
|
403
|
+
:playbackReferenceDestination nil
|
|
404
|
+
:webSpeechFallbackTimer nil))
|
|
405
|
+
|
|
406
|
+
(defn stop-web-speech-handle!
|
|
407
|
+
"Cancel a Web Speech handle returned by the browser or an injected provider."
|
|
408
|
+
[handle]
|
|
409
|
+
(when-let [stop (and handle (aget handle "stop"))]
|
|
410
|
+
(when (js-callable? stop)
|
|
411
|
+
(.call stop handle))))
|
|
412
|
+
|
|
413
|
+
(defn play-html-audio!
|
|
414
|
+
"Play Azure audio through HTMLAudio and report a clock once playback begins."
|
|
415
|
+
[resources audio-base64 audio-format volume]
|
|
416
|
+
(let [audio (or (:audio @resources) (js/Audio.))
|
|
417
|
+
url (base64->object-url audio-base64 audio-format)]
|
|
418
|
+
(reset-primed-audio! audio volume)
|
|
419
|
+
(swap! resources assoc :audio audio :audioUrl url)
|
|
420
|
+
(ensure-html-audio-playback-graph! resources audio volume)
|
|
421
|
+
(set! (.-preload audio) "auto")
|
|
422
|
+
(set! (.-volume audio) volume)
|
|
423
|
+
(set! (.-src audio) url)
|
|
424
|
+
(.load audio)
|
|
425
|
+
(-> (.play audio)
|
|
426
|
+
(.then (fn []
|
|
427
|
+
{:usingHtmlAudio true
|
|
428
|
+
:durationSec (if (state/finite-number? (.-duration audio)) (.-duration audio) 0)
|
|
429
|
+
:clock {:currentTime (fn [] (.-currentTime audio))
|
|
430
|
+
:shouldContinue (fn [] (and (not (.-paused audio)) (not (.-ended audio))))}
|
|
431
|
+
:audio audio
|
|
432
|
+
:playbackReferenceTrack (get-playback-reference-track resources)})))))
|
|
433
|
+
|
|
434
|
+
(defn play-azure-audio!
|
|
435
|
+
"Use an injected audio player when present, otherwise use browser HTMLAudio."
|
|
436
|
+
[config resources plan]
|
|
437
|
+
(if-let [custom (custom-provider config :azurePlayAudio)]
|
|
438
|
+
(promise-resolve (custom (clj->js plan)))
|
|
439
|
+
(play-html-audio! resources
|
|
440
|
+
(:audioBase64 plan)
|
|
441
|
+
(:audioFormat plan)
|
|
442
|
+
(:volume plan))))
|
|
443
|
+
(defn load-web-speech-voices!
|
|
444
|
+
"Load browser Web Speech voices, using an injected provider in tests."
|
|
445
|
+
[config]
|
|
446
|
+
(if-let [custom (custom-provider config :webSpeechVoices)]
|
|
447
|
+
(promise-resolve (custom))
|
|
448
|
+
(if-let [synthesis (speech-synthesis*)]
|
|
449
|
+
(js/Promise.
|
|
450
|
+
(fn [resolve _reject]
|
|
451
|
+
(letfn [(finish []
|
|
452
|
+
(resolve (clj->js (mapv (fn [voice]
|
|
453
|
+
{:id (.-name voice)
|
|
454
|
+
:name (.-name voice)
|
|
455
|
+
:language (.-lang voice)
|
|
456
|
+
:gender ""
|
|
457
|
+
:styles []})
|
|
458
|
+
(array-seq (.getVoices synthesis))))))]
|
|
459
|
+
(if (pos? (.-length (.getVoices synthesis)))
|
|
460
|
+
(finish)
|
|
461
|
+
(do
|
|
462
|
+
(set! (.-onvoiceschanged synthesis) finish)
|
|
463
|
+
(when-let [window (window*)]
|
|
464
|
+
(.setTimeout window finish 500)))))))
|
|
465
|
+
(js/Promise.reject (js-error "Web Speech synthesis is not available")))))
|
|
466
|
+
|
|
467
|
+
(defn load-azure-voices!
|
|
468
|
+
"Ask the configured backend to validate Azure and return available voices."
|
|
469
|
+
[config]
|
|
470
|
+
(if-let [custom (custom-provider config :azureVoices)]
|
|
471
|
+
(promise-resolve (custom))
|
|
472
|
+
(let [base (:backendUrl config)]
|
|
473
|
+
(if (pos? (count base))
|
|
474
|
+
(-> (post-json (str base "/api/azure-tts/connect") {})
|
|
475
|
+
(.then (fn [response]
|
|
476
|
+
(let [data (js->clj response :keywordize-keys true)]
|
|
477
|
+
(if (and (:valid data) (seq (:voices data)))
|
|
478
|
+
data
|
|
479
|
+
(-> (get-json (str base "/api/azure-tts/voices"))
|
|
480
|
+
(.then (fn [fallback]
|
|
481
|
+
(assoc (js->clj fallback :keywordize-keys true)
|
|
482
|
+
:valid false
|
|
483
|
+
:message "Backend Azure credentials are not configured.")))))))))
|
|
484
|
+
(js/Promise.reject (js-error "Azure backendUrl is not configured"))))))
|
|
485
|
+
|
|
486
|
+
(defn synthesize-azure!
|
|
487
|
+
"Request Azure synthesis through Polymer-owned backend protocol."
|
|
488
|
+
[config command]
|
|
489
|
+
(if-let [custom (custom-provider config :azureSynthesize)]
|
|
490
|
+
(promise-resolve (custom (clj->js command)))
|
|
491
|
+
(let [base (backend-url config command)]
|
|
492
|
+
(if (pos? (count base))
|
|
493
|
+
(let [style (azure-style config command)
|
|
494
|
+
style-degree (cond
|
|
495
|
+
(state/finite-number? (:styleDegree command)) (:styleDegree command)
|
|
496
|
+
(state/finite-number? (:azureStyleDegree config)) (:azureStyleDegree config)
|
|
497
|
+
:else nil)
|
|
498
|
+
body (cond-> {:text (:text command)
|
|
499
|
+
:voice_name (azure-voice-name config command)
|
|
500
|
+
:rate (scalar->azure-percent (or (:rate command) (:rate config)))
|
|
501
|
+
:pitch (scalar->azure-pitch-percent (or (:pitch command) (:pitch config)))}
|
|
502
|
+
style (assoc :style style)
|
|
503
|
+
(some? style-degree) (assoc :style_degree style-degree))]
|
|
504
|
+
(post-json (str base "/api/azure-tts/synthesize") body))
|
|
505
|
+
(js/Promise.reject (js-error "Azure backendUrl is not configured"))))))
|
|
506
|
+
|
|
507
|
+
(defn extract-boundary-word
|
|
508
|
+
"Recover a word from Web Speech charIndex callbacks."
|
|
509
|
+
[text char-index]
|
|
510
|
+
(let [safe-index (max 0 (min (count text) (or char-index 0)))
|
|
511
|
+
tail (.slice text safe-index)
|
|
512
|
+
forward (.match tail #"[A-Za-z']+")]
|
|
513
|
+
(if (and forward (aget forward 0))
|
|
514
|
+
(aget forward 0)
|
|
515
|
+
(let [head (.slice text 0 safe-index)
|
|
516
|
+
backward (.match head #"[A-Za-z']+$")]
|
|
517
|
+
(or (when backward (aget backward 0)) "")))))
|
|
518
|
+
|
|
519
|
+
(defn normalize-voice
|
|
520
|
+
"Normalize Web Speech/Azure voice objects into one UI-friendly shape."
|
|
521
|
+
[voice provider]
|
|
522
|
+
{:id (or (:id voice) (:name voice) (:shortName voice) "")
|
|
523
|
+
:name (or (:name voice) (:localName voice) (:id voice) "")
|
|
524
|
+
:language (or (:language voice) (:lang voice) (:locale voice) "")
|
|
525
|
+
:gender (or (:gender voice) "")
|
|
526
|
+
:styles (vec (or (:styles voice) []))
|
|
527
|
+
:provider provider})
|
|
528
|
+
|
|
529
|
+
(defn voice-xf
|
|
530
|
+
"Build the provider-specific voice normalization transducer."
|
|
531
|
+
[provider]
|
|
532
|
+
;; Voice loading is a pure provider-boundary cleanup step. Keeping the map/filter
|
|
533
|
+
;; as a local transducer gives one pass over provider lists without hiding any
|
|
534
|
+
;; HTTP, browser, or runtime side effects inside the transform.
|
|
535
|
+
(comp
|
|
536
|
+
(map #(normalize-voice % provider))
|
|
537
|
+
(filter #(pos? (count (:id %))))))
|
|
538
|
+
|
|
539
|
+
(defn normalize-voices
|
|
540
|
+
"Normalize voice choices and drop empty placeholder rows."
|
|
541
|
+
[voices provider]
|
|
542
|
+
(into [] (voice-xf provider) (or voices [])))
|
|
543
|
+
|
|
544
|
+
(defn azure-cache-key
|
|
545
|
+
"Build a stable Azure synthesis cache key from actual synthesis inputs."
|
|
546
|
+
[text voice-name style rate pitch]
|
|
547
|
+
(pr-str [(str/trim (or text "")) voice-name style rate pitch]))
|
|
548
|
+
|
|
549
|
+
(defn remember-cache
|
|
550
|
+
"Store one Azure synthesis result and evict oldest entries past the limit."
|
|
551
|
+
[cache key result limit]
|
|
552
|
+
(let [without-key (dissoc cache key)
|
|
553
|
+
next-cache (assoc without-key key result)
|
|
554
|
+
overflow (- (count next-cache) limit)]
|
|
555
|
+
(if (pos? overflow)
|
|
556
|
+
(apply dissoc next-cache (take overflow (keys next-cache)))
|
|
557
|
+
next-cache)))
|
|
558
|
+
|
|
559
|
+
(defn speak-web-speech!
|
|
560
|
+
"Start browser Web Speech playback or an injected test provider."
|
|
561
|
+
[config plan]
|
|
562
|
+
(cond
|
|
563
|
+
(custom-provider config :webSpeechSpeak)
|
|
564
|
+
(promise-resolve ((custom-provider config :webSpeechSpeak) (clj->js plan)))
|
|
565
|
+
|
|
566
|
+
(speech-synthesis*)
|
|
567
|
+
(let [synthesis (speech-synthesis*)
|
|
568
|
+
window (window*)
|
|
569
|
+
utterance (js/SpeechSynthesisUtterance. (:text plan))
|
|
570
|
+
voice-name (:voiceName plan)
|
|
571
|
+
voices (.getVoices synthesis)
|
|
572
|
+
matching-voice (some #(when (= voice-name (.-name %)) %) (array-seq voices))
|
|
573
|
+
visuals-started (atom false)
|
|
574
|
+
visual-started-at (atom nil)
|
|
575
|
+
clear-fallback (fn []
|
|
576
|
+
(when-let [cancel (:cancelStartFallback plan)]
|
|
577
|
+
(cancel)))
|
|
578
|
+
start-visuals (fn []
|
|
579
|
+
(when-not @visuals-started
|
|
580
|
+
(reset! visuals-started true)
|
|
581
|
+
(reset! visual-started-at (now-ms))
|
|
582
|
+
(clear-fallback)
|
|
583
|
+
((:onAudioStarted plan) {:currentTimeSec 0
|
|
584
|
+
:durationSec 0
|
|
585
|
+
:usingHtmlAudio false})))]
|
|
586
|
+
(when matching-voice
|
|
587
|
+
(set! (.-voice utterance) matching-voice))
|
|
588
|
+
(set! (.-rate utterance) (:rate plan))
|
|
589
|
+
(set! (.-pitch utterance) (:pitch plan))
|
|
590
|
+
(set! (.-volume utterance) (:volume plan))
|
|
591
|
+
(set! (.-onstart utterance) start-visuals)
|
|
592
|
+
(set! (.-onboundary utterance)
|
|
593
|
+
(fn [^js event]
|
|
594
|
+
(start-visuals)
|
|
595
|
+
(when (or (not (.-name event)) (= "word" (.-name event)))
|
|
596
|
+
((:onBoundary plan)
|
|
597
|
+
{:word (extract-boundary-word (:text plan) (.-charIndex event))
|
|
598
|
+
:observedElapsedSec (.-elapsedTime event)
|
|
599
|
+
:hostElapsedSec (when @visual-started-at
|
|
600
|
+
(/ (- (now-ms) @visual-started-at) 1000))}))))
|
|
601
|
+
(set! (.-onend utterance)
|
|
602
|
+
(fn []
|
|
603
|
+
(clear-fallback)
|
|
604
|
+
((:onEnd plan))))
|
|
605
|
+
(set! (.-onerror utterance)
|
|
606
|
+
(fn [^js event]
|
|
607
|
+
(clear-fallback)
|
|
608
|
+
((:onError plan) (js-error (str "Web Speech failed: " (.-error event))))))
|
|
609
|
+
(.speak synthesis utterance)
|
|
610
|
+
(if-let [schedule-fallback (:scheduleStartFallback plan)]
|
|
611
|
+
(schedule-fallback 120
|
|
612
|
+
(fn []
|
|
613
|
+
(when (.-speaking synthesis)
|
|
614
|
+
(start-visuals))))
|
|
615
|
+
(when window
|
|
616
|
+
(.setTimeout window
|
|
617
|
+
(fn []
|
|
618
|
+
(when (.-speaking synthesis)
|
|
619
|
+
(start-visuals)))
|
|
620
|
+
120)))
|
|
621
|
+
(js/Promise.resolve #js {:stop (fn [] (.cancel synthesis))}))
|
|
622
|
+
|
|
623
|
+
:else
|
|
624
|
+
(js/Promise.reject (js-error "Web Speech synthesis is not available"))))
|