@lovelace_lol/polymer 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +226 -0
- package/dist/index.d.ts +2603 -0
- package/dist/index.js +1031 -0
- package/dist/test.cjs +1519 -0
- package/dist/worker/agency-worker.js +972 -0
- package/dist/worker/manifest.edn +1 -0
- package/package.json +57 -0
- package/shadow-cljs.edn +173 -0
- package/src/polymer/animation/agency.cljs +84 -0
- package/src/polymer/animation/domain.cljs +110 -0
- package/src/polymer/animation/planner.cljs +88 -0
- package/src/polymer/animation/runtime.cljs +137 -0
- package/src/polymer/animation/scheduler.cljs +178 -0
- package/src/polymer/animation/service.cljs +981 -0
- package/src/polymer/animation/state.cljs +112 -0
- package/src/polymer/blink/agency.cljs +131 -0
- package/src/polymer/blink/planner.cljs +115 -0
- package/src/polymer/blink/scheduler.cljs +90 -0
- package/src/polymer/blink/snippet.cljs +50 -0
- package/src/polymer/blink/state.cljs +102 -0
- package/src/polymer/camera_context/agency.cljs +132 -0
- package/src/polymer/camera_context/domain.cljs +186 -0
- package/src/polymer/camera_context/planner.cljs +50 -0
- package/src/polymer/camera_context/scheduler.cljs +113 -0
- package/src/polymer/camera_context/state.cljs +121 -0
- package/src/polymer/character.cljs +471 -0
- package/src/polymer/conversation/agency.cljs +83 -0
- package/src/polymer/conversation/domain.cljs +35 -0
- package/src/polymer/conversation/effector.cljs +130 -0
- package/src/polymer/conversation/planner.cljs +178 -0
- package/src/polymer/conversation/scheduler.cljs +202 -0
- package/src/polymer/conversation/service.cljs +329 -0
- package/src/polymer/conversation/state.cljs +186 -0
- package/src/polymer/core.cljs +172 -0
- package/src/polymer/eye_head/agency.cljs +75 -0
- package/src/polymer/eye_head/domain.cljs +133 -0
- package/src/polymer/eye_head/goap.cljs +106 -0
- package/src/polymer/eye_head/scheduler.cljs +176 -0
- package/src/polymer/eye_head/service.cljs +259 -0
- package/src/polymer/eye_head/snippet.cljs +92 -0
- package/src/polymer/eye_head/state.cljs +167 -0
- package/src/polymer/gaze/agency.cljs +85 -0
- package/src/polymer/gaze/domain.cljs +211 -0
- package/src/polymer/gaze/goap.cljs +208 -0
- package/src/polymer/gaze/scheduler.cljs +184 -0
- package/src/polymer/gaze/state.cljs +202 -0
- package/src/polymer/gesture/agency.cljs +185 -0
- package/src/polymer/gesture/goap.cljs +461 -0
- package/src/polymer/gesture/scheduler.cljs +122 -0
- package/src/polymer/gesture/snippet.cljs +198 -0
- package/src/polymer/gesture/state.cljs +273 -0
- package/src/polymer/lipsync/agency.cljs +593 -0
- package/src/polymer/lipsync/articulation/jaw.cljs +258 -0
- package/src/polymer/lipsync/articulation/snippet.cljs +605 -0
- package/src/polymer/lipsync/articulation/tongue.cljs +418 -0
- package/src/polymer/lipsync/articulation/visemes.cljs +429 -0
- package/src/polymer/lipsync/goap.cljs +132 -0
- package/src/polymer/lipsync/pronunciation/effector.cljs +51 -0
- package/src/polymer/lipsync/scheduler.cljs +302 -0
- package/src/polymer/lipsync/state.cljs +202 -0
- package/src/polymer/nlp/phonemes.cljs +164 -0
- package/src/polymer/prosodic/agency.cljs +196 -0
- package/src/polymer/prosodic/goap.cljs +122 -0
- package/src/polymer/prosodic/scheduler.cljs +99 -0
- package/src/polymer/prosodic/snippet.cljs +106 -0
- package/src/polymer/prosodic/state.cljs +139 -0
- package/src/polymer/runtime/embody.cljs +117 -0
- package/src/polymer/runtime/embody_wasm.cljs +177 -0
- package/src/polymer/runtime/rust_embody_host.cljs +645 -0
- package/src/polymer/society.cljs +239 -0
- package/src/polymer/stream.cljs +130 -0
- package/src/polymer/transcription/agency.cljs +69 -0
- package/src/polymer/transcription/domain.cljs +68 -0
- package/src/polymer/transcription/effector.cljs +611 -0
- package/src/polymer/transcription/planner.cljs +203 -0
- package/src/polymer/transcription/scheduler.cljs +225 -0
- package/src/polymer/transcription/service.cljs +209 -0
- package/src/polymer/transcription/state.cljs +239 -0
- package/src/polymer/tts/agency.cljs +503 -0
- package/src/polymer/tts/azure.cljs +290 -0
- package/src/polymer/tts/planner.cljs +178 -0
- package/src/polymer/tts/runtime.cljs +624 -0
- package/src/polymer/tts/scheduler.cljs +162 -0
- package/src/polymer/tts/state.cljs +191 -0
- package/src/polymer/worker.cljs +70 -0
|
@@ -0,0 +1,593 @@
|
|
|
1
|
+
(ns polymer.lipsync.agency
|
|
2
|
+
(:require [polymer.lipsync.goap :as goap]
|
|
3
|
+
[polymer.stream :as stream]
|
|
4
|
+
[polymer.tts.azure :as azure]
|
|
5
|
+
[polymer.lipsync.scheduler :as scheduler]
|
|
6
|
+
[polymer.lipsync.pronunciation.effector :as pronunciation-effector]
|
|
7
|
+
[polymer.lipsync.articulation.snippet :as snippet]
|
|
8
|
+
[polymer.lipsync.state :as state]
|
|
9
|
+
[polymer.lipsync.articulation.visemes :as visemes]))
|
|
10
|
+
|
|
11
|
+
;; LipSync accepts provider/text timing data and produces animation intent.
|
|
12
|
+
;; It does not own audio playback, provider credentials, HTTP, LiveKit, storage,
|
|
13
|
+
;; or DOM APIs. Those side-effect agencies feed plain messages into this agency.
|
|
14
|
+
;; Inside Polymer, the agency system routes LipSync animation requests to the
|
|
15
|
+
;; Animation agency.
|
|
16
|
+
|
|
17
|
+
(defn debug-flag-enabled? [name]
|
|
18
|
+
(try
|
|
19
|
+
(let [search (when (exists? js/window)
|
|
20
|
+
(.. js/window -location -search))
|
|
21
|
+
params (when search (js/URLSearchParams. search))
|
|
22
|
+
value (when params (.get params name))]
|
|
23
|
+
(or (= value "1") (= value "true")))
|
|
24
|
+
(catch :default _ false)))
|
|
25
|
+
|
|
26
|
+
(defn lipsync-debug-enabled? []
|
|
27
|
+
(or (debug-flag-enabled? "polymerVocalDebug")
|
|
28
|
+
(debug-flag-enabled? "polymerLipSyncDebug")))
|
|
29
|
+
|
|
30
|
+
(defn debug-log! [label payload]
|
|
31
|
+
(when (lipsync-debug-enabled?)
|
|
32
|
+
(.info js/console (str label " " (.stringify js/JSON (clj->js payload))))))
|
|
33
|
+
|
|
34
|
+
(defn viseme-event-summary [events]
|
|
35
|
+
(into []
|
|
36
|
+
(comp
|
|
37
|
+
(take 120)
|
|
38
|
+
(map (fn [event]
|
|
39
|
+
{:visemeId (:visemeId event)
|
|
40
|
+
:phoneme (:phoneme event)
|
|
41
|
+
:phonemeClass (:phonemeClass event)
|
|
42
|
+
:offsetMs (:offsetMs event)
|
|
43
|
+
:durationMs (:durationMs event)
|
|
44
|
+
:intensity (:intensity event)
|
|
45
|
+
:jawActivation (:jawActivation event)})))
|
|
46
|
+
events))
|
|
47
|
+
|
|
48
|
+
(defn channel-summary [snippet-data]
|
|
49
|
+
(into []
|
|
50
|
+
(map (fn [channel]
|
|
51
|
+
(let [keyframes (:keyframes channel)
|
|
52
|
+
peak (transduce (map :intensity) max 0 keyframes)]
|
|
53
|
+
{:target (:target channel)
|
|
54
|
+
:frames (count keyframes)
|
|
55
|
+
:firstSec (:time (first keyframes))
|
|
56
|
+
:lastSec (:time (last keyframes))
|
|
57
|
+
:peak peak})))
|
|
58
|
+
(:channels snippet-data)))
|
|
59
|
+
|
|
60
|
+
(defn js-command [type value]
|
|
61
|
+
#js {:type type :value value})
|
|
62
|
+
|
|
63
|
+
(defn plan-failure-message
|
|
64
|
+
"Turn the failed LipSync plan step into a readable domain error."
|
|
65
|
+
[plan]
|
|
66
|
+
(let [step (first (:steps plan))]
|
|
67
|
+
(case (:reason step)
|
|
68
|
+
"unsupported-command" (str "lipSync plan failed: unsupported command " (:commandType step))
|
|
69
|
+
"missing-text" "lipSync plan failed: startText requires text"
|
|
70
|
+
"missing-timeline-visemes" "lipSync plan failed: startTimeline requires visemes"
|
|
71
|
+
"missing-provider-visemes" "lipSync plan failed: processAzureVisemes requires provider visemes"
|
|
72
|
+
"missing-word" "lipSync plan failed: wordBoundary requires a word"
|
|
73
|
+
(str "lipSync plan failed: " (:reason step)))))
|
|
74
|
+
|
|
75
|
+
(defn visual-lead-ms [input config]
|
|
76
|
+
(state/clamp 0 250 (state/number-or (:visualLeadMs input) (:visualLeadMs config))))
|
|
77
|
+
|
|
78
|
+
(defn apply-visual-lead [events lead-ms]
|
|
79
|
+
(let [lead (max 0 (state/number-or lead-ms 0))]
|
|
80
|
+
(mapv (fn [event]
|
|
81
|
+
(update event :offsetMs #(max 0 (- % lead))))
|
|
82
|
+
events)))
|
|
83
|
+
|
|
84
|
+
(defn normalize-timeline [timeline config]
|
|
85
|
+
(let [input (or timeline {})
|
|
86
|
+
source (or (:source input) "external")
|
|
87
|
+
text (:text input)
|
|
88
|
+
lead-ms (visual-lead-ms input config)
|
|
89
|
+
viseme-events (apply-visual-lead (snippet/normalize-events (:visemes input)) lead-ms)]
|
|
90
|
+
{:name (:name input)
|
|
91
|
+
:text text
|
|
92
|
+
:source source
|
|
93
|
+
:visemes viseme-events
|
|
94
|
+
:visualLeadMs lead-ms
|
|
95
|
+
:wordTimings (state/normalize-word-timings (:wordTimings input))
|
|
96
|
+
:durationSec (state/number-or (:durationSec input)
|
|
97
|
+
(if (empty? viseme-events)
|
|
98
|
+
0
|
|
99
|
+
(apply max (map #(/ (+ (:offsetMs %) (:durationMs %)) 1000) viseme-events))))
|
|
100
|
+
:config config}))
|
|
101
|
+
|
|
102
|
+
(defn normalize-observed-elapsed-sec [value expected max-time]
|
|
103
|
+
;; Web Speech boundary clocks are not consistent enough to trust blindly
|
|
104
|
+
;; across browsers. The spec says seconds, but some engines/adapters have
|
|
105
|
+
;; reported millisecond-looking values. Treat a value as milliseconds only
|
|
106
|
+
;; when it is clearly farther from the expected word start than value/1000.
|
|
107
|
+
;; Without this guard, the first nonzero boundary can seek a whole utterance
|
|
108
|
+
;; clip to its end, which looks like one lip movement at the beginning.
|
|
109
|
+
(when (state/finite-number? value)
|
|
110
|
+
(let [raw (max 0 value)
|
|
111
|
+
millis (/ raw 1000)
|
|
112
|
+
expected-start (when expected (:startSec expected))
|
|
113
|
+
raw-error (when (state/finite-number? expected-start)
|
|
114
|
+
(js/Math.abs (- raw expected-start)))
|
|
115
|
+
millis-error (when (state/finite-number? expected-start)
|
|
116
|
+
(js/Math.abs (- millis expected-start)))
|
|
117
|
+
max-time (max 0 (state/number-or max-time 0))
|
|
118
|
+
within-utterance? (or (zero? max-time)
|
|
119
|
+
(<= millis (+ max-time 2)))
|
|
120
|
+
expected-says-ms? (and (> raw 10)
|
|
121
|
+
raw-error
|
|
122
|
+
millis-error
|
|
123
|
+
(< millis-error raw-error)
|
|
124
|
+
within-utterance?)
|
|
125
|
+
max-time-says-ms? (and (> raw (+ max-time 2))
|
|
126
|
+
(> raw 50)
|
|
127
|
+
within-utterance?)]
|
|
128
|
+
(if (or expected-says-ms? max-time-says-ms?)
|
|
129
|
+
millis
|
|
130
|
+
raw))))
|
|
131
|
+
|
|
132
|
+
(defn choose-boundary-elapsed-sec [payload expected max-time]
|
|
133
|
+
;; Prefer Web Speech's elapsedTime after unit normalization. Some browsers
|
|
134
|
+
;; report 0 for later word boundaries, so LoomLarge also sends a monotonic
|
|
135
|
+
;; host clock. Use that host clock only when the provider clock is clearly not
|
|
136
|
+
;; advancing; otherwise provider timing remains the authority.
|
|
137
|
+
(let [observed (normalize-observed-elapsed-sec (:observedElapsedSec payload)
|
|
138
|
+
expected
|
|
139
|
+
max-time)
|
|
140
|
+
host (when (state/finite-number? (:hostElapsedSec payload))
|
|
141
|
+
(max 0 (:hostElapsedSec payload)))
|
|
142
|
+
expected-start (when expected (:startSec expected))]
|
|
143
|
+
(cond
|
|
144
|
+
(and host
|
|
145
|
+
observed
|
|
146
|
+
(state/finite-number? expected-start)
|
|
147
|
+
(> expected-start 0.08)
|
|
148
|
+
(< observed 0.001)
|
|
149
|
+
(> host 0.001))
|
|
150
|
+
host
|
|
151
|
+
|
|
152
|
+
observed
|
|
153
|
+
observed
|
|
154
|
+
|
|
155
|
+
host
|
|
156
|
+
host
|
|
157
|
+
|
|
158
|
+
:else
|
|
159
|
+
nil)))
|
|
160
|
+
|
|
161
|
+
(defn payload-duration-sec [payload]
|
|
162
|
+
(or (when (state/finite-number? (:durationSec payload))
|
|
163
|
+
(max 0 (:durationSec payload)))
|
|
164
|
+
(when (state/finite-number? (:totalDurationMs payload))
|
|
165
|
+
(/ (max 0 (:totalDurationMs payload)) 1000))))
|
|
166
|
+
|
|
167
|
+
(defn timing-duration-sec [word-timings]
|
|
168
|
+
(when (seq word-timings)
|
|
169
|
+
(apply max 0 (map :endSec (state/normalize-word-timings word-timings)))))
|
|
170
|
+
|
|
171
|
+
(defn prepare-text-timeline [dictionary text speech-rate payload]
|
|
172
|
+
(let [source (or (:source payload) "text")
|
|
173
|
+
events (visemes/text->visemes dictionary text speech-rate)
|
|
174
|
+
generated-word-timings (visemes/text->word-timings
|
|
175
|
+
dictionary text speech-rate)
|
|
176
|
+
supplied-word-timings (state/normalize-word-timings (:wordTimings payload))
|
|
177
|
+
base-duration-ms (visemes/timeline-duration-ms events)
|
|
178
|
+
explicit-duration-sec (or (payload-duration-sec payload)
|
|
179
|
+
(timing-duration-sec supplied-word-timings))
|
|
180
|
+
web-speech-duration-sec (when (= "webSpeech" source)
|
|
181
|
+
(/ (visemes/web-speech-duration-ms text speech-rate base-duration-ms)
|
|
182
|
+
1000))
|
|
183
|
+
target-duration-sec (or explicit-duration-sec
|
|
184
|
+
web-speech-duration-sec
|
|
185
|
+
(/ base-duration-ms 1000))
|
|
186
|
+
base-duration-sec (/ base-duration-ms 1000)
|
|
187
|
+
scale (if (and (pos? base-duration-sec)
|
|
188
|
+
(state/finite-number? target-duration-sec)
|
|
189
|
+
(> target-duration-sec 0))
|
|
190
|
+
(/ target-duration-sec base-duration-sec)
|
|
191
|
+
1)
|
|
192
|
+
should-scale? (> (js/Math.abs (- scale 1)) 0.02)
|
|
193
|
+
word-timings (if (seq supplied-word-timings)
|
|
194
|
+
supplied-word-timings
|
|
195
|
+
(if should-scale?
|
|
196
|
+
(visemes/scale-word-timings generated-word-timings scale)
|
|
197
|
+
generated-word-timings))]
|
|
198
|
+
{:name (:name payload)
|
|
199
|
+
:text text
|
|
200
|
+
:source source
|
|
201
|
+
:visemes (if should-scale?
|
|
202
|
+
(visemes/scale-events events scale)
|
|
203
|
+
events)
|
|
204
|
+
:wordTimings word-timings
|
|
205
|
+
:durationSec (if (state/finite-number? target-duration-sec)
|
|
206
|
+
target-duration-sec
|
|
207
|
+
base-duration-sec)}))
|
|
208
|
+
|
|
209
|
+
(defn timeline-name [timeline]
|
|
210
|
+
(or (:name timeline)
|
|
211
|
+
(str "lipSync_timeline_" (.now js/Date))))
|
|
212
|
+
|
|
213
|
+
(defn create-lipsync-agency
|
|
214
|
+
([config]
|
|
215
|
+
(create-lipsync-agency config nil))
|
|
216
|
+
([config runtime]
|
|
217
|
+
(let [input-stream (stream/create-stream)
|
|
218
|
+
event-stream (stream/create-stream)
|
|
219
|
+
state-atom (atom (state/config->state config))
|
|
220
|
+
disposed? (atom false)
|
|
221
|
+
emit-input (:emit input-stream)
|
|
222
|
+
emit-event (:emit event-stream)
|
|
223
|
+
pronunciation-provider (pronunciation-effector/create-effector runtime)
|
|
224
|
+
agency-scheduler (atom nil)]
|
|
225
|
+
;; Stream contract for this agency:
|
|
226
|
+
;; - input records accepted commands for tests/workers/debugging.
|
|
227
|
+
;; - events carry domain facts and cross-agency requests.
|
|
228
|
+
(letfn [(audio-clock-sec [payload]
|
|
229
|
+
(max 0
|
|
230
|
+
(state/number-or (or (:audioTimeSec payload)
|
|
231
|
+
(:currentTimeSec payload)
|
|
232
|
+
(:offsetSec payload))
|
|
233
|
+
0)))
|
|
234
|
+
|
|
235
|
+
(record-stop-state! [reason]
|
|
236
|
+
(let [stopped-at (state/now-ms)]
|
|
237
|
+
(swap! state-atom state/record-stop stopped-at reason)
|
|
238
|
+
(emit-event {:type "lipSyncTimelineStopped"
|
|
239
|
+
:agency "lipSync"
|
|
240
|
+
:reason reason
|
|
241
|
+
:stoppedAt stopped-at})))
|
|
242
|
+
|
|
243
|
+
(stop-local! [reason remove?]
|
|
244
|
+
(when-let [s @agency-scheduler]
|
|
245
|
+
((:stop-timeline s) reason remove?))
|
|
246
|
+
(record-stop-state! reason))
|
|
247
|
+
|
|
248
|
+
(finish-local! []
|
|
249
|
+
(when (:speaking @state-atom)
|
|
250
|
+
(stop-local! "completed" false)))
|
|
251
|
+
|
|
252
|
+
(start-timeline! [timeline]
|
|
253
|
+
(let [config (:config @state-atom)
|
|
254
|
+
normalized (normalize-timeline timeline config)
|
|
255
|
+
pending-audio-started-at (:audioStartedAt @state-atom)
|
|
256
|
+
pending-audio-time-sec (:audioTimeSec @state-atom)]
|
|
257
|
+
(if (empty? (:visemes normalized))
|
|
258
|
+
(emit-event {:type "error"
|
|
259
|
+
:agency "lipSync"
|
|
260
|
+
:message "lipSync timeline requires at least one viseme event"})
|
|
261
|
+
(do
|
|
262
|
+
;; One active utterance at a time keeps lip sync from
|
|
263
|
+
;; accumulating stale viseme snippets. State records the
|
|
264
|
+
;; replacement fact; the scheduler owns the animation
|
|
265
|
+
;; removal/start ordering for the old and new snippets.
|
|
266
|
+
(when (:speaking @state-atom)
|
|
267
|
+
(record-stop-state! "replaced"))
|
|
268
|
+
(let [name (timeline-name normalized)
|
|
269
|
+
built (if (:text normalized)
|
|
270
|
+
(snippet/build-text-snippet (:text normalized)
|
|
271
|
+
(:visemes normalized)
|
|
272
|
+
config)
|
|
273
|
+
(snippet/build-lipsync-snippet (:visemes normalized)
|
|
274
|
+
config
|
|
275
|
+
name))
|
|
276
|
+
;; Keep one snippet as the atomic animation unit for
|
|
277
|
+
;; an utterance. Individual visemes, jaw, and tongue
|
|
278
|
+
;; gestures are channels inside the snippet rather
|
|
279
|
+
;; than separately scheduled clips.
|
|
280
|
+
snippet-data (cond-> (assoc built :name (or (:name normalized) (:name built)))
|
|
281
|
+
(state/finite-number? (:durationSec normalized))
|
|
282
|
+
(assoc :maxTime (max (:maxTime built) (:durationSec normalized))))
|
|
283
|
+
started-at (state/now-ms)]
|
|
284
|
+
(debug-log! "[Polymer LipSync CLJS] normalized timeline"
|
|
285
|
+
{:name (:name snippet-data)
|
|
286
|
+
:source (:source normalized)
|
|
287
|
+
:visemeCount (count (:visemes normalized))
|
|
288
|
+
:uniqueVisemeIds (->> (:visemes normalized)
|
|
289
|
+
(map :visemeId)
|
|
290
|
+
set
|
|
291
|
+
sort
|
|
292
|
+
vec)
|
|
293
|
+
:visualLeadMs (:visualLeadMs normalized)
|
|
294
|
+
:maxTime (:maxTime snippet-data)
|
|
295
|
+
:visemes (viseme-event-summary (:visemes normalized))})
|
|
296
|
+
(debug-log! "[Polymer LipSync CLJS] snippet channels"
|
|
297
|
+
{:name (:name snippet-data)
|
|
298
|
+
:channelCount (count (:channels snippet-data))
|
|
299
|
+
:channels (channel-summary snippet-data)})
|
|
300
|
+
(swap! state-atom state/record-start normalized (:name snippet-data) started-at (:maxTime snippet-data))
|
|
301
|
+
(emit-event {:type "lipSyncTimelineStarted"
|
|
302
|
+
:agency "lipSync"
|
|
303
|
+
:name (:name snippet-data)
|
|
304
|
+
:source (:source normalized)
|
|
305
|
+
:text (:text normalized)
|
|
306
|
+
:visemeCount (count (:visemes normalized))
|
|
307
|
+
:maxTime (:maxTime snippet-data)
|
|
308
|
+
:startedAt started-at})
|
|
309
|
+
(when-let [s @agency-scheduler]
|
|
310
|
+
((:start-timeline s) snippet-data {:autoPlay true}))
|
|
311
|
+
(when (and (= "webSpeech" (:source normalized))
|
|
312
|
+
pending-audio-started-at)
|
|
313
|
+
(let [elapsed-since-start
|
|
314
|
+
(/ (max 0 (- started-at
|
|
315
|
+
pending-audio-started-at))
|
|
316
|
+
1000)
|
|
317
|
+
target-sec
|
|
318
|
+
(min (:maxTime snippet-data)
|
|
319
|
+
(+ (state/number-or
|
|
320
|
+
pending-audio-time-sec 0)
|
|
321
|
+
elapsed-since-start))]
|
|
322
|
+
(swap! state-atom
|
|
323
|
+
state/record-audio-started
|
|
324
|
+
target-sec
|
|
325
|
+
pending-audio-started-at)
|
|
326
|
+
(when-let [s @agency-scheduler]
|
|
327
|
+
((:seek s)
|
|
328
|
+
(:name snippet-data)
|
|
329
|
+
target-sec
|
|
330
|
+
"late-pronunciation-ready"))))
|
|
331
|
+
snippet-data)))))
|
|
332
|
+
|
|
333
|
+
(start-text! [payload]
|
|
334
|
+
(let [text (:text payload)
|
|
335
|
+
agency-scheduler @agency-scheduler]
|
|
336
|
+
(if (and (string? text) (pos? (count text)))
|
|
337
|
+
((:prepare-text agency-scheduler) payload)
|
|
338
|
+
(emit-event {:type "error"
|
|
339
|
+
:agency "lipSync"
|
|
340
|
+
:message "lipSync startText command requires text"}))))
|
|
341
|
+
|
|
342
|
+
(process-azure! [payload]
|
|
343
|
+
;; Compatibility command for Azure/LiveKit provider facts. The
|
|
344
|
+
;; Azure-specific viseme-id mapping lives in TTS; LipSync only
|
|
345
|
+
;; normalizes word timing metadata and then uses the shared
|
|
346
|
+
;; articulated timeline path.
|
|
347
|
+
(let [raw-options (or (:options payload) {})
|
|
348
|
+
word-timings (state/normalize-word-timings
|
|
349
|
+
(or (:wordTimings payload)
|
|
350
|
+
(:wordTimings raw-options)))
|
|
351
|
+
options (assoc (merge {:visualLeadMs (get-in @state-atom [:config :visualLeadMs])}
|
|
352
|
+
raw-options)
|
|
353
|
+
:wordTimings word-timings)
|
|
354
|
+
timeline (azure/azure-visemes->timeline (:visemes payload)
|
|
355
|
+
(:totalDurationMs payload)
|
|
356
|
+
options)]
|
|
357
|
+
(start-timeline! {:name (:name payload)
|
|
358
|
+
:text (:text payload)
|
|
359
|
+
:source (or (:source payload) "azure")
|
|
360
|
+
:visemes timeline
|
|
361
|
+
:visualLeadMs (:visualLeadMs options)
|
|
362
|
+
:wordTimings (:wordTimings options)
|
|
363
|
+
:durationSec (when (state/finite-number? (:totalDurationMs payload))
|
|
364
|
+
(/ (:totalDurationMs payload) 1000))})))
|
|
365
|
+
|
|
366
|
+
(handle-word-boundary! [payload]
|
|
367
|
+
(let [word (:word payload)
|
|
368
|
+
word-index (int (state/number-or (:wordIndex payload) (:wordIndex @state-atom)))
|
|
369
|
+
observed-at (state/now-ms)]
|
|
370
|
+
(swap! state-atom state/record-word-boundary word word-index observed-at)
|
|
371
|
+
(emit-event {:type "lipSyncWordBoundary"
|
|
372
|
+
:agency "lipSync"
|
|
373
|
+
:word word
|
|
374
|
+
:wordIndex word-index
|
|
375
|
+
:observedAt observed-at})
|
|
376
|
+
(let [current @state-atom
|
|
377
|
+
expected (get (:wordTimings current) word-index)
|
|
378
|
+
observed-elapsed-sec (choose-boundary-elapsed-sec payload
|
|
379
|
+
expected
|
|
380
|
+
(:maxTime current))
|
|
381
|
+
elapsed-sec (state/number-or observed-elapsed-sec
|
|
382
|
+
(if (:startTime current)
|
|
383
|
+
(/ (- observed-at (:startTime current)) 1000)
|
|
384
|
+
0))
|
|
385
|
+
drift-sec (when expected (- elapsed-sec (:startSec expected)))
|
|
386
|
+
threshold (get-in current [:config :wordDriftThresholdSec])
|
|
387
|
+
name (:snippetName current)]
|
|
388
|
+
(when (and expected name drift-sec (> (js/Math.abs drift-sec) threshold))
|
|
389
|
+
(let [target-sec (min (:maxTime current) (max 0 elapsed-sec))
|
|
390
|
+
corrected-at (state/now-ms)]
|
|
391
|
+
(swap! state-atom state/record-sync-correction name target-sec corrected-at)
|
|
392
|
+
(emit-event {:type "lipSyncSyncDrift"
|
|
393
|
+
:agency "lipSync"
|
|
394
|
+
:name name
|
|
395
|
+
:word word
|
|
396
|
+
:wordIndex word-index
|
|
397
|
+
:expectedSec (:startSec expected)
|
|
398
|
+
:observedSec elapsed-sec
|
|
399
|
+
:driftSec drift-sec
|
|
400
|
+
:targetSec target-sec
|
|
401
|
+
:correctedAt corrected-at})
|
|
402
|
+
(when-let [s @agency-scheduler]
|
|
403
|
+
((:seek s) name target-sec "word-boundary-drift")))))))
|
|
404
|
+
|
|
405
|
+
(handle-audio-started! [payload]
|
|
406
|
+
;; The host owns audio playback. This command tells LipSync when
|
|
407
|
+
;; that side effect actually started so Animation can align the
|
|
408
|
+
;; scheduled snippet to the audio clock without LoomLarge calling
|
|
409
|
+
;; Animation directly.
|
|
410
|
+
(let [current @state-atom
|
|
411
|
+
name (or (:name payload) (:snippetName current))
|
|
412
|
+
raw-audio-time-sec (audio-clock-sec payload)
|
|
413
|
+
audio-time-sec (if (:speaking current)
|
|
414
|
+
(min (:maxTime current)
|
|
415
|
+
raw-audio-time-sec)
|
|
416
|
+
raw-audio-time-sec)
|
|
417
|
+
observed-at (state/now-ms)]
|
|
418
|
+
(swap! state-atom state/record-audio-started audio-time-sec observed-at)
|
|
419
|
+
(emit-event {:type "lipSyncAudioStarted"
|
|
420
|
+
:agency "lipSync"
|
|
421
|
+
:name name
|
|
422
|
+
:audioTimeSec audio-time-sec
|
|
423
|
+
:observedAt observed-at})
|
|
424
|
+
(when (and (:speaking current)
|
|
425
|
+
(:snippetName current)
|
|
426
|
+
name)
|
|
427
|
+
(when-let [s @agency-scheduler]
|
|
428
|
+
((:seek s) name audio-time-sec "audio-started")))))
|
|
429
|
+
|
|
430
|
+
(handle-audio-time! [payload]
|
|
431
|
+
;; Optional low-frequency host clock correction. This is not a
|
|
432
|
+
;; frame loop; callers should send it only when a material drift
|
|
433
|
+
;; correction is needed.
|
|
434
|
+
(let [current @state-atom
|
|
435
|
+
name (or (:name payload) (:snippetName current))
|
|
436
|
+
raw-audio-time-sec (audio-clock-sec payload)
|
|
437
|
+
audio-time-sec (if (:speaking current)
|
|
438
|
+
(min (:maxTime current)
|
|
439
|
+
raw-audio-time-sec)
|
|
440
|
+
raw-audio-time-sec)
|
|
441
|
+
observed-at (state/now-ms)]
|
|
442
|
+
(swap! state-atom state/record-audio-time audio-time-sec observed-at)
|
|
443
|
+
(emit-event {:type "lipSyncAudioTime"
|
|
444
|
+
:agency "lipSync"
|
|
445
|
+
:name name
|
|
446
|
+
:audioTimeSec audio-time-sec
|
|
447
|
+
:observedAt observed-at})
|
|
448
|
+
(when name
|
|
449
|
+
(when-let [s @agency-scheduler]
|
|
450
|
+
((:seek s) name audio-time-sec "audio-time")))))
|
|
451
|
+
|
|
452
|
+
(update-word-timings! [payload]
|
|
453
|
+
(let [updated-at (state/now-ms)
|
|
454
|
+
timings (:wordTimings payload)]
|
|
455
|
+
(swap! state-atom state/record-word-timings timings updated-at)
|
|
456
|
+
(emit-event {:type "lipSyncWordTimingsUpdated"
|
|
457
|
+
:agency "lipSync"
|
|
458
|
+
:count (count (state/normalize-word-timings timings))
|
|
459
|
+
:updatedAt updated-at})))
|
|
460
|
+
|
|
461
|
+
(run-action! [action payload]
|
|
462
|
+
;; `goap/plan-command` decides which agency operation is allowed
|
|
463
|
+
;; for a command. Dispatch only echoes the input and reports the
|
|
464
|
+
;; plan; accepted actions run here so the command router does not
|
|
465
|
+
;; grow another copy of LipSync's state/scheduler policy.
|
|
466
|
+
(case (:op action)
|
|
467
|
+
"configure"
|
|
468
|
+
(do
|
|
469
|
+
(swap! state-atom state/configure (:config payload))
|
|
470
|
+
(emit-event {:type "lipSyncConfigChanged"
|
|
471
|
+
:agency "lipSync"
|
|
472
|
+
:state @state-atom}))
|
|
473
|
+
|
|
474
|
+
"start-text"
|
|
475
|
+
(start-text! payload)
|
|
476
|
+
|
|
477
|
+
"start-timeline"
|
|
478
|
+
(start-timeline! (:timeline payload))
|
|
479
|
+
|
|
480
|
+
"process-provider-visemes"
|
|
481
|
+
(process-azure! payload)
|
|
482
|
+
|
|
483
|
+
"record-word-boundary"
|
|
484
|
+
(handle-word-boundary! payload)
|
|
485
|
+
|
|
486
|
+
"align-audio-start"
|
|
487
|
+
(handle-audio-started! payload)
|
|
488
|
+
|
|
489
|
+
"align-audio-time"
|
|
490
|
+
(handle-audio-time! payload)
|
|
491
|
+
|
|
492
|
+
"update-word-timings"
|
|
493
|
+
(update-word-timings! payload)
|
|
494
|
+
|
|
495
|
+
"stop"
|
|
496
|
+
(stop-local! (:reason action) (:remove? action))
|
|
497
|
+
|
|
498
|
+
"reset-state"
|
|
499
|
+
(do
|
|
500
|
+
(reset! state-atom (state/config->state nil))
|
|
501
|
+
(emit-event {:type "lipSyncConfigChanged"
|
|
502
|
+
:agency "lipSync"
|
|
503
|
+
:state @state-atom}))
|
|
504
|
+
|
|
505
|
+
(emit-event {:type "error"
|
|
506
|
+
:agency "lipSync"
|
|
507
|
+
:message (str "Unknown LipSync planner action: " (:op action))})))
|
|
508
|
+
|
|
509
|
+
(dispatch! [command]
|
|
510
|
+
(when-not @disposed?
|
|
511
|
+
(let [payload (js->clj command :keywordize-keys true)
|
|
512
|
+
plan (goap/plan-command payload {:speaking (:speaking @state-atom)})]
|
|
513
|
+
;; Every command produces an input echo and a plan event before
|
|
514
|
+
;; any mutation. That makes failed/ignored commands observable
|
|
515
|
+
;; through the event stream instead of mutable state polling.
|
|
516
|
+
(emit-input {:type "command"
|
|
517
|
+
:agency "lipSync"
|
|
518
|
+
:command payload})
|
|
519
|
+
(emit-event {:type "lipSyncPlanCreated"
|
|
520
|
+
:agency "lipSync"
|
|
521
|
+
:plan plan})
|
|
522
|
+
(if-not (:ok plan)
|
|
523
|
+
(emit-event {:type "error"
|
|
524
|
+
:agency "lipSync"
|
|
525
|
+
:message (plan-failure-message plan)})
|
|
526
|
+
(doseq [action (:actions plan)]
|
|
527
|
+
(run-action! action payload))))))]
|
|
528
|
+
(reset! agency-scheduler
|
|
529
|
+
(scheduler/create-scheduler
|
|
530
|
+
{:emit-event emit-event
|
|
531
|
+
:on-finished finish-local!
|
|
532
|
+
:pronunciation-effector pronunciation-provider
|
|
533
|
+
:on-text-ready
|
|
534
|
+
(fn [payload dictionary]
|
|
535
|
+
(when-not @disposed?
|
|
536
|
+
(let [text (:text payload)
|
|
537
|
+
speech-rate (get-in @state-atom
|
|
538
|
+
[:config :speechRate])]
|
|
539
|
+
(start-timeline!
|
|
540
|
+
(prepare-text-timeline
|
|
541
|
+
dictionary text speech-rate payload)))))}))
|
|
542
|
+
((:preload-pronunciations @agency-scheduler))
|
|
543
|
+
#js {:dispatch dispatch!
|
|
544
|
+
:snapshot (fn [] (state/visible-state @state-atom))
|
|
545
|
+
:schedulerQueue (fn []
|
|
546
|
+
(clj->js (if-let [s @agency-scheduler]
|
|
547
|
+
((:queue s))
|
|
548
|
+
[])))
|
|
549
|
+
:input (stream/writable-port input-stream dispatch!)
|
|
550
|
+
:events (stream/readable-port event-stream)
|
|
551
|
+
;; Empty compatibility stream. Cross-agency animation requests are
|
|
552
|
+
;; domain events so the Polymer agency system can route them to
|
|
553
|
+
;; Animation without host UI code becoming the interpreter.
|
|
554
|
+
:subscribeInput (fn [listener] ((:subscribe input-stream) listener))
|
|
555
|
+
:subscribeEvents (fn [listener] ((:subscribe event-stream) listener))
|
|
556
|
+
:subscribe (fn [listener] ((:subscribe event-stream) listener))
|
|
557
|
+
:subscribeStatus (fn [listener] ((:subscribe event-stream) listener))
|
|
558
|
+
:configure (fn [next-config] (dispatch! #js {:type "configure" :config next-config}))
|
|
559
|
+
:startText (fn [text] (dispatch! #js {:type "startText" :text text}))
|
|
560
|
+
:startTimeline (fn [timeline] (dispatch! #js {:type "startTimeline" :timeline timeline}))
|
|
561
|
+
:processAzureVisemes (fn
|
|
562
|
+
([visemes] (dispatch! #js {:type "processAzureVisemes" :visemes visemes}))
|
|
563
|
+
([visemes total-duration-ms]
|
|
564
|
+
(dispatch! #js {:type "processAzureVisemes"
|
|
565
|
+
:visemes visemes
|
|
566
|
+
:totalDurationMs total-duration-ms})))
|
|
567
|
+
:wordBoundary (fn
|
|
568
|
+
([word] (dispatch! #js {:type "wordBoundary" :word word}))
|
|
569
|
+
([word word-index observed-elapsed-sec]
|
|
570
|
+
(dispatch! #js {:type "wordBoundary"
|
|
571
|
+
:word word
|
|
572
|
+
:wordIndex word-index
|
|
573
|
+
:observedElapsedSec observed-elapsed-sec})))
|
|
574
|
+
:audioStarted (fn
|
|
575
|
+
([] (dispatch! #js {:type "audioStarted"}))
|
|
576
|
+
([audio-time-sec]
|
|
577
|
+
(dispatch! #js {:type "audioStarted"
|
|
578
|
+
:audioTimeSec audio-time-sec})))
|
|
579
|
+
:audioTime (fn [audio-time-sec]
|
|
580
|
+
(dispatch! #js {:type "audioTime"
|
|
581
|
+
:audioTimeSec audio-time-sec}))
|
|
582
|
+
:updateWordTimings (fn [word-timings]
|
|
583
|
+
(dispatch! #js {:type "updateWordTimings"
|
|
584
|
+
:wordTimings word-timings}))
|
|
585
|
+
:stop (fn [] (dispatch! #js {:type "stop"}))
|
|
586
|
+
:reset (fn [] (dispatch! #js {:type "reset"}))
|
|
587
|
+
:dispose (fn []
|
|
588
|
+
(when-not @disposed?
|
|
589
|
+
(reset! disposed? true)
|
|
590
|
+
(when-let [s @agency-scheduler]
|
|
591
|
+
((:dispose s)))
|
|
592
|
+
((:dispose input-stream))
|
|
593
|
+
((:dispose event-stream))))}))))
|