@lovelace_lol/polymer 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/README.md +226 -0
  2. package/dist/index.d.ts +2603 -0
  3. package/dist/index.js +1031 -0
  4. package/dist/test.cjs +1519 -0
  5. package/dist/worker/agency-worker.js +972 -0
  6. package/dist/worker/manifest.edn +1 -0
  7. package/package.json +57 -0
  8. package/shadow-cljs.edn +173 -0
  9. package/src/polymer/animation/agency.cljs +84 -0
  10. package/src/polymer/animation/domain.cljs +110 -0
  11. package/src/polymer/animation/planner.cljs +88 -0
  12. package/src/polymer/animation/runtime.cljs +137 -0
  13. package/src/polymer/animation/scheduler.cljs +178 -0
  14. package/src/polymer/animation/service.cljs +981 -0
  15. package/src/polymer/animation/state.cljs +112 -0
  16. package/src/polymer/blink/agency.cljs +131 -0
  17. package/src/polymer/blink/planner.cljs +115 -0
  18. package/src/polymer/blink/scheduler.cljs +90 -0
  19. package/src/polymer/blink/snippet.cljs +50 -0
  20. package/src/polymer/blink/state.cljs +102 -0
  21. package/src/polymer/camera_context/agency.cljs +132 -0
  22. package/src/polymer/camera_context/domain.cljs +186 -0
  23. package/src/polymer/camera_context/planner.cljs +50 -0
  24. package/src/polymer/camera_context/scheduler.cljs +113 -0
  25. package/src/polymer/camera_context/state.cljs +121 -0
  26. package/src/polymer/character.cljs +471 -0
  27. package/src/polymer/conversation/agency.cljs +83 -0
  28. package/src/polymer/conversation/domain.cljs +35 -0
  29. package/src/polymer/conversation/effector.cljs +130 -0
  30. package/src/polymer/conversation/planner.cljs +178 -0
  31. package/src/polymer/conversation/scheduler.cljs +202 -0
  32. package/src/polymer/conversation/service.cljs +329 -0
  33. package/src/polymer/conversation/state.cljs +186 -0
  34. package/src/polymer/core.cljs +172 -0
  35. package/src/polymer/eye_head/agency.cljs +75 -0
  36. package/src/polymer/eye_head/domain.cljs +133 -0
  37. package/src/polymer/eye_head/goap.cljs +106 -0
  38. package/src/polymer/eye_head/scheduler.cljs +176 -0
  39. package/src/polymer/eye_head/service.cljs +259 -0
  40. package/src/polymer/eye_head/snippet.cljs +92 -0
  41. package/src/polymer/eye_head/state.cljs +167 -0
  42. package/src/polymer/gaze/agency.cljs +85 -0
  43. package/src/polymer/gaze/domain.cljs +211 -0
  44. package/src/polymer/gaze/goap.cljs +208 -0
  45. package/src/polymer/gaze/scheduler.cljs +184 -0
  46. package/src/polymer/gaze/state.cljs +202 -0
  47. package/src/polymer/gesture/agency.cljs +185 -0
  48. package/src/polymer/gesture/goap.cljs +461 -0
  49. package/src/polymer/gesture/scheduler.cljs +122 -0
  50. package/src/polymer/gesture/snippet.cljs +198 -0
  51. package/src/polymer/gesture/state.cljs +273 -0
  52. package/src/polymer/lipsync/agency.cljs +593 -0
  53. package/src/polymer/lipsync/articulation/jaw.cljs +258 -0
  54. package/src/polymer/lipsync/articulation/snippet.cljs +605 -0
  55. package/src/polymer/lipsync/articulation/tongue.cljs +418 -0
  56. package/src/polymer/lipsync/articulation/visemes.cljs +429 -0
  57. package/src/polymer/lipsync/goap.cljs +132 -0
  58. package/src/polymer/lipsync/pronunciation/effector.cljs +51 -0
  59. package/src/polymer/lipsync/scheduler.cljs +302 -0
  60. package/src/polymer/lipsync/state.cljs +202 -0
  61. package/src/polymer/nlp/phonemes.cljs +164 -0
  62. package/src/polymer/prosodic/agency.cljs +196 -0
  63. package/src/polymer/prosodic/goap.cljs +122 -0
  64. package/src/polymer/prosodic/scheduler.cljs +99 -0
  65. package/src/polymer/prosodic/snippet.cljs +106 -0
  66. package/src/polymer/prosodic/state.cljs +139 -0
  67. package/src/polymer/runtime/embody.cljs +117 -0
  68. package/src/polymer/runtime/embody_wasm.cljs +177 -0
  69. package/src/polymer/runtime/rust_embody_host.cljs +645 -0
  70. package/src/polymer/society.cljs +239 -0
  71. package/src/polymer/stream.cljs +130 -0
  72. package/src/polymer/transcription/agency.cljs +69 -0
  73. package/src/polymer/transcription/domain.cljs +68 -0
  74. package/src/polymer/transcription/effector.cljs +611 -0
  75. package/src/polymer/transcription/planner.cljs +203 -0
  76. package/src/polymer/transcription/scheduler.cljs +225 -0
  77. package/src/polymer/transcription/service.cljs +209 -0
  78. package/src/polymer/transcription/state.cljs +239 -0
  79. package/src/polymer/tts/agency.cljs +503 -0
  80. package/src/polymer/tts/azure.cljs +290 -0
  81. package/src/polymer/tts/planner.cljs +178 -0
  82. package/src/polymer/tts/runtime.cljs +624 -0
  83. package/src/polymer/tts/scheduler.cljs +162 -0
  84. package/src/polymer/tts/state.cljs +191 -0
  85. package/src/polymer/worker.cljs +70 -0
@@ -0,0 +1,593 @@
1
+ (ns polymer.lipsync.agency
2
+ (:require [polymer.lipsync.goap :as goap]
3
+ [polymer.stream :as stream]
4
+ [polymer.tts.azure :as azure]
5
+ [polymer.lipsync.scheduler :as scheduler]
6
+ [polymer.lipsync.pronunciation.effector :as pronunciation-effector]
7
+ [polymer.lipsync.articulation.snippet :as snippet]
8
+ [polymer.lipsync.state :as state]
9
+ [polymer.lipsync.articulation.visemes :as visemes]))
10
+
11
+ ;; LipSync accepts provider/text timing data and produces animation intent.
12
+ ;; It does not own audio playback, provider credentials, HTTP, LiveKit, storage,
13
+ ;; or DOM APIs. Those side-effect agencies feed plain messages into this agency.
14
+ ;; Inside Polymer, the agency system routes LipSync animation requests to the
15
+ ;; Animation agency.
16
+
17
+ (defn debug-flag-enabled? [name]
18
+ (try
19
+ (let [search (when (exists? js/window)
20
+ (.. js/window -location -search))
21
+ params (when search (js/URLSearchParams. search))
22
+ value (when params (.get params name))]
23
+ (or (= value "1") (= value "true")))
24
+ (catch :default _ false)))
25
+
26
+ (defn lipsync-debug-enabled? []
27
+ (or (debug-flag-enabled? "polymerVocalDebug")
28
+ (debug-flag-enabled? "polymerLipSyncDebug")))
29
+
30
+ (defn debug-log! [label payload]
31
+ (when (lipsync-debug-enabled?)
32
+ (.info js/console (str label " " (.stringify js/JSON (clj->js payload))))))
33
+
34
+ (defn viseme-event-summary [events]
35
+ (into []
36
+ (comp
37
+ (take 120)
38
+ (map (fn [event]
39
+ {:visemeId (:visemeId event)
40
+ :phoneme (:phoneme event)
41
+ :phonemeClass (:phonemeClass event)
42
+ :offsetMs (:offsetMs event)
43
+ :durationMs (:durationMs event)
44
+ :intensity (:intensity event)
45
+ :jawActivation (:jawActivation event)})))
46
+ events))
47
+
48
+ (defn channel-summary [snippet-data]
49
+ (into []
50
+ (map (fn [channel]
51
+ (let [keyframes (:keyframes channel)
52
+ peak (transduce (map :intensity) max 0 keyframes)]
53
+ {:target (:target channel)
54
+ :frames (count keyframes)
55
+ :firstSec (:time (first keyframes))
56
+ :lastSec (:time (last keyframes))
57
+ :peak peak})))
58
+ (:channels snippet-data)))
59
+
60
+ (defn js-command [type value]
61
+ #js {:type type :value value})
62
+
63
+ (defn plan-failure-message
64
+ "Turn the failed LipSync plan step into a readable domain error."
65
+ [plan]
66
+ (let [step (first (:steps plan))]
67
+ (case (:reason step)
68
+ "unsupported-command" (str "lipSync plan failed: unsupported command " (:commandType step))
69
+ "missing-text" "lipSync plan failed: startText requires text"
70
+ "missing-timeline-visemes" "lipSync plan failed: startTimeline requires visemes"
71
+ "missing-provider-visemes" "lipSync plan failed: processAzureVisemes requires provider visemes"
72
+ "missing-word" "lipSync plan failed: wordBoundary requires a word"
73
+ (str "lipSync plan failed: " (:reason step)))))
74
+
75
+ (defn visual-lead-ms [input config]
76
+ (state/clamp 0 250 (state/number-or (:visualLeadMs input) (:visualLeadMs config))))
77
+
78
+ (defn apply-visual-lead [events lead-ms]
79
+ (let [lead (max 0 (state/number-or lead-ms 0))]
80
+ (mapv (fn [event]
81
+ (update event :offsetMs #(max 0 (- % lead))))
82
+ events)))
83
+
84
+ (defn normalize-timeline [timeline config]
85
+ (let [input (or timeline {})
86
+ source (or (:source input) "external")
87
+ text (:text input)
88
+ lead-ms (visual-lead-ms input config)
89
+ viseme-events (apply-visual-lead (snippet/normalize-events (:visemes input)) lead-ms)]
90
+ {:name (:name input)
91
+ :text text
92
+ :source source
93
+ :visemes viseme-events
94
+ :visualLeadMs lead-ms
95
+ :wordTimings (state/normalize-word-timings (:wordTimings input))
96
+ :durationSec (state/number-or (:durationSec input)
97
+ (if (empty? viseme-events)
98
+ 0
99
+ (apply max (map #(/ (+ (:offsetMs %) (:durationMs %)) 1000) viseme-events))))
100
+ :config config}))
101
+
102
+ (defn normalize-observed-elapsed-sec [value expected max-time]
103
+ ;; Web Speech boundary clocks are not consistent enough to trust blindly
104
+ ;; across browsers. The spec says seconds, but some engines/adapters have
105
+ ;; reported millisecond-looking values. Treat a value as milliseconds only
106
+ ;; when it is clearly farther from the expected word start than value/1000.
107
+ ;; Without this guard, the first nonzero boundary can seek a whole utterance
108
+ ;; clip to its end, which looks like one lip movement at the beginning.
109
+ (when (state/finite-number? value)
110
+ (let [raw (max 0 value)
111
+ millis (/ raw 1000)
112
+ expected-start (when expected (:startSec expected))
113
+ raw-error (when (state/finite-number? expected-start)
114
+ (js/Math.abs (- raw expected-start)))
115
+ millis-error (when (state/finite-number? expected-start)
116
+ (js/Math.abs (- millis expected-start)))
117
+ max-time (max 0 (state/number-or max-time 0))
118
+ within-utterance? (or (zero? max-time)
119
+ (<= millis (+ max-time 2)))
120
+ expected-says-ms? (and (> raw 10)
121
+ raw-error
122
+ millis-error
123
+ (< millis-error raw-error)
124
+ within-utterance?)
125
+ max-time-says-ms? (and (> raw (+ max-time 2))
126
+ (> raw 50)
127
+ within-utterance?)]
128
+ (if (or expected-says-ms? max-time-says-ms?)
129
+ millis
130
+ raw))))
131
+
132
+ (defn choose-boundary-elapsed-sec [payload expected max-time]
133
+ ;; Prefer Web Speech's elapsedTime after unit normalization. Some browsers
134
+ ;; report 0 for later word boundaries, so LoomLarge also sends a monotonic
135
+ ;; host clock. Use that host clock only when the provider clock is clearly not
136
+ ;; advancing; otherwise provider timing remains the authority.
137
+ (let [observed (normalize-observed-elapsed-sec (:observedElapsedSec payload)
138
+ expected
139
+ max-time)
140
+ host (when (state/finite-number? (:hostElapsedSec payload))
141
+ (max 0 (:hostElapsedSec payload)))
142
+ expected-start (when expected (:startSec expected))]
143
+ (cond
144
+ (and host
145
+ observed
146
+ (state/finite-number? expected-start)
147
+ (> expected-start 0.08)
148
+ (< observed 0.001)
149
+ (> host 0.001))
150
+ host
151
+
152
+ observed
153
+ observed
154
+
155
+ host
156
+ host
157
+
158
+ :else
159
+ nil)))
160
+
161
+ (defn payload-duration-sec [payload]
162
+ (or (when (state/finite-number? (:durationSec payload))
163
+ (max 0 (:durationSec payload)))
164
+ (when (state/finite-number? (:totalDurationMs payload))
165
+ (/ (max 0 (:totalDurationMs payload)) 1000))))
166
+
167
+ (defn timing-duration-sec [word-timings]
168
+ (when (seq word-timings)
169
+ (apply max 0 (map :endSec (state/normalize-word-timings word-timings)))))
170
+
171
+ (defn prepare-text-timeline [dictionary text speech-rate payload]
172
+ (let [source (or (:source payload) "text")
173
+ events (visemes/text->visemes dictionary text speech-rate)
174
+ generated-word-timings (visemes/text->word-timings
175
+ dictionary text speech-rate)
176
+ supplied-word-timings (state/normalize-word-timings (:wordTimings payload))
177
+ base-duration-ms (visemes/timeline-duration-ms events)
178
+ explicit-duration-sec (or (payload-duration-sec payload)
179
+ (timing-duration-sec supplied-word-timings))
180
+ web-speech-duration-sec (when (= "webSpeech" source)
181
+ (/ (visemes/web-speech-duration-ms text speech-rate base-duration-ms)
182
+ 1000))
183
+ target-duration-sec (or explicit-duration-sec
184
+ web-speech-duration-sec
185
+ (/ base-duration-ms 1000))
186
+ base-duration-sec (/ base-duration-ms 1000)
187
+ scale (if (and (pos? base-duration-sec)
188
+ (state/finite-number? target-duration-sec)
189
+ (> target-duration-sec 0))
190
+ (/ target-duration-sec base-duration-sec)
191
+ 1)
192
+ should-scale? (> (js/Math.abs (- scale 1)) 0.02)
193
+ word-timings (if (seq supplied-word-timings)
194
+ supplied-word-timings
195
+ (if should-scale?
196
+ (visemes/scale-word-timings generated-word-timings scale)
197
+ generated-word-timings))]
198
+ {:name (:name payload)
199
+ :text text
200
+ :source source
201
+ :visemes (if should-scale?
202
+ (visemes/scale-events events scale)
203
+ events)
204
+ :wordTimings word-timings
205
+ :durationSec (if (state/finite-number? target-duration-sec)
206
+ target-duration-sec
207
+ base-duration-sec)}))
208
+
209
+ (defn timeline-name [timeline]
210
+ (or (:name timeline)
211
+ (str "lipSync_timeline_" (.now js/Date))))
212
+
213
+ (defn create-lipsync-agency
214
+ ([config]
215
+ (create-lipsync-agency config nil))
216
+ ([config runtime]
217
+ (let [input-stream (stream/create-stream)
218
+ event-stream (stream/create-stream)
219
+ state-atom (atom (state/config->state config))
220
+ disposed? (atom false)
221
+ emit-input (:emit input-stream)
222
+ emit-event (:emit event-stream)
223
+ pronunciation-provider (pronunciation-effector/create-effector runtime)
224
+ agency-scheduler (atom nil)]
225
+ ;; Stream contract for this agency:
226
+ ;; - input records accepted commands for tests/workers/debugging.
227
+ ;; - events carry domain facts and cross-agency requests.
228
+ (letfn [(audio-clock-sec [payload]
229
+ (max 0
230
+ (state/number-or (or (:audioTimeSec payload)
231
+ (:currentTimeSec payload)
232
+ (:offsetSec payload))
233
+ 0)))
234
+
235
+ (record-stop-state! [reason]
236
+ (let [stopped-at (state/now-ms)]
237
+ (swap! state-atom state/record-stop stopped-at reason)
238
+ (emit-event {:type "lipSyncTimelineStopped"
239
+ :agency "lipSync"
240
+ :reason reason
241
+ :stoppedAt stopped-at})))
242
+
243
+ (stop-local! [reason remove?]
244
+ (when-let [s @agency-scheduler]
245
+ ((:stop-timeline s) reason remove?))
246
+ (record-stop-state! reason))
247
+
248
+ (finish-local! []
249
+ (when (:speaking @state-atom)
250
+ (stop-local! "completed" false)))
251
+
252
+ (start-timeline! [timeline]
253
+ (let [config (:config @state-atom)
254
+ normalized (normalize-timeline timeline config)
255
+ pending-audio-started-at (:audioStartedAt @state-atom)
256
+ pending-audio-time-sec (:audioTimeSec @state-atom)]
257
+ (if (empty? (:visemes normalized))
258
+ (emit-event {:type "error"
259
+ :agency "lipSync"
260
+ :message "lipSync timeline requires at least one viseme event"})
261
+ (do
262
+ ;; One active utterance at a time keeps lip sync from
263
+ ;; accumulating stale viseme snippets. State records the
264
+ ;; replacement fact; the scheduler owns the animation
265
+ ;; removal/start ordering for the old and new snippets.
266
+ (when (:speaking @state-atom)
267
+ (record-stop-state! "replaced"))
268
+ (let [name (timeline-name normalized)
269
+ built (if (:text normalized)
270
+ (snippet/build-text-snippet (:text normalized)
271
+ (:visemes normalized)
272
+ config)
273
+ (snippet/build-lipsync-snippet (:visemes normalized)
274
+ config
275
+ name))
276
+ ;; Keep one snippet as the atomic animation unit for
277
+ ;; an utterance. Individual visemes, jaw, and tongue
278
+ ;; gestures are channels inside the snippet rather
279
+ ;; than separately scheduled clips.
280
+ snippet-data (cond-> (assoc built :name (or (:name normalized) (:name built)))
281
+ (state/finite-number? (:durationSec normalized))
282
+ (assoc :maxTime (max (:maxTime built) (:durationSec normalized))))
283
+ started-at (state/now-ms)]
284
+ (debug-log! "[Polymer LipSync CLJS] normalized timeline"
285
+ {:name (:name snippet-data)
286
+ :source (:source normalized)
287
+ :visemeCount (count (:visemes normalized))
288
+ :uniqueVisemeIds (->> (:visemes normalized)
289
+ (map :visemeId)
290
+ set
291
+ sort
292
+ vec)
293
+ :visualLeadMs (:visualLeadMs normalized)
294
+ :maxTime (:maxTime snippet-data)
295
+ :visemes (viseme-event-summary (:visemes normalized))})
296
+ (debug-log! "[Polymer LipSync CLJS] snippet channels"
297
+ {:name (:name snippet-data)
298
+ :channelCount (count (:channels snippet-data))
299
+ :channels (channel-summary snippet-data)})
300
+ (swap! state-atom state/record-start normalized (:name snippet-data) started-at (:maxTime snippet-data))
301
+ (emit-event {:type "lipSyncTimelineStarted"
302
+ :agency "lipSync"
303
+ :name (:name snippet-data)
304
+ :source (:source normalized)
305
+ :text (:text normalized)
306
+ :visemeCount (count (:visemes normalized))
307
+ :maxTime (:maxTime snippet-data)
308
+ :startedAt started-at})
309
+ (when-let [s @agency-scheduler]
310
+ ((:start-timeline s) snippet-data {:autoPlay true}))
311
+ (when (and (= "webSpeech" (:source normalized))
312
+ pending-audio-started-at)
313
+ (let [elapsed-since-start
314
+ (/ (max 0 (- started-at
315
+ pending-audio-started-at))
316
+ 1000)
317
+ target-sec
318
+ (min (:maxTime snippet-data)
319
+ (+ (state/number-or
320
+ pending-audio-time-sec 0)
321
+ elapsed-since-start))]
322
+ (swap! state-atom
323
+ state/record-audio-started
324
+ target-sec
325
+ pending-audio-started-at)
326
+ (when-let [s @agency-scheduler]
327
+ ((:seek s)
328
+ (:name snippet-data)
329
+ target-sec
330
+ "late-pronunciation-ready"))))
331
+ snippet-data)))))
332
+
333
+ (start-text! [payload]
334
+ (let [text (:text payload)
335
+ agency-scheduler @agency-scheduler]
336
+ (if (and (string? text) (pos? (count text)))
337
+ ((:prepare-text agency-scheduler) payload)
338
+ (emit-event {:type "error"
339
+ :agency "lipSync"
340
+ :message "lipSync startText command requires text"}))))
341
+
342
+ (process-azure! [payload]
343
+ ;; Compatibility command for Azure/LiveKit provider facts. The
344
+ ;; Azure-specific viseme-id mapping lives in TTS; LipSync only
345
+ ;; normalizes word timing metadata and then uses the shared
346
+ ;; articulated timeline path.
347
+ (let [raw-options (or (:options payload) {})
348
+ word-timings (state/normalize-word-timings
349
+ (or (:wordTimings payload)
350
+ (:wordTimings raw-options)))
351
+ options (assoc (merge {:visualLeadMs (get-in @state-atom [:config :visualLeadMs])}
352
+ raw-options)
353
+ :wordTimings word-timings)
354
+ timeline (azure/azure-visemes->timeline (:visemes payload)
355
+ (:totalDurationMs payload)
356
+ options)]
357
+ (start-timeline! {:name (:name payload)
358
+ :text (:text payload)
359
+ :source (or (:source payload) "azure")
360
+ :visemes timeline
361
+ :visualLeadMs (:visualLeadMs options)
362
+ :wordTimings (:wordTimings options)
363
+ :durationSec (when (state/finite-number? (:totalDurationMs payload))
364
+ (/ (:totalDurationMs payload) 1000))})))
365
+
366
+ (handle-word-boundary! [payload]
367
+ (let [word (:word payload)
368
+ word-index (int (state/number-or (:wordIndex payload) (:wordIndex @state-atom)))
369
+ observed-at (state/now-ms)]
370
+ (swap! state-atom state/record-word-boundary word word-index observed-at)
371
+ (emit-event {:type "lipSyncWordBoundary"
372
+ :agency "lipSync"
373
+ :word word
374
+ :wordIndex word-index
375
+ :observedAt observed-at})
376
+ (let [current @state-atom
377
+ expected (get (:wordTimings current) word-index)
378
+ observed-elapsed-sec (choose-boundary-elapsed-sec payload
379
+ expected
380
+ (:maxTime current))
381
+ elapsed-sec (state/number-or observed-elapsed-sec
382
+ (if (:startTime current)
383
+ (/ (- observed-at (:startTime current)) 1000)
384
+ 0))
385
+ drift-sec (when expected (- elapsed-sec (:startSec expected)))
386
+ threshold (get-in current [:config :wordDriftThresholdSec])
387
+ name (:snippetName current)]
388
+ (when (and expected name drift-sec (> (js/Math.abs drift-sec) threshold))
389
+ (let [target-sec (min (:maxTime current) (max 0 elapsed-sec))
390
+ corrected-at (state/now-ms)]
391
+ (swap! state-atom state/record-sync-correction name target-sec corrected-at)
392
+ (emit-event {:type "lipSyncSyncDrift"
393
+ :agency "lipSync"
394
+ :name name
395
+ :word word
396
+ :wordIndex word-index
397
+ :expectedSec (:startSec expected)
398
+ :observedSec elapsed-sec
399
+ :driftSec drift-sec
400
+ :targetSec target-sec
401
+ :correctedAt corrected-at})
402
+ (when-let [s @agency-scheduler]
403
+ ((:seek s) name target-sec "word-boundary-drift")))))))
404
+
405
+ (handle-audio-started! [payload]
406
+ ;; The host owns audio playback. This command tells LipSync when
407
+ ;; that side effect actually started so Animation can align the
408
+ ;; scheduled snippet to the audio clock without LoomLarge calling
409
+ ;; Animation directly.
410
+ (let [current @state-atom
411
+ name (or (:name payload) (:snippetName current))
412
+ raw-audio-time-sec (audio-clock-sec payload)
413
+ audio-time-sec (if (:speaking current)
414
+ (min (:maxTime current)
415
+ raw-audio-time-sec)
416
+ raw-audio-time-sec)
417
+ observed-at (state/now-ms)]
418
+ (swap! state-atom state/record-audio-started audio-time-sec observed-at)
419
+ (emit-event {:type "lipSyncAudioStarted"
420
+ :agency "lipSync"
421
+ :name name
422
+ :audioTimeSec audio-time-sec
423
+ :observedAt observed-at})
424
+ (when (and (:speaking current)
425
+ (:snippetName current)
426
+ name)
427
+ (when-let [s @agency-scheduler]
428
+ ((:seek s) name audio-time-sec "audio-started")))))
429
+
430
+ (handle-audio-time! [payload]
431
+ ;; Optional low-frequency host clock correction. This is not a
432
+ ;; frame loop; callers should send it only when a material drift
433
+ ;; correction is needed.
434
+ (let [current @state-atom
435
+ name (or (:name payload) (:snippetName current))
436
+ raw-audio-time-sec (audio-clock-sec payload)
437
+ audio-time-sec (if (:speaking current)
438
+ (min (:maxTime current)
439
+ raw-audio-time-sec)
440
+ raw-audio-time-sec)
441
+ observed-at (state/now-ms)]
442
+ (swap! state-atom state/record-audio-time audio-time-sec observed-at)
443
+ (emit-event {:type "lipSyncAudioTime"
444
+ :agency "lipSync"
445
+ :name name
446
+ :audioTimeSec audio-time-sec
447
+ :observedAt observed-at})
448
+ (when name
449
+ (when-let [s @agency-scheduler]
450
+ ((:seek s) name audio-time-sec "audio-time")))))
451
+
452
+ (update-word-timings! [payload]
453
+ (let [updated-at (state/now-ms)
454
+ timings (:wordTimings payload)]
455
+ (swap! state-atom state/record-word-timings timings updated-at)
456
+ (emit-event {:type "lipSyncWordTimingsUpdated"
457
+ :agency "lipSync"
458
+ :count (count (state/normalize-word-timings timings))
459
+ :updatedAt updated-at})))
460
+
461
+ (run-action! [action payload]
462
+ ;; `goap/plan-command` decides which agency operation is allowed
463
+ ;; for a command. Dispatch only echoes the input and reports the
464
+ ;; plan; accepted actions run here so the command router does not
465
+ ;; grow another copy of LipSync's state/scheduler policy.
466
+ (case (:op action)
467
+ "configure"
468
+ (do
469
+ (swap! state-atom state/configure (:config payload))
470
+ (emit-event {:type "lipSyncConfigChanged"
471
+ :agency "lipSync"
472
+ :state @state-atom}))
473
+
474
+ "start-text"
475
+ (start-text! payload)
476
+
477
+ "start-timeline"
478
+ (start-timeline! (:timeline payload))
479
+
480
+ "process-provider-visemes"
481
+ (process-azure! payload)
482
+
483
+ "record-word-boundary"
484
+ (handle-word-boundary! payload)
485
+
486
+ "align-audio-start"
487
+ (handle-audio-started! payload)
488
+
489
+ "align-audio-time"
490
+ (handle-audio-time! payload)
491
+
492
+ "update-word-timings"
493
+ (update-word-timings! payload)
494
+
495
+ "stop"
496
+ (stop-local! (:reason action) (:remove? action))
497
+
498
+ "reset-state"
499
+ (do
500
+ (reset! state-atom (state/config->state nil))
501
+ (emit-event {:type "lipSyncConfigChanged"
502
+ :agency "lipSync"
503
+ :state @state-atom}))
504
+
505
+ (emit-event {:type "error"
506
+ :agency "lipSync"
507
+ :message (str "Unknown LipSync planner action: " (:op action))})))
508
+
509
+ (dispatch! [command]
510
+ (when-not @disposed?
511
+ (let [payload (js->clj command :keywordize-keys true)
512
+ plan (goap/plan-command payload {:speaking (:speaking @state-atom)})]
513
+ ;; Every command produces an input echo and a plan event before
514
+ ;; any mutation. That makes failed/ignored commands observable
515
+ ;; through the event stream instead of mutable state polling.
516
+ (emit-input {:type "command"
517
+ :agency "lipSync"
518
+ :command payload})
519
+ (emit-event {:type "lipSyncPlanCreated"
520
+ :agency "lipSync"
521
+ :plan plan})
522
+ (if-not (:ok plan)
523
+ (emit-event {:type "error"
524
+ :agency "lipSync"
525
+ :message (plan-failure-message plan)})
526
+ (doseq [action (:actions plan)]
527
+ (run-action! action payload))))))]
528
+ (reset! agency-scheduler
529
+ (scheduler/create-scheduler
530
+ {:emit-event emit-event
531
+ :on-finished finish-local!
532
+ :pronunciation-effector pronunciation-provider
533
+ :on-text-ready
534
+ (fn [payload dictionary]
535
+ (when-not @disposed?
536
+ (let [text (:text payload)
537
+ speech-rate (get-in @state-atom
538
+ [:config :speechRate])]
539
+ (start-timeline!
540
+ (prepare-text-timeline
541
+ dictionary text speech-rate payload)))))}))
542
+ ((:preload-pronunciations @agency-scheduler))
543
+ #js {:dispatch dispatch!
544
+ :snapshot (fn [] (state/visible-state @state-atom))
545
+ :schedulerQueue (fn []
546
+ (clj->js (if-let [s @agency-scheduler]
547
+ ((:queue s))
548
+ [])))
549
+ :input (stream/writable-port input-stream dispatch!)
550
+ :events (stream/readable-port event-stream)
551
+ ;; Empty compatibility stream. Cross-agency animation requests are
552
+ ;; domain events so the Polymer agency system can route them to
553
+ ;; Animation without host UI code becoming the interpreter.
554
+ :subscribeInput (fn [listener] ((:subscribe input-stream) listener))
555
+ :subscribeEvents (fn [listener] ((:subscribe event-stream) listener))
556
+ :subscribe (fn [listener] ((:subscribe event-stream) listener))
557
+ :subscribeStatus (fn [listener] ((:subscribe event-stream) listener))
558
+ :configure (fn [next-config] (dispatch! #js {:type "configure" :config next-config}))
559
+ :startText (fn [text] (dispatch! #js {:type "startText" :text text}))
560
+ :startTimeline (fn [timeline] (dispatch! #js {:type "startTimeline" :timeline timeline}))
561
+ :processAzureVisemes (fn
562
+ ([visemes] (dispatch! #js {:type "processAzureVisemes" :visemes visemes}))
563
+ ([visemes total-duration-ms]
564
+ (dispatch! #js {:type "processAzureVisemes"
565
+ :visemes visemes
566
+ :totalDurationMs total-duration-ms})))
567
+ :wordBoundary (fn
568
+ ([word] (dispatch! #js {:type "wordBoundary" :word word}))
569
+ ([word word-index observed-elapsed-sec]
570
+ (dispatch! #js {:type "wordBoundary"
571
+ :word word
572
+ :wordIndex word-index
573
+ :observedElapsedSec observed-elapsed-sec})))
574
+ :audioStarted (fn
575
+ ([] (dispatch! #js {:type "audioStarted"}))
576
+ ([audio-time-sec]
577
+ (dispatch! #js {:type "audioStarted"
578
+ :audioTimeSec audio-time-sec})))
579
+ :audioTime (fn [audio-time-sec]
580
+ (dispatch! #js {:type "audioTime"
581
+ :audioTimeSec audio-time-sec}))
582
+ :updateWordTimings (fn [word-timings]
583
+ (dispatch! #js {:type "updateWordTimings"
584
+ :wordTimings word-timings}))
585
+ :stop (fn [] (dispatch! #js {:type "stop"}))
586
+ :reset (fn [] (dispatch! #js {:type "reset"}))
587
+ :dispose (fn []
588
+ (when-not @disposed?
589
+ (reset! disposed? true)
590
+ (when-let [s @agency-scheduler]
591
+ ((:dispose s)))
592
+ ((:dispose input-stream))
593
+ ((:dispose event-stream))))}))))