@lovelace_lol/polymer 0.0.6 → 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +434 -12
  2. package/dist/index.d.ts +1945 -244
  3. package/dist/index.js +1686 -1017
  4. package/dist/vision/face-worker.js +296 -0
  5. package/dist/vision/face_landmarker.task +0 -0
  6. package/dist/vision/manifest.edn +1 -0
  7. package/dist/vision/vision_bundle.js +2 -0
  8. package/dist/vision/wasm/vision_wasm_internal.js +8841 -0
  9. package/dist/vision/wasm/vision_wasm_internal.wasm +0 -0
  10. package/dist/vision/wasm/vision_wasm_module_internal.js +8837 -0
  11. package/dist/vision/wasm/vision_wasm_module_internal.wasm +0 -0
  12. package/dist/vision/wasm/vision_wasm_nosimd_internal.js +8832 -0
  13. package/dist/vision/wasm/vision_wasm_nosimd_internal.wasm +0 -0
  14. package/dist/worker/agency-worker.js +1276 -962
  15. package/dist/worker/manifest.edn +1 -1
  16. package/package.json +33 -20
  17. package/dist/test.cjs +0 -1519
  18. package/shadow-cljs.edn +0 -173
  19. package/src/polymer/animation/agency.cljs +0 -84
  20. package/src/polymer/animation/domain.cljs +0 -110
  21. package/src/polymer/animation/planner.cljs +0 -88
  22. package/src/polymer/animation/runtime.cljs +0 -137
  23. package/src/polymer/animation/scheduler.cljs +0 -178
  24. package/src/polymer/animation/service.cljs +0 -981
  25. package/src/polymer/animation/state.cljs +0 -112
  26. package/src/polymer/blink/agency.cljs +0 -131
  27. package/src/polymer/blink/planner.cljs +0 -115
  28. package/src/polymer/blink/scheduler.cljs +0 -90
  29. package/src/polymer/blink/snippet.cljs +0 -50
  30. package/src/polymer/blink/state.cljs +0 -102
  31. package/src/polymer/camera_context/agency.cljs +0 -132
  32. package/src/polymer/camera_context/domain.cljs +0 -186
  33. package/src/polymer/camera_context/planner.cljs +0 -50
  34. package/src/polymer/camera_context/scheduler.cljs +0 -113
  35. package/src/polymer/camera_context/state.cljs +0 -121
  36. package/src/polymer/character.cljs +0 -403
  37. package/src/polymer/conversation/agency.cljs +0 -83
  38. package/src/polymer/conversation/domain.cljs +0 -35
  39. package/src/polymer/conversation/effector.cljs +0 -130
  40. package/src/polymer/conversation/planner.cljs +0 -178
  41. package/src/polymer/conversation/scheduler.cljs +0 -202
  42. package/src/polymer/conversation/service.cljs +0 -329
  43. package/src/polymer/conversation/state.cljs +0 -186
  44. package/src/polymer/core.cljs +0 -172
  45. package/src/polymer/eye_head/agency.cljs +0 -75
  46. package/src/polymer/eye_head/domain.cljs +0 -133
  47. package/src/polymer/eye_head/goap.cljs +0 -106
  48. package/src/polymer/eye_head/scheduler.cljs +0 -176
  49. package/src/polymer/eye_head/service.cljs +0 -259
  50. package/src/polymer/eye_head/snippet.cljs +0 -92
  51. package/src/polymer/eye_head/state.cljs +0 -167
  52. package/src/polymer/gaze/agency.cljs +0 -85
  53. package/src/polymer/gaze/domain.cljs +0 -211
  54. package/src/polymer/gaze/goap.cljs +0 -208
  55. package/src/polymer/gaze/scheduler.cljs +0 -184
  56. package/src/polymer/gaze/state.cljs +0 -202
  57. package/src/polymer/gesture/agency.cljs +0 -185
  58. package/src/polymer/gesture/goap.cljs +0 -461
  59. package/src/polymer/gesture/scheduler.cljs +0 -122
  60. package/src/polymer/gesture/snippet.cljs +0 -198
  61. package/src/polymer/gesture/state.cljs +0 -273
  62. package/src/polymer/lipsync/agency.cljs +0 -593
  63. package/src/polymer/lipsync/articulation/jaw.cljs +0 -258
  64. package/src/polymer/lipsync/articulation/snippet.cljs +0 -605
  65. package/src/polymer/lipsync/articulation/tongue.cljs +0 -418
  66. package/src/polymer/lipsync/articulation/visemes.cljs +0 -429
  67. package/src/polymer/lipsync/goap.cljs +0 -132
  68. package/src/polymer/lipsync/pronunciation/effector.cljs +0 -51
  69. package/src/polymer/lipsync/scheduler.cljs +0 -302
  70. package/src/polymer/lipsync/state.cljs +0 -202
  71. package/src/polymer/nlp/phonemes.cljs +0 -164
  72. package/src/polymer/prosodic/agency.cljs +0 -196
  73. package/src/polymer/prosodic/goap.cljs +0 -122
  74. package/src/polymer/prosodic/scheduler.cljs +0 -99
  75. package/src/polymer/prosodic/snippet.cljs +0 -106
  76. package/src/polymer/prosodic/state.cljs +0 -139
  77. package/src/polymer/runtime/embody.cljs +0 -117
  78. package/src/polymer/runtime/embody_wasm.cljs +0 -177
  79. package/src/polymer/runtime/rust_embody_host.cljs +0 -647
  80. package/src/polymer/society.cljs +0 -253
  81. package/src/polymer/stream.cljs +0 -130
  82. package/src/polymer/transcription/agency.cljs +0 -69
  83. package/src/polymer/transcription/domain.cljs +0 -68
  84. package/src/polymer/transcription/effector.cljs +0 -611
  85. package/src/polymer/transcription/planner.cljs +0 -203
  86. package/src/polymer/transcription/scheduler.cljs +0 -225
  87. package/src/polymer/transcription/service.cljs +0 -209
  88. package/src/polymer/transcription/state.cljs +0 -239
  89. package/src/polymer/tts/agency.cljs +0 -503
  90. package/src/polymer/tts/azure.cljs +0 -290
  91. package/src/polymer/tts/planner.cljs +0 -178
  92. package/src/polymer/tts/runtime.cljs +0 -624
  93. package/src/polymer/tts/scheduler.cljs +0 -162
  94. package/src/polymer/tts/state.cljs +0 -191
  95. package/src/polymer/worker.cljs +0 -81
@@ -1,593 +0,0 @@
1
- (ns polymer.lipsync.agency
2
- (:require [polymer.lipsync.goap :as goap]
3
- [polymer.stream :as stream]
4
- [polymer.tts.azure :as azure]
5
- [polymer.lipsync.scheduler :as scheduler]
6
- [polymer.lipsync.pronunciation.effector :as pronunciation-effector]
7
- [polymer.lipsync.articulation.snippet :as snippet]
8
- [polymer.lipsync.state :as state]
9
- [polymer.lipsync.articulation.visemes :as visemes]))
10
-
11
- ;; LipSync accepts provider/text timing data and produces animation intent.
12
- ;; It does not own audio playback, provider credentials, HTTP, LiveKit, storage,
13
- ;; or DOM APIs. Those side-effect agencies feed plain messages into this agency.
14
- ;; Inside Polymer, the agency system routes LipSync animation requests to the
15
- ;; Animation agency.
16
-
17
- (defn debug-flag-enabled? [name]
18
- (try
19
- (let [search (when (exists? js/window)
20
- (.. js/window -location -search))
21
- params (when search (js/URLSearchParams. search))
22
- value (when params (.get params name))]
23
- (or (= value "1") (= value "true")))
24
- (catch :default _ false)))
25
-
26
- (defn lipsync-debug-enabled? []
27
- (or (debug-flag-enabled? "polymerVocalDebug")
28
- (debug-flag-enabled? "polymerLipSyncDebug")))
29
-
30
- (defn debug-log! [label payload]
31
- (when (lipsync-debug-enabled?)
32
- (.info js/console (str label " " (.stringify js/JSON (clj->js payload))))))
33
-
34
- (defn viseme-event-summary [events]
35
- (into []
36
- (comp
37
- (take 120)
38
- (map (fn [event]
39
- {:visemeId (:visemeId event)
40
- :phoneme (:phoneme event)
41
- :phonemeClass (:phonemeClass event)
42
- :offsetMs (:offsetMs event)
43
- :durationMs (:durationMs event)
44
- :intensity (:intensity event)
45
- :jawActivation (:jawActivation event)})))
46
- events))
47
-
48
- (defn channel-summary [snippet-data]
49
- (into []
50
- (map (fn [channel]
51
- (let [keyframes (:keyframes channel)
52
- peak (transduce (map :intensity) max 0 keyframes)]
53
- {:target (:target channel)
54
- :frames (count keyframes)
55
- :firstSec (:time (first keyframes))
56
- :lastSec (:time (last keyframes))
57
- :peak peak})))
58
- (:channels snippet-data)))
59
-
60
- (defn js-command [type value]
61
- #js {:type type :value value})
62
-
63
- (defn plan-failure-message
64
- "Turn the failed LipSync plan step into a readable domain error."
65
- [plan]
66
- (let [step (first (:steps plan))]
67
- (case (:reason step)
68
- "unsupported-command" (str "lipSync plan failed: unsupported command " (:commandType step))
69
- "missing-text" "lipSync plan failed: startText requires text"
70
- "missing-timeline-visemes" "lipSync plan failed: startTimeline requires visemes"
71
- "missing-provider-visemes" "lipSync plan failed: processAzureVisemes requires provider visemes"
72
- "missing-word" "lipSync plan failed: wordBoundary requires a word"
73
- (str "lipSync plan failed: " (:reason step)))))
74
-
75
- (defn visual-lead-ms [input config]
76
- (state/clamp 0 250 (state/number-or (:visualLeadMs input) (:visualLeadMs config))))
77
-
78
- (defn apply-visual-lead [events lead-ms]
79
- (let [lead (max 0 (state/number-or lead-ms 0))]
80
- (mapv (fn [event]
81
- (update event :offsetMs #(max 0 (- % lead))))
82
- events)))
83
-
84
- (defn normalize-timeline [timeline config]
85
- (let [input (or timeline {})
86
- source (or (:source input) "external")
87
- text (:text input)
88
- lead-ms (visual-lead-ms input config)
89
- viseme-events (apply-visual-lead (snippet/normalize-events (:visemes input)) lead-ms)]
90
- {:name (:name input)
91
- :text text
92
- :source source
93
- :visemes viseme-events
94
- :visualLeadMs lead-ms
95
- :wordTimings (state/normalize-word-timings (:wordTimings input))
96
- :durationSec (state/number-or (:durationSec input)
97
- (if (empty? viseme-events)
98
- 0
99
- (apply max (map #(/ (+ (:offsetMs %) (:durationMs %)) 1000) viseme-events))))
100
- :config config}))
101
-
102
- (defn normalize-observed-elapsed-sec [value expected max-time]
103
- ;; Web Speech boundary clocks are not consistent enough to trust blindly
104
- ;; across browsers. The spec says seconds, but some engines/adapters have
105
- ;; reported millisecond-looking values. Treat a value as milliseconds only
106
- ;; when it is clearly farther from the expected word start than value/1000.
107
- ;; Without this guard, the first nonzero boundary can seek a whole utterance
108
- ;; clip to its end, which looks like one lip movement at the beginning.
109
- (when (state/finite-number? value)
110
- (let [raw (max 0 value)
111
- millis (/ raw 1000)
112
- expected-start (when expected (:startSec expected))
113
- raw-error (when (state/finite-number? expected-start)
114
- (js/Math.abs (- raw expected-start)))
115
- millis-error (when (state/finite-number? expected-start)
116
- (js/Math.abs (- millis expected-start)))
117
- max-time (max 0 (state/number-or max-time 0))
118
- within-utterance? (or (zero? max-time)
119
- (<= millis (+ max-time 2)))
120
- expected-says-ms? (and (> raw 10)
121
- raw-error
122
- millis-error
123
- (< millis-error raw-error)
124
- within-utterance?)
125
- max-time-says-ms? (and (> raw (+ max-time 2))
126
- (> raw 50)
127
- within-utterance?)]
128
- (if (or expected-says-ms? max-time-says-ms?)
129
- millis
130
- raw))))
131
-
132
- (defn choose-boundary-elapsed-sec [payload expected max-time]
133
- ;; Prefer Web Speech's elapsedTime after unit normalization. Some browsers
134
- ;; report 0 for later word boundaries, so LoomLarge also sends a monotonic
135
- ;; host clock. Use that host clock only when the provider clock is clearly not
136
- ;; advancing; otherwise provider timing remains the authority.
137
- (let [observed (normalize-observed-elapsed-sec (:observedElapsedSec payload)
138
- expected
139
- max-time)
140
- host (when (state/finite-number? (:hostElapsedSec payload))
141
- (max 0 (:hostElapsedSec payload)))
142
- expected-start (when expected (:startSec expected))]
143
- (cond
144
- (and host
145
- observed
146
- (state/finite-number? expected-start)
147
- (> expected-start 0.08)
148
- (< observed 0.001)
149
- (> host 0.001))
150
- host
151
-
152
- observed
153
- observed
154
-
155
- host
156
- host
157
-
158
- :else
159
- nil)))
160
-
161
- (defn payload-duration-sec [payload]
162
- (or (when (state/finite-number? (:durationSec payload))
163
- (max 0 (:durationSec payload)))
164
- (when (state/finite-number? (:totalDurationMs payload))
165
- (/ (max 0 (:totalDurationMs payload)) 1000))))
166
-
167
- (defn timing-duration-sec [word-timings]
168
- (when (seq word-timings)
169
- (apply max 0 (map :endSec (state/normalize-word-timings word-timings)))))
170
-
171
- (defn prepare-text-timeline [dictionary text speech-rate payload]
172
- (let [source (or (:source payload) "text")
173
- events (visemes/text->visemes dictionary text speech-rate)
174
- generated-word-timings (visemes/text->word-timings
175
- dictionary text speech-rate)
176
- supplied-word-timings (state/normalize-word-timings (:wordTimings payload))
177
- base-duration-ms (visemes/timeline-duration-ms events)
178
- explicit-duration-sec (or (payload-duration-sec payload)
179
- (timing-duration-sec supplied-word-timings))
180
- web-speech-duration-sec (when (= "webSpeech" source)
181
- (/ (visemes/web-speech-duration-ms text speech-rate base-duration-ms)
182
- 1000))
183
- target-duration-sec (or explicit-duration-sec
184
- web-speech-duration-sec
185
- (/ base-duration-ms 1000))
186
- base-duration-sec (/ base-duration-ms 1000)
187
- scale (if (and (pos? base-duration-sec)
188
- (state/finite-number? target-duration-sec)
189
- (> target-duration-sec 0))
190
- (/ target-duration-sec base-duration-sec)
191
- 1)
192
- should-scale? (> (js/Math.abs (- scale 1)) 0.02)
193
- word-timings (if (seq supplied-word-timings)
194
- supplied-word-timings
195
- (if should-scale?
196
- (visemes/scale-word-timings generated-word-timings scale)
197
- generated-word-timings))]
198
- {:name (:name payload)
199
- :text text
200
- :source source
201
- :visemes (if should-scale?
202
- (visemes/scale-events events scale)
203
- events)
204
- :wordTimings word-timings
205
- :durationSec (if (state/finite-number? target-duration-sec)
206
- target-duration-sec
207
- base-duration-sec)}))
208
-
209
- (defn timeline-name [timeline]
210
- (or (:name timeline)
211
- (str "lipSync_timeline_" (.now js/Date))))
212
-
213
- (defn create-lipsync-agency
214
- ([config]
215
- (create-lipsync-agency config nil))
216
- ([config runtime]
217
- (let [input-stream (stream/create-stream)
218
- event-stream (stream/create-stream)
219
- state-atom (atom (state/config->state config))
220
- disposed? (atom false)
221
- emit-input (:emit input-stream)
222
- emit-event (:emit event-stream)
223
- pronunciation-provider (pronunciation-effector/create-effector runtime)
224
- agency-scheduler (atom nil)]
225
- ;; Stream contract for this agency:
226
- ;; - input records accepted commands for tests/workers/debugging.
227
- ;; - events carry domain facts and cross-agency requests.
228
- (letfn [(audio-clock-sec [payload]
229
- (max 0
230
- (state/number-or (or (:audioTimeSec payload)
231
- (:currentTimeSec payload)
232
- (:offsetSec payload))
233
- 0)))
234
-
235
- (record-stop-state! [reason]
236
- (let [stopped-at (state/now-ms)]
237
- (swap! state-atom state/record-stop stopped-at reason)
238
- (emit-event {:type "lipSyncTimelineStopped"
239
- :agency "lipSync"
240
- :reason reason
241
- :stoppedAt stopped-at})))
242
-
243
- (stop-local! [reason remove?]
244
- (when-let [s @agency-scheduler]
245
- ((:stop-timeline s) reason remove?))
246
- (record-stop-state! reason))
247
-
248
- (finish-local! []
249
- (when (:speaking @state-atom)
250
- (stop-local! "completed" false)))
251
-
252
- (start-timeline! [timeline]
253
- (let [config (:config @state-atom)
254
- normalized (normalize-timeline timeline config)
255
- pending-audio-started-at (:audioStartedAt @state-atom)
256
- pending-audio-time-sec (:audioTimeSec @state-atom)]
257
- (if (empty? (:visemes normalized))
258
- (emit-event {:type "error"
259
- :agency "lipSync"
260
- :message "lipSync timeline requires at least one viseme event"})
261
- (do
262
- ;; One active utterance at a time keeps lip sync from
263
- ;; accumulating stale viseme snippets. State records the
264
- ;; replacement fact; the scheduler owns the animation
265
- ;; removal/start ordering for the old and new snippets.
266
- (when (:speaking @state-atom)
267
- (record-stop-state! "replaced"))
268
- (let [name (timeline-name normalized)
269
- built (if (:text normalized)
270
- (snippet/build-text-snippet (:text normalized)
271
- (:visemes normalized)
272
- config)
273
- (snippet/build-lipsync-snippet (:visemes normalized)
274
- config
275
- name))
276
- ;; Keep one snippet as the atomic animation unit for
277
- ;; an utterance. Individual visemes, jaw, and tongue
278
- ;; gestures are channels inside the snippet rather
279
- ;; than separately scheduled clips.
280
- snippet-data (cond-> (assoc built :name (or (:name normalized) (:name built)))
281
- (state/finite-number? (:durationSec normalized))
282
- (assoc :maxTime (max (:maxTime built) (:durationSec normalized))))
283
- started-at (state/now-ms)]
284
- (debug-log! "[Polymer LipSync CLJS] normalized timeline"
285
- {:name (:name snippet-data)
286
- :source (:source normalized)
287
- :visemeCount (count (:visemes normalized))
288
- :uniqueVisemeIds (->> (:visemes normalized)
289
- (map :visemeId)
290
- set
291
- sort
292
- vec)
293
- :visualLeadMs (:visualLeadMs normalized)
294
- :maxTime (:maxTime snippet-data)
295
- :visemes (viseme-event-summary (:visemes normalized))})
296
- (debug-log! "[Polymer LipSync CLJS] snippet channels"
297
- {:name (:name snippet-data)
298
- :channelCount (count (:channels snippet-data))
299
- :channels (channel-summary snippet-data)})
300
- (swap! state-atom state/record-start normalized (:name snippet-data) started-at (:maxTime snippet-data))
301
- (emit-event {:type "lipSyncTimelineStarted"
302
- :agency "lipSync"
303
- :name (:name snippet-data)
304
- :source (:source normalized)
305
- :text (:text normalized)
306
- :visemeCount (count (:visemes normalized))
307
- :maxTime (:maxTime snippet-data)
308
- :startedAt started-at})
309
- (when-let [s @agency-scheduler]
310
- ((:start-timeline s) snippet-data {:autoPlay true}))
311
- (when (and (= "webSpeech" (:source normalized))
312
- pending-audio-started-at)
313
- (let [elapsed-since-start
314
- (/ (max 0 (- started-at
315
- pending-audio-started-at))
316
- 1000)
317
- target-sec
318
- (min (:maxTime snippet-data)
319
- (+ (state/number-or
320
- pending-audio-time-sec 0)
321
- elapsed-since-start))]
322
- (swap! state-atom
323
- state/record-audio-started
324
- target-sec
325
- pending-audio-started-at)
326
- (when-let [s @agency-scheduler]
327
- ((:seek s)
328
- (:name snippet-data)
329
- target-sec
330
- "late-pronunciation-ready"))))
331
- snippet-data)))))
332
-
333
- (start-text! [payload]
334
- (let [text (:text payload)
335
- agency-scheduler @agency-scheduler]
336
- (if (and (string? text) (pos? (count text)))
337
- ((:prepare-text agency-scheduler) payload)
338
- (emit-event {:type "error"
339
- :agency "lipSync"
340
- :message "lipSync startText command requires text"}))))
341
-
342
- (process-azure! [payload]
343
- ;; Compatibility command for Azure/LiveKit provider facts. The
344
- ;; Azure-specific viseme-id mapping lives in TTS; LipSync only
345
- ;; normalizes word timing metadata and then uses the shared
346
- ;; articulated timeline path.
347
- (let [raw-options (or (:options payload) {})
348
- word-timings (state/normalize-word-timings
349
- (or (:wordTimings payload)
350
- (:wordTimings raw-options)))
351
- options (assoc (merge {:visualLeadMs (get-in @state-atom [:config :visualLeadMs])}
352
- raw-options)
353
- :wordTimings word-timings)
354
- timeline (azure/azure-visemes->timeline (:visemes payload)
355
- (:totalDurationMs payload)
356
- options)]
357
- (start-timeline! {:name (:name payload)
358
- :text (:text payload)
359
- :source (or (:source payload) "azure")
360
- :visemes timeline
361
- :visualLeadMs (:visualLeadMs options)
362
- :wordTimings (:wordTimings options)
363
- :durationSec (when (state/finite-number? (:totalDurationMs payload))
364
- (/ (:totalDurationMs payload) 1000))})))
365
-
366
- (handle-word-boundary! [payload]
367
- (let [word (:word payload)
368
- word-index (int (state/number-or (:wordIndex payload) (:wordIndex @state-atom)))
369
- observed-at (state/now-ms)]
370
- (swap! state-atom state/record-word-boundary word word-index observed-at)
371
- (emit-event {:type "lipSyncWordBoundary"
372
- :agency "lipSync"
373
- :word word
374
- :wordIndex word-index
375
- :observedAt observed-at})
376
- (let [current @state-atom
377
- expected (get (:wordTimings current) word-index)
378
- observed-elapsed-sec (choose-boundary-elapsed-sec payload
379
- expected
380
- (:maxTime current))
381
- elapsed-sec (state/number-or observed-elapsed-sec
382
- (if (:startTime current)
383
- (/ (- observed-at (:startTime current)) 1000)
384
- 0))
385
- drift-sec (when expected (- elapsed-sec (:startSec expected)))
386
- threshold (get-in current [:config :wordDriftThresholdSec])
387
- name (:snippetName current)]
388
- (when (and expected name drift-sec (> (js/Math.abs drift-sec) threshold))
389
- (let [target-sec (min (:maxTime current) (max 0 elapsed-sec))
390
- corrected-at (state/now-ms)]
391
- (swap! state-atom state/record-sync-correction name target-sec corrected-at)
392
- (emit-event {:type "lipSyncSyncDrift"
393
- :agency "lipSync"
394
- :name name
395
- :word word
396
- :wordIndex word-index
397
- :expectedSec (:startSec expected)
398
- :observedSec elapsed-sec
399
- :driftSec drift-sec
400
- :targetSec target-sec
401
- :correctedAt corrected-at})
402
- (when-let [s @agency-scheduler]
403
- ((:seek s) name target-sec "word-boundary-drift")))))))
404
-
405
- (handle-audio-started! [payload]
406
- ;; The host owns audio playback. This command tells LipSync when
407
- ;; that side effect actually started so Animation can align the
408
- ;; scheduled snippet to the audio clock without LoomLarge calling
409
- ;; Animation directly.
410
- (let [current @state-atom
411
- name (or (:name payload) (:snippetName current))
412
- raw-audio-time-sec (audio-clock-sec payload)
413
- audio-time-sec (if (:speaking current)
414
- (min (:maxTime current)
415
- raw-audio-time-sec)
416
- raw-audio-time-sec)
417
- observed-at (state/now-ms)]
418
- (swap! state-atom state/record-audio-started audio-time-sec observed-at)
419
- (emit-event {:type "lipSyncAudioStarted"
420
- :agency "lipSync"
421
- :name name
422
- :audioTimeSec audio-time-sec
423
- :observedAt observed-at})
424
- (when (and (:speaking current)
425
- (:snippetName current)
426
- name)
427
- (when-let [s @agency-scheduler]
428
- ((:seek s) name audio-time-sec "audio-started")))))
429
-
430
- (handle-audio-time! [payload]
431
- ;; Optional low-frequency host clock correction. This is not a
432
- ;; frame loop; callers should send it only when a material drift
433
- ;; correction is needed.
434
- (let [current @state-atom
435
- name (or (:name payload) (:snippetName current))
436
- raw-audio-time-sec (audio-clock-sec payload)
437
- audio-time-sec (if (:speaking current)
438
- (min (:maxTime current)
439
- raw-audio-time-sec)
440
- raw-audio-time-sec)
441
- observed-at (state/now-ms)]
442
- (swap! state-atom state/record-audio-time audio-time-sec observed-at)
443
- (emit-event {:type "lipSyncAudioTime"
444
- :agency "lipSync"
445
- :name name
446
- :audioTimeSec audio-time-sec
447
- :observedAt observed-at})
448
- (when name
449
- (when-let [s @agency-scheduler]
450
- ((:seek s) name audio-time-sec "audio-time")))))
451
-
452
- (update-word-timings! [payload]
453
- (let [updated-at (state/now-ms)
454
- timings (:wordTimings payload)]
455
- (swap! state-atom state/record-word-timings timings updated-at)
456
- (emit-event {:type "lipSyncWordTimingsUpdated"
457
- :agency "lipSync"
458
- :count (count (state/normalize-word-timings timings))
459
- :updatedAt updated-at})))
460
-
461
- (run-action! [action payload]
462
- ;; `goap/plan-command` decides which agency operation is allowed
463
- ;; for a command. Dispatch only echoes the input and reports the
464
- ;; plan; accepted actions run here so the command router does not
465
- ;; grow another copy of LipSync's state/scheduler policy.
466
- (case (:op action)
467
- "configure"
468
- (do
469
- (swap! state-atom state/configure (:config payload))
470
- (emit-event {:type "lipSyncConfigChanged"
471
- :agency "lipSync"
472
- :state @state-atom}))
473
-
474
- "start-text"
475
- (start-text! payload)
476
-
477
- "start-timeline"
478
- (start-timeline! (:timeline payload))
479
-
480
- "process-provider-visemes"
481
- (process-azure! payload)
482
-
483
- "record-word-boundary"
484
- (handle-word-boundary! payload)
485
-
486
- "align-audio-start"
487
- (handle-audio-started! payload)
488
-
489
- "align-audio-time"
490
- (handle-audio-time! payload)
491
-
492
- "update-word-timings"
493
- (update-word-timings! payload)
494
-
495
- "stop"
496
- (stop-local! (:reason action) (:remove? action))
497
-
498
- "reset-state"
499
- (do
500
- (reset! state-atom (state/config->state nil))
501
- (emit-event {:type "lipSyncConfigChanged"
502
- :agency "lipSync"
503
- :state @state-atom}))
504
-
505
- (emit-event {:type "error"
506
- :agency "lipSync"
507
- :message (str "Unknown LipSync planner action: " (:op action))})))
508
-
509
- (dispatch! [command]
510
- (when-not @disposed?
511
- (let [payload (js->clj command :keywordize-keys true)
512
- plan (goap/plan-command payload {:speaking (:speaking @state-atom)})]
513
- ;; Every command produces an input echo and a plan event before
514
- ;; any mutation. That makes failed/ignored commands observable
515
- ;; through the event stream instead of mutable state polling.
516
- (emit-input {:type "command"
517
- :agency "lipSync"
518
- :command payload})
519
- (emit-event {:type "lipSyncPlanCreated"
520
- :agency "lipSync"
521
- :plan plan})
522
- (if-not (:ok plan)
523
- (emit-event {:type "error"
524
- :agency "lipSync"
525
- :message (plan-failure-message plan)})
526
- (doseq [action (:actions plan)]
527
- (run-action! action payload))))))]
528
- (reset! agency-scheduler
529
- (scheduler/create-scheduler
530
- {:emit-event emit-event
531
- :on-finished finish-local!
532
- :pronunciation-effector pronunciation-provider
533
- :on-text-ready
534
- (fn [payload dictionary]
535
- (when-not @disposed?
536
- (let [text (:text payload)
537
- speech-rate (get-in @state-atom
538
- [:config :speechRate])]
539
- (start-timeline!
540
- (prepare-text-timeline
541
- dictionary text speech-rate payload)))))}))
542
- ((:preload-pronunciations @agency-scheduler))
543
- #js {:dispatch dispatch!
544
- :snapshot (fn [] (state/visible-state @state-atom))
545
- :schedulerQueue (fn []
546
- (clj->js (if-let [s @agency-scheduler]
547
- ((:queue s))
548
- [])))
549
- :input (stream/writable-port input-stream dispatch!)
550
- :events (stream/readable-port event-stream)
551
- ;; Empty compatibility stream. Cross-agency animation requests are
552
- ;; domain events so the Polymer agency system can route them to
553
- ;; Animation without host UI code becoming the interpreter.
554
- :subscribeInput (fn [listener] ((:subscribe input-stream) listener))
555
- :subscribeEvents (fn [listener] ((:subscribe event-stream) listener))
556
- :subscribe (fn [listener] ((:subscribe event-stream) listener))
557
- :subscribeStatus (fn [listener] ((:subscribe event-stream) listener))
558
- :configure (fn [next-config] (dispatch! #js {:type "configure" :config next-config}))
559
- :startText (fn [text] (dispatch! #js {:type "startText" :text text}))
560
- :startTimeline (fn [timeline] (dispatch! #js {:type "startTimeline" :timeline timeline}))
561
- :processAzureVisemes (fn
562
- ([visemes] (dispatch! #js {:type "processAzureVisemes" :visemes visemes}))
563
- ([visemes total-duration-ms]
564
- (dispatch! #js {:type "processAzureVisemes"
565
- :visemes visemes
566
- :totalDurationMs total-duration-ms})))
567
- :wordBoundary (fn
568
- ([word] (dispatch! #js {:type "wordBoundary" :word word}))
569
- ([word word-index observed-elapsed-sec]
570
- (dispatch! #js {:type "wordBoundary"
571
- :word word
572
- :wordIndex word-index
573
- :observedElapsedSec observed-elapsed-sec})))
574
- :audioStarted (fn
575
- ([] (dispatch! #js {:type "audioStarted"}))
576
- ([audio-time-sec]
577
- (dispatch! #js {:type "audioStarted"
578
- :audioTimeSec audio-time-sec})))
579
- :audioTime (fn [audio-time-sec]
580
- (dispatch! #js {:type "audioTime"
581
- :audioTimeSec audio-time-sec}))
582
- :updateWordTimings (fn [word-timings]
583
- (dispatch! #js {:type "updateWordTimings"
584
- :wordTimings word-timings}))
585
- :stop (fn [] (dispatch! #js {:type "stop"}))
586
- :reset (fn [] (dispatch! #js {:type "reset"}))
587
- :dispose (fn []
588
- (when-not @disposed?
589
- (reset! disposed? true)
590
- (when-let [s @agency-scheduler]
591
- ((:dispose s)))
592
- ((:dispose input-stream))
593
- ((:dispose event-stream))))}))))